pepepper/fluxer
1
0 Fork 0
mirror of https://github.com/fluxerapp/fluxer synced 2026-10-18 13:35:42 +09:00
Code Issues Packages Projects Releases Wiki Activity

feat(reports): delete reports and configure retention (#3338)

Browse Source
This commit is contained in:
Hampus authored and GitHub committed 2026-10-09 01:41:47 +02:00
1 parent 93d1f8af46
commit 82abd03a37
106 files changed
+1457 -170
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

fluxer_api/src/api/worker/tasks/ExpireReportSnapshots.ts
+53 -95
View File
@@ -2,31 +2,29 @@
import type {ReportID} from '@app/api/BrandedTypes';
import {Config} from '@app/api/Config';
import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import {Logger} from '@app/api/Logger';
import type {IARSubmission, IReportRepository} from '@app/api/report/IReportRepository';
import {type IARSubmission, ReportStatus} from '@app/api/report/IReportRepository';
import {
deleteReportWithEvidence,
forEachStoredReport,
isReportUnderLegalHold,
type ReportDeletionDeps,
referencedObjectKeys,
} from '@app/api/report/ReportDeletion';
import {getReportSearchService} from '@app/api/SearchFactory';
import type {IReportSearchService} from '@app/api/search/IReportSearchService';
import {readOptionalEnv} from '@app/api/utils/IntegerOptions';
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
import {listReportProfileSnapshotAssets} from '@fluxer/schema/src/domains/report/ReportProfileSnapshotSchemas';
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
import {ms} from 'itty-time';
const REPORT_RETENTION_DAYS = 365;
const REPORT_RETENTION_MS = REPORT_RETENTION_DAYS * ms('1 day');
const DEFAULT_SCAN_PAGE_SIZE = 500;
export interface ReportRetentionDeps {
reportRepository: IReportRepository;
storageService: IStorageService;
reportSearchService: IReportSearchService | null;
export interface ReportRetentionPolicy {
retentionDays: number;
resolvedRetentionDays: number | null;
}
export interface ReportRetentionOptions {
now: Date;
dryRun: boolean;
policy: ReportRetentionPolicy;
pageSize?: number;
}
@@ -43,63 +41,36 @@ export interface ReportRetentionSummary {
type ReportRetentionState = 'expired' | 'held' | 'retained';
export function resolveReportRetentionDryRun(raw: string | undefined): boolean {
const value = raw?.trim().toLowerCase();
return value !== 'false' && value !== '0';
function reportExpiresAt(report: IARSubmission, policy: ReportRetentionPolicy): number {
const reportedAt = report.reportedAt instanceof Date ? report.reportedAt.getTime() : Number.NaN;
const byAge = reportedAt + policy.retentionDays * ms('1 day');
if (policy.resolvedRetentionDays === null || report.status !== ReportStatus.RESOLVED) {
return byAge;
}
const resolvedAt = report.resolvedAt instanceof Date ? report.resolvedAt.getTime() : Number.NaN;
if (Number.isNaN(resolvedAt)) {
return byAge;
}
return Math.min(byAge, resolvedAt + policy.resolvedRetentionDays * ms('1 day'));
}
function reportRetentionState(report: IARSubmission, now: Date): ReportRetentionState {
const reportedAt = report.reportedAt instanceof Date ? report.reportedAt.getTime() : Number.NaN;
if (Number.isNaN(reportedAt) || reportedAt > now.getTime() - REPORT_RETENTION_MS) {
function reportRetentionState(report: IARSubmission, now: Date, policy: ReportRetentionPolicy): ReportRetentionState {
const expiresAt = reportExpiresAt(report, policy);
if (Number.isNaN(expiresAt) || expiresAt > now.getTime()) {
return 'retained';
}
if (report.legalHoldUntil && report.legalHoldUntil.getTime() > now.getTime()) {
if (isReportUnderLegalHold(report, now)) {
return 'held';
}
return 'expired';
}
function reportProfilePrefix(reportId: ReportID): string {
return `reports/${reportId}/profile/`;
}
function referencedObjectKeys(report: IARSubmission): Array<string> {
const keys: Array<string> = [];
for (const message of report.messageContext ?? []) {
const channelId = message.channelId ?? report.reportedChannelId;
if (!channelId) continue;
for (const attachment of message.attachments) {
if (attachment.attachment_id == null || !attachment.filename) continue;
keys.push(makeAttachmentCdnKey(channelId, attachment.attachment_id, String(attachment.filename)));
}
}
for (const asset of listReportProfileSnapshotAssets(report.reportedProfileSnapshot)) {
if (asset.key) {
keys.push(asset.key);
}
}
return keys;
}
async function storedObjectKeys(storageService: IStorageService, report: IARSubmission): Promise<Set<string>> {
const keys = new Set(referencedObjectKeys(report));
const listed = await storageService.listObjects({
bucket: Config.s3.buckets.reports,
prefix: reportProfilePrefix(report.reportId),
});
for (const object of listed) {
keys.add(object.key);
}
return keys;
}
export async function processReportRetention(
deps: ReportRetentionDeps,
deps: ReportDeletionDeps,
options: ReportRetentionOptions,
): Promise<ReportRetentionSummary> {
const {reportRepository, storageService, reportSearchService} = deps;
const {now, dryRun} = options;
const pageSize = options.pageSize ?? DEFAULT_SCAN_PAGE_SIZE;
const {reportRepository} = deps;
const {now, dryRun, policy} = options;
const summary: ReportRetentionSummary = {
dryRun,
scanned: 0,
@@ -112,15 +83,14 @@ export async function processReportRetention(
};
const retainedKeys = new Set<string>();
const expiredIds: Array<ReportID> = [];
let cursor: ReportID | undefined;
while (true) {
const page = await reportRepository.listAllReportsPaginated(pageSize, cursor);
for (const report of page) {
await forEachStoredReport(
reportRepository,
(report) => {
summary.scanned++;
const state = reportRetentionState(report, now);
const state = reportRetentionState(report, now, policy);
if (state === 'expired') {
expiredIds.push(report.reportId);
continue;
return;
}
if (state === 'held') {
summary.held++;
@@ -128,59 +98,47 @@ export async function processReportRetention(
for (const key of referencedObjectKeys(report)) {
retainedKeys.add(key);
}
}
if (page.length < pageSize) break;
cursor = page[page.length - 1]!.reportId;
}
},
options.pageSize,
);
summary.expired = expiredIds.length;
const deletedKeys = new Set<string>();
const ledger = {retainedKeys, deletedKeys: new Set<string>()};
for (const reportId of expiredIds) {
try {
const report = await reportRepository.getReport(reportId);
if (!report) continue;
if (reportRetentionState(report, now) !== 'expired') {
if (reportRetentionState(report, now, policy) !== 'expired') {
for (const key of referencedObjectKeys(report)) {
retainedKeys.add(key);
}
continue;
}
for (const key of await storedObjectKeys(storageService, report)) {
if (retainedKeys.has(key)) {
summary.sharedObjectsKept++;
continue;
}
if (deletedKeys.has(key)) continue;
const metadata = await storageService.getObjectMetadata(Config.s3.buckets.reports, key);
if (metadata?.lastModified && metadata.lastModified.getTime() >= now.getTime()) {
retainedKeys.add(key);
summary.sharedObjectsKept++;
continue;
}
if (!dryRun) {
await storageService.deleteObject(Config.s3.buckets.reports, key);
}
deletedKeys.add(key);
summary.objectsDeleted++;
}
if (!dryRun) {
await reportSearchService?.deleteReport(reportId);
await reportRepository.deleteReport(reportId);
}
const result = await deleteReportWithEvidence(deps, report, ledger, {now, dryRun});
summary.objectsDeleted += result.objectsDeleted;
summary.sharedObjectsKept += result.sharedObjectsKept;
summary.deleted++;
} catch (error) {
summary.failed++;
Logger.error({error, reportId: reportId.toString()}, 'Failed to delete an expired report');
}
}
Logger.info({...summary, retentionDays: REPORT_RETENTION_DAYS}, 'Processed report retention');
Logger.info(
{
...summary,
retentionDays: policy.retentionDays,
resolvedRetentionDays: policy.resolvedRetentionDays,
},
'Processed report retention',
);
return summary;
}
const expireReportSnapshots: WorkerTaskHandler = async () => {
const {reportRepository, storageService} = getWorkerDependencies();
const {days, resolvedDays, dryRun} = Config.reportRetention;
await processReportRetention(
{reportRepository, storageService, reportSearchService: getReportSearchService()},
{now: new Date(), dryRun: resolveReportRetentionDryRun(readOptionalEnv('FLUXER_REPORT_RETENTION_DRY_RUN'))},
{now: new Date(), dryRun, policy: {retentionDays: days, resolvedRetentionDays: resolvedDays}},
);
};