feat(reports): delete reports and configure retention (#3338)

This commit is contained in:
Hampus authored and GitHub committed 2026-10-09 01:41:47 +02:00
1 parent 93d1f8af46
commit 82abd03a37
106 files changed
+1457 -170

No files matched your search

+5
View File
@@ -448,6 +448,11 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
attachmentDecayEnabled: master.attachment_decay_enabled,
deletionGracePeriodHours: master.dev.test_mode_enabled ? 0.01 : master.deletion_grace_period_hours,
inactivityDeletionThresholdDays: master.inactivity_deletion_threshold_days,
reportRetention: {
days: master.report_retention.days,
resolvedDays: master.report_retention.resolved_days,
dryRun: master.report_retention.dry_run,
},
push: {
publicVapidKey: master.auth.vapid.public_key,
apns: {
@@ -8,6 +8,8 @@ import {RateLimitMiddleware} from '@app/api/middleware/RateLimitMiddleware';
import {OpenAPI} from '@app/api/middleware/ResponseTypeMiddleware';
import {getReportRepository} from '@app/api/middleware/ServiceSingletons';
import {RateLimitConfigs} from '@app/api/RateLimitConfig';
import {deleteReportNow} from '@app/api/report/ReportDeletion';
import {getReportSearchService} from '@app/api/SearchFactory';
import type {HonoApp} from '@app/api/types/HonoEnv';
import {Validator} from '@app/api/Validator';
import {AdminACLs} from '@fluxer/constants/src/AdminACLs';
@@ -241,7 +243,7 @@ export function ReportAdminController(app: HonoApp) {
operationId: 'set_admin_report_legal_hold',
summary: 'Set report legal hold',
description:
'Places or clears a legal hold on a report. Reports are deleted 365 days after they were filed, together with their stored evidence, unless a hold that ends later is in place. A hold needs an end time in the future and a reason. A null legal_hold_until clears the hold. Creates an audit log entry. Requires REPORT_RESOLVE permission.',
'Places or clears a legal hold on a report. A report is deleted together with its stored evidence when its retention period ends, unless a hold that ends later is in place, and a held report cannot be deleted through DELETE /admin/reports/{report_id} either. A hold needs an end time in the future and a reason. A null legal_hold_until clears the hold. Creates an audit log entry. Requires REPORT_RESOLVE permission.',
responseSchema: ReportLegalHoldResponse,
statusCode: 200,
security: 'adminApiKey',
@@ -289,4 +291,48 @@ export function ReportAdminController(app: HonoApp) {
});
},
);
app.delete(
'/admin/reports/:report_id',
RateLimitMiddleware(RateLimitConfigs.ADMIN_LOOKUP),
requireAdminACL(AdminACLs.REPORT_DELETE),
Validator('param', ReportIdParam),
OpenAPI({
operationId: 'delete_admin_report',
summary: 'Delete report',
description:
'Deletes a report now: its record with the captured message context and profile snapshot, its stored evidence copies, its search index entry, and the duplicate-report reservation it holds for the reporter. An evidence copy that another report also uses is kept. A report under a legal hold that has not ended is refused with REPORT_UNDER_LEGAL_HOLD. This cannot be undone. Creates an audit log entry. Requires REPORT_DELETE permission.',
responseSchema: null,
statusCode: 204,
security: 'adminApiKey',
tags: 'Admin',
}),
async (ctx) => {
const {report_id} = ctx.req.valid('param');
const reportId = createReportID(report_id);
const deleted = await deleteReportNow(
{
reportRepository: getReportRepository(),
storageService: ctx.get('storageService'),
reportSearchService: getReportSearchService(),
},
reportId,
new Date(),
);
await ctx.get('adminService').auditService.createAuditLog({
adminUserId: ctx.get('adminUserId'),
targetType: 'report',
targetId: BigInt(reportId),
action: 'delete_report',
auditLogReason: ctx.get('auditLogReason'),
metadata: new Map([
['report_id', reportId.toString()],
['report_type', deleted.report.reportType.toString()],
['status', deleted.report.status.toString()],
['objects_deleted', deleted.objectsDeleted.toString()],
['shared_objects_kept', deleted.sharedObjectsKept.toString()],
]),
});
return ctx.body(null, 204);
},
);
}
@@ -32,6 +32,7 @@ const adminEndpoints: Array<AdminEndpointCase> = [
{method: 'GET', path: '/admin/reports', requiredACL: 'report:view'},
{method: 'GET', path: '/admin/reports/1', requiredACL: 'report:view'},
{method: 'PATCH', path: '/admin/reports/1', requiredACL: 'report:resolve'},
{method: 'DELETE', path: '/admin/reports/1', requiredACL: 'report:delete'},
{
method: 'POST',
path: '/admin/bulk-jobs',
@@ -0,0 +1,296 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {AdminAuditLog} from '@app/api/admin/IAdminRepository';
import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';
import {
createAttachmentID,
createChannelID,
createMessageID,
createReportID,
createUserID,
type ReportID,
} from '@app/api/BrandedTypes';
import {Config} from '@app/api/Config';
import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';
import {
IAR_SUBMISSION_COLUMNS,
type IARMessageContextRow,
type IARSubmissionRow,
} from '@app/api/database/types/ReportTypes';
import {getAdminRepository} from '@app/api/middleware/ServiceSingletons';
import {ReportRepository} from '@app/api/report/ReportRepository';
import {getReportSearchService} from '@app/api/SearchFactory';
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
import {HTTP_STATUS} from '@app/api/test/TestConstants';
import {createBuilder} from '@app/api/test/TestRequestBuilder';
import {AdminACLs} from '@fluxer/constants/src/AdminACLs';
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
import {serializeReportProfileSnapshot} from '@fluxer/schema/src/domains/report/ReportProfileSnapshotSchemas';
import {ms} from 'itty-time';
import {afterEach, beforeEach, describe, expect, test} from 'vitest';
const REPORTS_BUCKET = Config.s3.buckets.reports;
const AUDIT_REASON = 'Erasure request 77';
let sequence = 1_498_000_000_000_000_000n;
function nextId(): bigint {
sequence += 1n;
return sequence;
}
interface Evidence {
channelId: bigint;
attachmentId: bigint;
filename: string;
}
interface SeededReport {
reportId: ReportID;
reporterId: bigint;
channelId: bigint;
messageId: bigint;
attachmentKey: string;
profileKey: string;
}
function evidenceKey(evidence: Evidence): string {
return makeAttachmentCdnKey(createChannelID(evidence.channelId), evidence.attachmentId, evidence.filename);
}
function contextRow(evidence: Evidence, authorId: bigint): IARMessageContextRow {
return {
message_id: nextId(),
channel_id: evidence.channelId,
author_id: authorId,
webhook_id: null,
author_username: 'reported',
author_discriminator: 1,
author_avatar_hash: null,
content: 'the reported message',
timestamp: new Date(),
edited_timestamp: null,
type: 0,
flags: 0,
mention_everyone: false,
mention_users: null,
mention_roles: null,
mention_channels: null,
attachments: [
{
attachment_id: createAttachmentID(evidence.attachmentId),
filename: evidence.filename,
size: 1024n,
title: null,
description: null,
width: 64,
height: 64,
content_type: 'image/png',
content_hash: null,
placeholder: null,
flags: 0,
duration: null,
nsfw: false,
waveform: null,
},
],
embeds: null,
sticker_items: null,
};
}
describe('DELETE /admin/reports/:report_id', () => {
let harness: ApiTestHarness;
let admin: TestAccount;
let repository: ReportRepository;
beforeEach(async () => {
harness = await createApiTestHarness({search: 'enabled'});
admin = await setUserACLs(harness, await createTestAccount(harness), [
'admin:authenticate',
AdminACLs.REPORT_VIEW,
AdminACLs.REPORT_DELETE,
]);
repository = new ReportRepository();
});
afterEach(async () => {
await harness?.shutdown();
});
async function seedMessageReport(
evidence: Evidence,
options: {legalHoldUntil?: Date | null} = {},
): Promise<SeededReport> {
const id = nextId();
const reportId = createReportID(id);
const reporterId = nextId();
const reportedUserId = nextId();
const messageId = nextId();
const attachmentKey = evidenceKey(evidence);
const profileKey = `reports/${reportId}/profile/user_avatar/abc${id}`;
await harness.storageService.uploadObject({bucket: REPORTS_BUCKET, key: attachmentKey, body: new Uint8Array([1])});
await harness.storageService.uploadObject({bucket: REPORTS_BUCKET, key: profileKey, body: new Uint8Array([2])});
const report = await repository.createReport({
...Object.fromEntries(IAR_SUBMISSION_COLUMNS.map((column) => [column, null])),
report_id: id,
reporter_id: reporterId,
reported_at: new Date(Date.now() - ms('1 day')),
status: 1,
resolved_at: new Date(),
report_type: 0,
category: 'spam',
reported_user_id: reportedUserId,
reported_channel_id: evidence.channelId,
reported_message_id: messageId,
message_context: [contextRow(evidence, reportedUserId)],
reported_profile_snapshot: serializeReportProfileSnapshot({
captured_at: new Date().toISOString(),
user: {
id: reportedUserId.toString(),
username: 'reported',
discriminator: 1,
global_name: null,
bio: null,
pronouns: null,
avatar: {hash: `abc${id}`, key: profileKey},
banner: null,
},
member: null,
guild: null,
}),
legal_hold_until: options.legalHoldUntil ?? null,
legal_hold_reason: options.legalHoldUntil ? 'Preservation request' : null,
} as IARSubmissionRow);
await getReportSearchService()!.indexReport(report);
const reserved = await repository.reserveMessageReportByReporter({
reporter_id: reporterId,
channel_id: evidence.channelId,
message_id: messageId,
report_id: id,
reported_at: report.reportedAt,
});
expect(reserved).toBe(true);
return {reportId, reporterId, channelId: evidence.channelId, messageId, attachmentKey, profileKey};
}
function newEvidence(): Evidence {
return {channelId: nextId(), attachmentId: nextId(), filename: 'evidence.png'};
}
async function indexedIds(): Promise<Array<string>> {
const result = await getReportSearchService()!.searchReports('', {}, {limit: 1000});
return result.hits.map((hit) => hit.id);
}
function reservationIsFree(seeded: SeededReport): Promise<boolean> {
return repository.reserveMessageReportByReporter({
reporter_id: createUserID(seeded.reporterId),
channel_id: createChannelID(seeded.channelId),
message_id: createMessageID(seeded.messageId),
report_id: createReportID(nextId()),
reported_at: new Date(),
});
}
function deleteRequest(account: TestAccount, reportId: ReportID | string) {
return createBuilder<{code?: string}>(harness, account.token)
.delete(`/admin/reports/${reportId}`)
.header('X-Audit-Log-Reason', AUDIT_REASON);
}
async function auditEntries(action: string): Promise<Array<AdminAuditLog>> {
const logs = await getAdminRepository().listAllAuditLogsPaginated(10000);
return logs.filter((log) => log.action === action);
}
test('deletes the record, message context, evidence, search entry and reservation, and records the reason', async () => {
const seeded = await seedMessageReport(newEvidence());
const report = await repository.getReport(seeded.reportId);
expect(report?.messageContext).toHaveLength(1);
expect(await indexedIds()).toContain(seeded.reportId.toString());
await deleteRequest(admin, seeded.reportId).expect(HTTP_STATUS.NO_CONTENT).execute();
expect(await repository.getReport(seeded.reportId)).toBeNull();
expect(harness.storageService.hasObject(REPORTS_BUCKET, seeded.attachmentKey)).toBe(false);
expect(harness.storageService.hasObject(REPORTS_BUCKET, seeded.profileKey)).toBe(false);
expect(await indexedIds()).not.toContain(seeded.reportId.toString());
expect(await reservationIsFree(seeded)).toBe(true);
const [entry, ...rest] = await auditEntries('delete_report');
expect(rest).toEqual([]);
expect(entry?.adminUserId.toString()).toBe(admin.userId);
expect(entry?.targetType).toBe('report');
expect(entry?.targetId.toString()).toBe(seeded.reportId.toString());
expect(entry?.auditLogReason).toBe(AUDIT_REASON);
expect(Object.fromEntries(entry?.metadata ?? [])).toEqual({
report_id: seeded.reportId.toString(),
report_type: '0',
status: '1',
objects_deleted: '2',
shared_objects_kept: '0',
});
});
test('keeps an evidence copy another report still uses', async () => {
const shared = newEvidence();
const deleted = await seedMessageReport(shared);
const other = await seedMessageReport(shared);
await deleteRequest(admin, deleted.reportId).expect(HTTP_STATUS.NO_CONTENT).execute();
expect(await repository.getReport(deleted.reportId)).toBeNull();
expect(harness.storageService.hasObject(REPORTS_BUCKET, deleted.profileKey)).toBe(false);
expect(harness.storageService.hasObject(REPORTS_BUCKET, evidenceKey(shared))).toBe(true);
expect(await repository.getReport(other.reportId)).not.toBeNull();
expect(harness.storageService.hasObject(REPORTS_BUCKET, other.profileKey)).toBe(true);
expect(await indexedIds()).toContain(other.reportId.toString());
expect(await reservationIsFree(other)).toBe(false);
const [entry] = await auditEntries('delete_report');
expect(Object.fromEntries(entry?.metadata ?? [])).toMatchObject({objects_deleted: '1', shared_objects_kept: '1'});
});
test('refuses a report under a legal hold and changes nothing', async () => {
const seeded = await seedMessageReport(newEvidence(), {legalHoldUntil: new Date(Date.now() + ms('30 days'))});
const response = await deleteRequest(admin, seeded.reportId).expect(HTTP_STATUS.CONFLICT).execute();
expect(response.code).toBe(APIErrorCodes.REPORT_UNDER_LEGAL_HOLD);
expect(await repository.getReport(seeded.reportId)).not.toBeNull();
expect(harness.storageService.hasObject(REPORTS_BUCKET, seeded.attachmentKey)).toBe(true);
expect(harness.storageService.hasObject(REPORTS_BUCKET, seeded.profileKey)).toBe(true);
expect(await indexedIds()).toContain(seeded.reportId.toString());
expect(await reservationIsFree(seeded)).toBe(false);
expect(await auditEntries('delete_report')).toEqual([]);
});
test('deletes a report whose legal hold has ended', async () => {
const seeded = await seedMessageReport(newEvidence(), {legalHoldUntil: new Date(Date.now() - ms('1 minute'))});
await deleteRequest(admin, seeded.reportId).expect(HTTP_STATUS.NO_CONTENT).execute();
expect(await repository.getReport(seeded.reportId)).toBeNull();
});
test('an unknown report gives UNKNOWN_REPORT and records nothing', async () => {
await deleteRequest(admin, '1499999999999999999')
.expect(HTTP_STATUS.NOT_FOUND, APIErrorCodes.UNKNOWN_REPORT)
.execute();
expect(await auditEntries('delete_report')).toEqual([]);
});
test('needs REPORT_DELETE, which REPORT_RESOLVE does not grant', async () => {
const seeded = await seedMessageReport(newEvidence());
const resolver = await setUserACLs(harness, await createTestAccount(harness), [
'admin:authenticate',
AdminACLs.REPORT_VIEW,
AdminACLs.REPORT_RESOLVE,
]);
await deleteRequest(resolver, seeded.reportId).expect(HTTP_STATUS.FORBIDDEN, 'MISSING_ACL').execute();
expect(await repository.getReport(seeded.reportId)).not.toBeNull();
expect(harness.storageService.hasObject(REPORTS_BUCKET, seeded.attachmentKey)).toBe(true);
expect(await auditEntries('delete_report')).toEqual([]);
});
});
@@ -70,7 +70,7 @@ describe('report legal hold', () => {
async function runRetention() {
return processReportRetention(
{reportRepository: repository, storageService: new MockStorageService(), reportSearchService: null},
{now: new Date(), dryRun: false},
{now: new Date(), dryRun: false, policy: {retentionDays: 365, resolvedRetentionDays: null}},
);
}
@@ -277,6 +277,28 @@ export const ReportAdminAuditCases: ReadonlyArray<AdminAuditCoverageCase> = [
};
},
},
{
method: 'DELETE',
route: '/admin/reports/:report_id',
async prepare(context) {
const {reportId} = await fileUserReport(context);
return {
request: {path: `/admin/reports/${reportId}`, expectStatus: 204},
expected: {
action: 'delete_report',
targetType: 'report',
targetId: reportId,
metadata: {
report_id: reportId,
report_type: '1',
status: '0',
objects_deleted: '0',
shared_objects_kept: '0',
},
},
};
},
},
{
method: 'POST',
route: '/admin/reports/:report_id/legal-hold',
+5
View File
@@ -312,6 +312,11 @@ export interface APIConfig {
presignedHarvestDownloadsEnabled: boolean;
attachmentDecayEnabled: boolean;
deletionGracePeriodHours: number;
reportRetention: {
days: number;
resolvedDays: number | null;
dryRun: boolean;
};
inactivityDeletionThresholdDays?: number;
push: {
publicVapidKey?: string;
+2
View File
@@ -39762,6 +39762,7 @@
"REPORT_ALREADY_RESOLVED",
"REPORT_BANNED",
"REPORT_FLOW_OUTDATED",
"REPORT_UNDER_LEGAL_HOLD",
"RESPONSE_VALIDATION_ERROR",
"RESOURCE_LOCKED",
"SERVICE_UNAVAILABLE",
@@ -40057,6 +40058,7 @@
"Report already resolved.",
"You've been banned from submitting reports.",
"The report form changed while you were filling it in. Start the report again.",
"This report is under a legal hold and can't be deleted.",
"Response validation failed: {errors}.",
"This resource is being modified. Please try again shortly.",
"Service unavailable.",
+151
View File
@@ -0,0 +1,151 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {ReportID} from '@app/api/BrandedTypes';
import {Config} from '@app/api/Config';
import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import type {IARSubmission, IReportRepository} from '@app/api/report/IReportRepository';
import type {IReportSearchService} from '@app/api/search/IReportSearchService';
import {ReportUnderLegalHoldError} from '@fluxer/errors/src/domains/moderation/ReportUnderLegalHoldError';
import {UnknownReportError} from '@fluxer/errors/src/domains/moderation/UnknownReportError';
import {listReportProfileSnapshotAssets} from '@fluxer/schema/src/domains/report/ReportProfileSnapshotSchemas';
const SCAN_PAGE_SIZE = 500;
export interface ReportDeletionDeps {
reportRepository: IReportRepository;
storageService: IStorageService;
reportSearchService: IReportSearchService | null;
}
export interface ReportEvidenceLedger {
retainedKeys: Set<string>;
deletedKeys: Set<string>;
}
export interface ReportDeletionResult {
objectsDeleted: number;
sharedObjectsKept: number;
}
export function isReportUnderLegalHold(report: IARSubmission, now: Date): boolean {
return report.legalHoldUntil != null && report.legalHoldUntil.getTime() > now.getTime();
}
export function referencedObjectKeys(report: IARSubmission): Array<string> {
const keys: Array<string> = [];
for (const message of report.messageContext ?? []) {
const channelId = message.channelId ?? report.reportedChannelId;
if (!channelId) continue;
for (const attachment of message.attachments) {
if (attachment.attachment_id == null || !attachment.filename) continue;
keys.push(makeAttachmentCdnKey(channelId, attachment.attachment_id, String(attachment.filename)));
}
}
for (const asset of listReportProfileSnapshotAssets(report.reportedProfileSnapshot)) {
if (asset.key) {
keys.push(asset.key);
}
}
return keys;
}
export async function forEachStoredReport(
reportRepository: IReportRepository,
visit: (report: IARSubmission) => void,
pageSize = SCAN_PAGE_SIZE,
): Promise<void> {
let cursor: ReportID | undefined;
while (true) {
const page = await reportRepository.listAllReportsPaginated(pageSize, cursor);
for (const report of page) {
visit(report);
}
if (page.length < pageSize) return;
cursor = page[page.length - 1]!.reportId;
}
}
async function storedObjectKeys(storageService: IStorageService, report: IARSubmission): Promise<Set<string>> {
const keys = new Set(referencedObjectKeys(report));
const listed = await storageService.listObjects({
bucket: Config.s3.buckets.reports,
prefix: `reports/${report.reportId}/profile/`,
});
for (const object of listed) {
keys.add(object.key);
}
return keys;
}
export async function deleteReportWithEvidence(
deps: ReportDeletionDeps,
report: IARSubmission,
ledger: ReportEvidenceLedger,
options: {now: Date; dryRun: boolean},
): Promise<ReportDeletionResult> {
const {reportRepository, storageService, reportSearchService} = deps;
const result: ReportDeletionResult = {objectsDeleted: 0, sharedObjectsKept: 0};
for (const key of await storedObjectKeys(storageService, report)) {
if (ledger.retainedKeys.has(key)) {
result.sharedObjectsKept++;
continue;
}
if (ledger.deletedKeys.has(key)) continue;
const metadata = await storageService.getObjectMetadata(Config.s3.buckets.reports, key);
if (metadata?.lastModified && metadata.lastModified.getTime() >= options.now.getTime()) {
ledger.retainedKeys.add(key);
result.sharedObjectsKept++;
continue;
}
if (!options.dryRun) {
await storageService.deleteObject(Config.s3.buckets.reports, key);
}
ledger.deletedKeys.add(key);
result.objectsDeleted++;
}
if (!options.dryRun) {
await reportSearchService?.deleteReport(report.reportId);
await reportRepository.deleteReport(report.reportId);
}
return result;
}
async function loadDeletableReport(
reportRepository: IReportRepository,
reportId: ReportID,
now: Date,
): Promise<IARSubmission> {
const report = await reportRepository.getReport(reportId);
if (!report) {
throw new UnknownReportError();
}
if (isReportUnderLegalHold(report, now)) {
throw new ReportUnderLegalHoldError();
}
return report;
}
export async function deleteReportNow(
deps: ReportDeletionDeps,
reportId: ReportID,
now: Date,
): Promise<{report: IARSubmission} & ReportDeletionResult> {
const {reportRepository} = deps;
await loadDeletableReport(reportRepository, reportId, now);
const retainedKeys = new Set<string>();
await forEachStoredReport(reportRepository, (other) => {
if (other.reportId === reportId) return;
for (const key of referencedObjectKeys(other)) {
retainedKeys.add(key);
}
});
const report = await loadDeletableReport(reportRepository, reportId, now);
const result = await deleteReportWithEvidence(
deps,
report,
{retainedKeys, deletedKeys: new Set()},
{now, dryRun: false},
);
return {report, ...result};
}
@@ -27,6 +27,7 @@ import {
type IARSubmissionRow,
type IReportRepository,
ReportStatus,
ReportType,
} from '@app/api/report/IReportRepository';
import {
DSAReportEmailVerifications,
@@ -196,11 +197,31 @@ export class ReportRepository implements IReportRepository {
if (!report) {
return;
}
await this.releaseReporterReservation(report);
await executeConditional(
IARSubmissions.conditionalDeleteByPk({report_id: reportId}, {report_type: report.reportType}),
);
}
private async releaseReporterReservation(report: IARSubmission): Promise<void> {
const {reporterId, reportId, reportedAt} = report;
if (!reporterId) {
return;
}
const base = {reporter_id: reporterId, report_id: reportId, reported_at: reportedAt};
if (report.reportType === ReportType.MESSAGE && report.reportedChannelId && report.reportedMessageId) {
await this.releaseMessageReportByReporter({
...base,
channel_id: report.reportedChannelId,
message_id: report.reportedMessageId,
});
} else if (report.reportType === ReportType.USER && report.reportedUserId) {
await this.releaseUserReportByReporter({...base, reported_user_id: report.reportedUserId});
} else if (report.reportType === ReportType.GUILD && report.reportedGuildId) {
await this.releaseGuildReportByReporter({...base, reported_guild_id: report.reportedGuildId});
}
}
private mapRowToSubmission(row: IARSubmissionRow): IARSubmission {
return {
reportId: createReportID(row.report_id),
@@ -2,31 +2,29 @@
import type {ReportID} from '@app/api/BrandedTypes';
import {Config} from '@app/api/Config';
import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import {Logger} from '@app/api/Logger';
import type {IARSubmission, IReportRepository} from '@app/api/report/IReportRepository';
import {type IARSubmission, ReportStatus} from '@app/api/report/IReportRepository';
import {
deleteReportWithEvidence,
forEachStoredReport,
isReportUnderLegalHold,
type ReportDeletionDeps,
referencedObjectKeys,
} from '@app/api/report/ReportDeletion';
import {getReportSearchService} from '@app/api/SearchFactory';
import type {IReportSearchService} from '@app/api/search/IReportSearchService';
import {readOptionalEnv} from '@app/api/utils/IntegerOptions';
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
import {listReportProfileSnapshotAssets} from '@fluxer/schema/src/domains/report/ReportProfileSnapshotSchemas';
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
import {ms} from 'itty-time';
const REPORT_RETENTION_DAYS = 365;
const REPORT_RETENTION_MS = REPORT_RETENTION_DAYS * ms('1 day');
const DEFAULT_SCAN_PAGE_SIZE = 500;
export interface ReportRetentionDeps {
reportRepository: IReportRepository;
storageService: IStorageService;
reportSearchService: IReportSearchService | null;
export interface ReportRetentionPolicy {
retentionDays: number;
resolvedRetentionDays: number | null;
}
export interface ReportRetentionOptions {
now: Date;
dryRun: boolean;
policy: ReportRetentionPolicy;
pageSize?: number;
}
@@ -43,63 +41,36 @@ export interface ReportRetentionSummary {
type ReportRetentionState = 'expired' | 'held' | 'retained';
export function resolveReportRetentionDryRun(raw: string | undefined): boolean {
const value = raw?.trim().toLowerCase();
return value !== 'false' && value !== '0';
function reportExpiresAt(report: IARSubmission, policy: ReportRetentionPolicy): number {
const reportedAt = report.reportedAt instanceof Date ? report.reportedAt.getTime() : Number.NaN;
const byAge = reportedAt + policy.retentionDays * ms('1 day');
if (policy.resolvedRetentionDays === null || report.status !== ReportStatus.RESOLVED) {
return byAge;
}
const resolvedAt = report.resolvedAt instanceof Date ? report.resolvedAt.getTime() : Number.NaN;
if (Number.isNaN(resolvedAt)) {
return byAge;
}
return Math.min(byAge, resolvedAt + policy.resolvedRetentionDays * ms('1 day'));
}
function reportRetentionState(report: IARSubmission, now: Date): ReportRetentionState {
const reportedAt = report.reportedAt instanceof Date ? report.reportedAt.getTime() : Number.NaN;
if (Number.isNaN(reportedAt) || reportedAt > now.getTime() - REPORT_RETENTION_MS) {
function reportRetentionState(report: IARSubmission, now: Date, policy: ReportRetentionPolicy): ReportRetentionState {
const expiresAt = reportExpiresAt(report, policy);
if (Number.isNaN(expiresAt) || expiresAt > now.getTime()) {
return 'retained';
}
if (report.legalHoldUntil && report.legalHoldUntil.getTime() > now.getTime()) {
if (isReportUnderLegalHold(report, now)) {
return 'held';
}
return 'expired';
}
function reportProfilePrefix(reportId: ReportID): string {
return `reports/${reportId}/profile/`;
}
function referencedObjectKeys(report: IARSubmission): Array<string> {
const keys: Array<string> = [];
for (const message of report.messageContext ?? []) {
const channelId = message.channelId ?? report.reportedChannelId;
if (!channelId) continue;
for (const attachment of message.attachments) {
if (attachment.attachment_id == null || !attachment.filename) continue;
keys.push(makeAttachmentCdnKey(channelId, attachment.attachment_id, String(attachment.filename)));
}
}
for (const asset of listReportProfileSnapshotAssets(report.reportedProfileSnapshot)) {
if (asset.key) {
keys.push(asset.key);
}
}
return keys;
}
async function storedObjectKeys(storageService: IStorageService, report: IARSubmission): Promise<Set<string>> {
const keys = new Set(referencedObjectKeys(report));
const listed = await storageService.listObjects({
bucket: Config.s3.buckets.reports,
prefix: reportProfilePrefix(report.reportId),
});
for (const object of listed) {
keys.add(object.key);
}
return keys;
}
export async function processReportRetention(
deps: ReportRetentionDeps,
deps: ReportDeletionDeps,
options: ReportRetentionOptions,
): Promise<ReportRetentionSummary> {
const {reportRepository, storageService, reportSearchService} = deps;
const {now, dryRun} = options;
const pageSize = options.pageSize ?? DEFAULT_SCAN_PAGE_SIZE;
const {reportRepository} = deps;
const {now, dryRun, policy} = options;
const summary: ReportRetentionSummary = {
dryRun,
scanned: 0,
@@ -112,15 +83,14 @@ export async function processReportRetention(
};
const retainedKeys = new Set<string>();
const expiredIds: Array<ReportID> = [];
let cursor: ReportID | undefined;
while (true) {
const page = await reportRepository.listAllReportsPaginated(pageSize, cursor);
for (const report of page) {
await forEachStoredReport(
reportRepository,
(report) => {
summary.scanned++;
const state = reportRetentionState(report, now);
const state = reportRetentionState(report, now, policy);
if (state === 'expired') {
expiredIds.push(report.reportId);
continue;
return;
}
if (state === 'held') {
summary.held++;
@@ -128,59 +98,47 @@ export async function processReportRetention(
for (const key of referencedObjectKeys(report)) {
retainedKeys.add(key);
}
}
if (page.length < pageSize) break;
cursor = page[page.length - 1]!.reportId;
}
},
options.pageSize,
);
summary.expired = expiredIds.length;
const deletedKeys = new Set<string>();
const ledger = {retainedKeys, deletedKeys: new Set<string>()};
for (const reportId of expiredIds) {
try {
const report = await reportRepository.getReport(reportId);
if (!report) continue;
if (reportRetentionState(report, now) !== 'expired') {
if (reportRetentionState(report, now, policy) !== 'expired') {
for (const key of referencedObjectKeys(report)) {
retainedKeys.add(key);
}
continue;
}
for (const key of await storedObjectKeys(storageService, report)) {
if (retainedKeys.has(key)) {
summary.sharedObjectsKept++;
continue;
}
if (deletedKeys.has(key)) continue;
const metadata = await storageService.getObjectMetadata(Config.s3.buckets.reports, key);
if (metadata?.lastModified && metadata.lastModified.getTime() >= now.getTime()) {
retainedKeys.add(key);
summary.sharedObjectsKept++;
continue;
}
if (!dryRun) {
await storageService.deleteObject(Config.s3.buckets.reports, key);
}
deletedKeys.add(key);
summary.objectsDeleted++;
}
if (!dryRun) {
await reportSearchService?.deleteReport(reportId);
await reportRepository.deleteReport(reportId);
}
const result = await deleteReportWithEvidence(deps, report, ledger, {now, dryRun});
summary.objectsDeleted += result.objectsDeleted;
summary.sharedObjectsKept += result.sharedObjectsKept;
summary.deleted++;
} catch (error) {
summary.failed++;
Logger.error({error, reportId: reportId.toString()}, 'Failed to delete an expired report');
}
}
Logger.info({...summary, retentionDays: REPORT_RETENTION_DAYS}, 'Processed report retention');
Logger.info(
{
...summary,
retentionDays: policy.retentionDays,
resolvedRetentionDays: policy.resolvedRetentionDays,
},
'Processed report retention',
);
return summary;
}
const expireReportSnapshots: WorkerTaskHandler = async () => {
const {reportRepository, storageService} = getWorkerDependencies();
const {days, resolvedDays, dryRun} = Config.reportRetention;
await processReportRetention(
{reportRepository, storageService, reportSearchService: getReportSearchService()},
{now: new Date(), dryRun: resolveReportRetentionDryRun(readOptionalEnv('FLUXER_REPORT_RETENTION_DRY_RUN'))},
{now: new Date(), dryRun, policy: {retentionDays: days, resolvedRetentionDays: resolvedDays}},
);
};
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createAttachmentID, createChannelID, createReportID, type ReportID} from '@app/api/BrandedTypes';
import {createAttachmentID, createChannelID, createReportID, createUserID, type ReportID} from '@app/api/BrandedTypes';
import {Config} from '@app/api/Config';
import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';
import type {MessageAttachment} from '@app/api/database/types/MessageTypes';
@@ -10,22 +10,28 @@ import {
type IARSubmissionRow,
} from '@app/api/database/types/ReportTypes';
import type {IARSubmission} from '@app/api/report/IReportRepository';
import type {ReportDeletionDeps} from '@app/api/report/ReportDeletion';
import {ReportRepository} from '@app/api/report/ReportRepository';
import type {IReportSearchService} from '@app/api/search/IReportSearchService';
import {MockStorageService} from '@app/api/test/mocks/MockStorageService';
import {NoopLogger} from '@app/api/test/mocks/NoopLogger';
import {InMemorySearchProvider} from '@app/api/test/search/InMemorySearchProvider';
import {
import expireReportSnapshots, {
processReportRetention,
type ReportRetentionDeps,
resolveReportRetentionDryRun,
type ReportRetentionOptions,
type ReportRetentionPolicy,
} from '@app/api/worker/tasks/ExpireReportSnapshots';
import {clearWorkerDependencies, setWorkerDependenciesForTest} from '@app/api/worker/WorkerContext';
import {serializeReportProfileSnapshot} from '@fluxer/schema/src/domains/report/ReportProfileSnapshotSchemas';
import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
import {ms} from 'itty-time';
import {beforeEach, describe, expect, it} from 'vitest';
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
const NOW = new Date('2027-06-01T12:00:00.000Z');
const DAY_MS = ms('1 day');
const REPORTS_BUCKET = Config.s3.buckets.reports;
const WORKER_HELPERS = {logger: new NoopLogger()} as unknown as WorkerTaskHelpers;
const DEFAULT_POLICY: ReportRetentionPolicy = {retentionDays: 365, resolvedRetentionDays: null};
let sequence = 1_480_000_000_000_000_000n;
function nextId(): bigint {
@@ -92,6 +98,7 @@ interface SeedOptions {
profile?: boolean;
legalHoldUntil?: Date | null;
status?: number;
resolvedDaysAgo?: number;
snapshotJson?: string;
}
@@ -113,7 +120,7 @@ describe('report retention', () => {
let repository: ReportRepository;
let storage: MockStorageService;
let search: IReportSearchService;
let deps: ReportRetentionDeps;
let deps: ReportDeletionDeps;
beforeEach(() => {
repository = new ReportRepository();
@@ -158,7 +165,8 @@ describe('report retention', () => {
...Object.fromEntries(IAR_SUBMISSION_COLUMNS.map((column) => [column, null])),
report_id: id,
reported_at: daysAgo(options.ageDays),
status: options.status ?? 0,
status: options.status ?? (options.resolvedDaysAgo === undefined ? 0 : 1),
resolved_at: options.resolvedDaysAgo === undefined ? null : daysAgo(options.resolvedDaysAgo),
report_type: 0,
category: 'other',
reported_channel_id: options.attachment?.channelId ?? null,
@@ -195,7 +203,7 @@ describe('report retention', () => {
const expired = await seed({ageDays: 365, attachment: newAttachment(), profile: true});
const kept = await seed({ageDays: 364, attachment: newAttachment(), profile: true});
const summary = await processReportRetention(deps, {now: NOW, dryRun: false});
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
await expectGone(expired);
expect(storage.hasObject(REPORTS_BUCKET, expired.attachmentKey!)).toBe(false);
@@ -216,7 +224,7 @@ describe('report retention', () => {
const pending = await seed({ageDays: 400, status: 0});
const resolved = await seed({ageDays: 400, status: 1});
await processReportRetention(deps, {now: NOW, dryRun: false});
await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
await expectGone(pending);
await expectGone(resolved);
@@ -227,7 +235,7 @@ describe('report retention', () => {
const expired = await seed({ageDays: 500, attachment: shared});
const retained = await seed({ageDays: 10, attachment: shared});
const summary = await processReportRetention(deps, {now: NOW, dryRun: false});
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
await expectGone(expired);
await expectKept(retained);
@@ -241,7 +249,7 @@ describe('report retention', () => {
const first = await seed({ageDays: 500, attachment: shared});
const second = await seed({ageDays: 450, attachment: shared});
const summary = await processReportRetention(deps, {now: NOW, dryRun: false});
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
await expectGone(first);
await expectGone(second);
@@ -261,7 +269,7 @@ describe('report retention', () => {
const holdEnded = await seed({ageDays: 500, profile: true, legalHoldUntil: daysAgo(1)});
const sharingWithHeld = await seed({ageDays: 500, attachment: shared});
const summary = await processReportRetention(deps, {now: NOW, dryRun: false});
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
await expectKept(held);
expect((await repository.getReport(held.reportId))?.legalHoldReason).toBe('Preservation request');
@@ -275,7 +283,7 @@ describe('report retention', () => {
const expired = await seed({ageDays: 366, attachment: newAttachment(), profile: true});
const kept = await seed({ageDays: 1, profile: true});
const summary = await processReportRetention(deps, {now: NOW, dryRun: true});
const summary = await processReportRetention(deps, {now: NOW, dryRun: true, policy: DEFAULT_POLICY});
await expectKept(expired);
await expectKept(kept);
@@ -305,7 +313,10 @@ describe('report retention', () => {
};
await expect(
processReportRetention({...deps, reportRepository: failingRepository}, {now: NOW, dryRun: false, pageSize: 2}),
processReportRetention(
{...deps, reportRepository: failingRepository},
{now: NOW, dryRun: false, policy: DEFAULT_POLICY, pageSize: 2},
),
).rejects.toThrow('read timeout');
await expectKept(expired);
@@ -321,7 +332,7 @@ describe('report retention', () => {
kept.push(await seed({ageDays: index, profile: true}));
}
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, pageSize: 3});
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY, pageSize: 3});
expect(summary).toMatchObject({scanned: 14, expired: 7, deleted: 7, objectsDeleted: 7, failed: 0});
for (const report of expired) await expectGone(report);
@@ -332,14 +343,14 @@ describe('report retention', () => {
const expired = await seed({ageDays: 400, attachment: newAttachment(), profile: true});
storage.configure({shouldFailDelete: true});
const failed = await processReportRetention(deps, {now: NOW, dryRun: false});
const failed = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
expect(failed).toMatchObject({expired: 1, deleted: 0, failed: 1});
expect(await repository.getReport(expired.reportId)).not.toBeNull();
expect(await indexedIds()).toContain(expired.reportId.toString());
storage.configure({shouldFailDelete: false});
const retried = await processReportRetention(deps, {now: NOW, dryRun: false});
const retried = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
expect(retried).toMatchObject({expired: 1, deleted: 1, failed: 0});
await expectGone(expired);
@@ -355,7 +366,7 @@ describe('report retention', () => {
const summary = await processReportRetention(
{...deps, reportSearchService: failingSearch},
{now: NOW, dryRun: false},
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
);
expect(summary).toMatchObject({deleted: 0, failed: 1});
@@ -365,7 +376,10 @@ describe('report retention', () => {
it('deletes rows when search is not configured', async () => {
const expired = await seed({ageDays: 400, profile: true});
const summary = await processReportRetention({...deps, reportSearchService: null}, {now: NOW, dryRun: false});
const summary = await processReportRetention(
{...deps, reportSearchService: null},
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
);
expect(summary).toMatchObject({deleted: 1, failed: 0});
expect(await repository.getReport(expired.reportId)).toBeNull();
@@ -394,7 +408,7 @@ describe('report retention', () => {
const summary = await processReportRetention(
{...deps, reportRepository: holdingRepository},
{now: NOW, dryRun: false},
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
);
await expectKept(first);
@@ -417,7 +431,7 @@ describe('report retention', () => {
const summary = await processReportRetention(
{...deps, storageService: rewritingStorage},
{now: NOW, dryRun: false},
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
);
await expectGone(expired);
@@ -428,22 +442,156 @@ describe('report retention', () => {
it('deletes profile images stored for a report whose snapshot cannot be read', async () => {
const expired = await seed({ageDays: 400, profile: true, snapshotJson: '{"not":"a snapshot"'});
const summary = await processReportRetention(deps, {now: NOW, dryRun: false});
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
expect(summary).toMatchObject({deleted: 1, objectsDeleted: 1});
await expectGone(expired);
});
it.each([
[undefined, true],
['', true],
['true', true],
['1', true],
['yes', true],
['false', false],
['FALSE', false],
[' 0 ', false],
])('FLUXER_REPORT_RETENTION_DRY_RUN=%j gives a dry run of %s', (raw, expected) => {
expect(resolveReportRetentionDryRun(raw)).toBe(expected);
function run(options: Partial<ReportRetentionOptions> = {}) {
return processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY, ...options});
}
it('deletes earlier when the retention period is shorter', async () => {
const expired = await seed({ageDays: 30, profile: true});
const kept = await seed({ageDays: 29, profile: true});
const summary = await run({policy: {retentionDays: 30, resolvedRetentionDays: null}});
await expectGone(expired);
await expectKept(kept);
expect(summary).toMatchObject({scanned: 2, expired: 1, deleted: 1});
});
it('deletes a resolved report once the resolved-report period has passed since it was resolved', async () => {
const policy = {retentionDays: 365, resolvedRetentionDays: 30};
const resolvedLongAgo = await seed({ageDays: 100, resolvedDaysAgo: 30, attachment: newAttachment()});
const resolvedRecently = await seed({ageDays: 100, resolvedDaysAgo: 29, profile: true});
const pending = await seed({ageDays: 100, profile: true});
const resolvedWithoutTime = await seed({ageDays: 100, status: 1, profile: true});
const summary = await run({policy});
await expectGone(resolvedLongAgo);
expect(storage.hasObject(REPORTS_BUCKET, resolvedLongAgo.attachmentKey!)).toBe(false);
await expectKept(resolvedRecently);
await expectKept(pending);
await expectKept(resolvedWithoutTime);
expect(summary).toMatchObject({scanned: 4, expired: 1, deleted: 1, held: 0});
});
it('applies whichever rule ends first', async () => {
const policy = {retentionDays: 100, resolvedRetentionDays: 300};
const byAge = await seed({ageDays: 100, resolvedDaysAgo: 10});
const neither = await seed({ageDays: 99, resolvedDaysAgo: 90});
await run({policy});
await expectGone(byAge);
await expectKept(neither);
});
it('keeps a held resolved report past the resolved-report period and keeps its evidence', async () => {
const shared = newAttachment();
const held = await seed({
ageDays: 50,
resolvedDaysAgo: 40,
attachment: shared,
profile: true,
legalHoldUntil: new Date(NOW.getTime() + DAY_MS),
});
const sharingWithHeld = await seed({ageDays: 50, resolvedDaysAgo: 40, attachment: shared});
const summary = await run({policy: {retentionDays: 365, resolvedRetentionDays: 7}});
await expectKept(held);
await expectGone(sharingWithHeld);
expect(storage.hasObject(REPORTS_BUCKET, attachmentKey(shared))).toBe(true);
expect(summary).toMatchObject({expired: 1, held: 1, deleted: 1, sharedObjectsKept: 1});
});
it('frees the reporter to report the same account again once the report is deleted', async () => {
const reporterId = nextId();
const reportedUserId = nextId();
const reportId = createReportID(nextId());
await repository.createReport({
...Object.fromEntries(IAR_SUBMISSION_COLUMNS.map((column) => [column, null])),
report_id: BigInt(reportId),
reporter_id: reporterId,
reported_user_id: reportedUserId,
reported_at: daysAgo(400),
status: 0,
report_type: 1,
category: 'harassment',
} as IARSubmissionRow);
const reservation = {
reporter_id: createUserID(reporterId),
reported_user_id: createUserID(reportedUserId),
report_id: reportId,
reported_at: daysAgo(400),
};
expect(await repository.reserveUserReportByReporter(reservation)).toBe(true);
await run();
expect(await repository.getReport(reportId)).toBeNull();
expect(await repository.reserveUserReportByReporter({...reservation, report_id: createReportID(nextId())})).toBe(
true,
);
});
describe('the daily task', () => {
const configured = {...Config.reportRetention};
beforeEach(() => {
setWorkerDependenciesForTest({reportRepository: repository, storageService: storage});
});
afterEach(() => {
vi.useRealTimers();
Object.assign(Config.reportRetention, configured);
clearWorkerDependencies();
});
async function runDailyTask() {
vi.useFakeTimers({now: NOW, toFake: ['Date']});
await expireReportSnapshots({}, WORKER_HELPERS);
}
it('deletes for real with the default settings', async () => {
expect(configured).toEqual({days: 365, resolvedDays: null, dryRun: false});
const expired = await seed({ageDays: 366, attachment: newAttachment(), profile: true});
const kept = await seed({ageDays: 364, profile: true});
await runDailyTask();
expect(await repository.getReport(expired.reportId)).toBeNull();
expect(storage.hasObject(REPORTS_BUCKET, expired.attachmentKey!)).toBe(false);
expect(storage.hasObject(REPORTS_BUCKET, expired.profileKey!)).toBe(false);
expect(await repository.getReport(kept.reportId)).not.toBeNull();
});
it('only counts what it would delete when FLUXER_REPORT_RETENTION_DRY_RUN is true', async () => {
Object.assign(Config.reportRetention, {dryRun: true});
const expired = await seed({ageDays: 366, attachment: newAttachment(), profile: true});
await runDailyTask();
expect(await repository.getReport(expired.reportId)).not.toBeNull();
expect(storage.deleteObjectSpy).not.toHaveBeenCalled();
});
it('reads both day settings from the config', async () => {
Object.assign(Config.reportRetention, {days: 60, resolvedDays: 5});
const byAge = await seed({ageDays: 60});
const byResolution = await seed({ageDays: 20, resolvedDaysAgo: 5});
const kept = await seed({ageDays: 20, resolvedDaysAgo: 4});
await runDailyTask();
expect(await repository.getReport(byAge.reportId)).toBeNull();
expect(await repository.getReport(byResolution.reportId)).toBeNull();
expect(await repository.getReport(kept.reportId)).not.toBeNull();
});
});
});