feat(reports): delete reports and configure retention (#3338)

fluxer_api/src/api/admin/controllers/ReportAdminController.ts

@@ -8,6 +8,8 @@ import {RateLimitMiddleware} from '@app/api/middleware/RateLimitMiddleware';
 import {OpenAPI} from '@app/api/middleware/ResponseTypeMiddleware';
 import {getReportRepository} from '@app/api/middleware/ServiceSingletons';
 import {RateLimitConfigs} from '@app/api/RateLimitConfig';
+import {deleteReportNow} from '@app/api/report/ReportDeletion';
+import {getReportSearchService} from '@app/api/SearchFactory';
 import type {HonoApp} from '@app/api/types/HonoEnv';
 import {Validator} from '@app/api/Validator';
 import {AdminACLs} from '@fluxer/constants/src/AdminACLs';
@@ -241,7 +243,7 @@ export function ReportAdminController(app: HonoApp) {
 			operationId: 'set_admin_report_legal_hold',
 			summary: 'Set report legal hold',
 			description:
-				'Places or clears a legal hold on a report. Reports are deleted 365 days after they were filed, together with their stored evidence, unless a hold that ends later is in place. A hold needs an end time in the future and a reason. A null legal_hold_until clears the hold. Creates an audit log entry. Requires REPORT_RESOLVE permission.',
+				'Places or clears a legal hold on a report. A report is deleted together with its stored evidence when its retention period ends, unless a hold that ends later is in place, and a held report cannot be deleted through DELETE /admin/reports/{report_id} either. A hold needs an end time in the future and a reason. A null legal_hold_until clears the hold. Creates an audit log entry. Requires REPORT_RESOLVE permission.',
 			responseSchema: ReportLegalHoldResponse,
 			statusCode: 200,
 			security: 'adminApiKey',
@@ -289,4 +291,48 @@ export function ReportAdminController(app: HonoApp) {
 			});
 		},
 	);
+	app.delete(
+		'/admin/reports/:report_id',
+		RateLimitMiddleware(RateLimitConfigs.ADMIN_LOOKUP),
+		requireAdminACL(AdminACLs.REPORT_DELETE),
+		Validator('param', ReportIdParam),
+		OpenAPI({
+			operationId: 'delete_admin_report',
+			summary: 'Delete report',
+			description:
+				'Deletes a report now: its record with the captured message context and profile snapshot, its stored evidence copies, its search index entry, and the duplicate-report reservation it holds for the reporter. An evidence copy that another report also uses is kept. A report under a legal hold that has not ended is refused with REPORT_UNDER_LEGAL_HOLD. This cannot be undone. Creates an audit log entry. Requires REPORT_DELETE permission.',
+			responseSchema: null,
+			statusCode: 204,
+			security: 'adminApiKey',
+			tags: 'Admin',
+		}),
+		async (ctx) => {
+			const {report_id} = ctx.req.valid('param');
+			const reportId = createReportID(report_id);
+			const deleted = await deleteReportNow(
+				{
+					reportRepository: getReportRepository(),
+					storageService: ctx.get('storageService'),
+					reportSearchService: getReportSearchService(),
+				},
+				reportId,
+				new Date(),
+			);
+			await ctx.get('adminService').auditService.createAuditLog({
+				adminUserId: ctx.get('adminUserId'),
+				targetType: 'report',
+				targetId: BigInt(reportId),
+				action: 'delete_report',
+				auditLogReason: ctx.get('auditLogReason'),
+				metadata: new Map([
+					['report_id', reportId.toString()],
+					['report_type', deleted.report.reportType.toString()],
+					['status', deleted.report.status.toString()],
+					['objects_deleted', deleted.objectsDeleted.toString()],
+					['shared_objects_kept', deleted.sharedObjectsKept.toString()],
+				]),
+			});
+			return ctx.body(null, 204);
+		},
+	);
 }

fluxer_api/src/api/admin/tests/AdminEndpointsAuthorization.test.ts

@@ -32,6 +32,7 @@ const adminEndpoints: Array<AdminEndpointCase> = [
 	{method: 'GET', path: '/admin/reports', requiredACL: 'report:view'},
 	{method: 'GET', path: '/admin/reports/1', requiredACL: 'report:view'},
 	{method: 'PATCH', path: '/admin/reports/1', requiredACL: 'report:resolve'},
+	{method: 'DELETE', path: '/admin/reports/1', requiredACL: 'report:delete'},
 	{
 		method: 'POST',
 		path: '/admin/bulk-jobs',

fluxer_api/src/api/admin/tests/AdminReportDelete.test.ts

@@ -0,0 +1,296 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import type {AdminAuditLog} from '@app/api/admin/IAdminRepository';
+import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';
+import {
+	createAttachmentID,
+	createChannelID,
+	createMessageID,
+	createReportID,
+	createUserID,
+	type ReportID,
+} from '@app/api/BrandedTypes';
+import {Config} from '@app/api/Config';
+import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';
+import {
+	IAR_SUBMISSION_COLUMNS,
+	type IARMessageContextRow,
+	type IARSubmissionRow,
+} from '@app/api/database/types/ReportTypes';
+import {getAdminRepository} from '@app/api/middleware/ServiceSingletons';
+import {ReportRepository} from '@app/api/report/ReportRepository';
+import {getReportSearchService} from '@app/api/SearchFactory';
+import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
+import {HTTP_STATUS} from '@app/api/test/TestConstants';
+import {createBuilder} from '@app/api/test/TestRequestBuilder';
+import {AdminACLs} from '@fluxer/constants/src/AdminACLs';
+import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
+import {serializeReportProfileSnapshot} from '@fluxer/schema/src/domains/report/ReportProfileSnapshotSchemas';
+import {ms} from 'itty-time';
+import {afterEach, beforeEach, describe, expect, test} from 'vitest';
+
+const REPORTS_BUCKET = Config.s3.buckets.reports;
+const AUDIT_REASON = 'Erasure request 77';
+let sequence = 1_498_000_000_000_000_000n;
+
+function nextId(): bigint {
+	sequence += 1n;
+	return sequence;
+}
+
+interface Evidence {
+	channelId: bigint;
+	attachmentId: bigint;
+	filename: string;
+}
+
+interface SeededReport {
+	reportId: ReportID;
+	reporterId: bigint;
+	channelId: bigint;
+	messageId: bigint;
+	attachmentKey: string;
+	profileKey: string;
+}
+
+function evidenceKey(evidence: Evidence): string {
+	return makeAttachmentCdnKey(createChannelID(evidence.channelId), evidence.attachmentId, evidence.filename);
+}
+
+function contextRow(evidence: Evidence, authorId: bigint): IARMessageContextRow {
+	return {
+		message_id: nextId(),
+		channel_id: evidence.channelId,
+		author_id: authorId,
+		webhook_id: null,
+		author_username: 'reported',
+		author_discriminator: 1,
+		author_avatar_hash: null,
+		content: 'the reported message',
+		timestamp: new Date(),
+		edited_timestamp: null,
+		type: 0,
+		flags: 0,
+		mention_everyone: false,
+		mention_users: null,
+		mention_roles: null,
+		mention_channels: null,
+		attachments: [
+			{
+				attachment_id: createAttachmentID(evidence.attachmentId),
+				filename: evidence.filename,
+				size: 1024n,
+				title: null,
+				description: null,
+				width: 64,
+				height: 64,
+				content_type: 'image/png',
+				content_hash: null,
+				placeholder: null,
+				flags: 0,
+				duration: null,
+				nsfw: false,
+				waveform: null,
+			},
+		],
+		embeds: null,
+		sticker_items: null,
+	};
+}
+
+describe('DELETE /admin/reports/:report_id', () => {
+	let harness: ApiTestHarness;
+	let admin: TestAccount;
+	let repository: ReportRepository;
+
+	beforeEach(async () => {
+		harness = await createApiTestHarness({search: 'enabled'});
+		admin = await setUserACLs(harness, await createTestAccount(harness), [
+			'admin:authenticate',
+			AdminACLs.REPORT_VIEW,
+			AdminACLs.REPORT_DELETE,
+		]);
+		repository = new ReportRepository();
+	});
+
+	afterEach(async () => {
+		await harness?.shutdown();
+	});
+
+	async function seedMessageReport(
+		evidence: Evidence,
+		options: {legalHoldUntil?: Date | null} = {},
+	): Promise<SeededReport> {
+		const id = nextId();
+		const reportId = createReportID(id);
+		const reporterId = nextId();
+		const reportedUserId = nextId();
+		const messageId = nextId();
+		const attachmentKey = evidenceKey(evidence);
+		const profileKey = `reports/${reportId}/profile/user_avatar/abc${id}`;
+		await harness.storageService.uploadObject({bucket: REPORTS_BUCKET, key: attachmentKey, body: new Uint8Array([1])});
+		await harness.storageService.uploadObject({bucket: REPORTS_BUCKET, key: profileKey, body: new Uint8Array([2])});
+		const report = await repository.createReport({
+			...Object.fromEntries(IAR_SUBMISSION_COLUMNS.map((column) => [column, null])),
+			report_id: id,
+			reporter_id: reporterId,
+			reported_at: new Date(Date.now() - ms('1 day')),
+			status: 1,
+			resolved_at: new Date(),
+			report_type: 0,
+			category: 'spam',
+			reported_user_id: reportedUserId,
+			reported_channel_id: evidence.channelId,
+			reported_message_id: messageId,
+			message_context: [contextRow(evidence, reportedUserId)],
+			reported_profile_snapshot: serializeReportProfileSnapshot({
+				captured_at: new Date().toISOString(),
+				user: {
+					id: reportedUserId.toString(),
+					username: 'reported',
+					discriminator: 1,
+					global_name: null,
+					bio: null,
+					pronouns: null,
+					avatar: {hash: `abc${id}`, key: profileKey},
+					banner: null,
+				},
+				member: null,
+				guild: null,
+			}),
+			legal_hold_until: options.legalHoldUntil ?? null,
+			legal_hold_reason: options.legalHoldUntil ? 'Preservation request' : null,
+		} as IARSubmissionRow);
+		await getReportSearchService()!.indexReport(report);
+		const reserved = await repository.reserveMessageReportByReporter({
+			reporter_id: reporterId,
+			channel_id: evidence.channelId,
+			message_id: messageId,
+			report_id: id,
+			reported_at: report.reportedAt,
+		});
+		expect(reserved).toBe(true);
+		return {reportId, reporterId, channelId: evidence.channelId, messageId, attachmentKey, profileKey};
+	}
+
+	function newEvidence(): Evidence {
+		return {channelId: nextId(), attachmentId: nextId(), filename: 'evidence.png'};
+	}
+
+	async function indexedIds(): Promise<Array<string>> {
+		const result = await getReportSearchService()!.searchReports('', {}, {limit: 1000});
+		return result.hits.map((hit) => hit.id);
+	}
+
+	function reservationIsFree(seeded: SeededReport): Promise<boolean> {
+		return repository.reserveMessageReportByReporter({
+			reporter_id: createUserID(seeded.reporterId),
+			channel_id: createChannelID(seeded.channelId),
+			message_id: createMessageID(seeded.messageId),
+			report_id: createReportID(nextId()),
+			reported_at: new Date(),
+		});
+	}
+
+	function deleteRequest(account: TestAccount, reportId: ReportID | string) {
+		return createBuilder<{code?: string}>(harness, account.token)
+			.delete(`/admin/reports/${reportId}`)
+			.header('X-Audit-Log-Reason', AUDIT_REASON);
+	}
+
+	async function auditEntries(action: string): Promise<Array<AdminAuditLog>> {
+		const logs = await getAdminRepository().listAllAuditLogsPaginated(10000);
+		return logs.filter((log) => log.action === action);
+	}
+
+	test('deletes the record, message context, evidence, search entry and reservation, and records the reason', async () => {
+		const seeded = await seedMessageReport(newEvidence());
+		const report = await repository.getReport(seeded.reportId);
+		expect(report?.messageContext).toHaveLength(1);
+		expect(await indexedIds()).toContain(seeded.reportId.toString());
+
+		await deleteRequest(admin, seeded.reportId).expect(HTTP_STATUS.NO_CONTENT).execute();
+
+		expect(await repository.getReport(seeded.reportId)).toBeNull();
+		expect(harness.storageService.hasObject(REPORTS_BUCKET, seeded.attachmentKey)).toBe(false);
+		expect(harness.storageService.hasObject(REPORTS_BUCKET, seeded.profileKey)).toBe(false);
+		expect(await indexedIds()).not.toContain(seeded.reportId.toString());
+		expect(await reservationIsFree(seeded)).toBe(true);
+		const [entry, ...rest] = await auditEntries('delete_report');
+		expect(rest).toEqual([]);
+		expect(entry?.adminUserId.toString()).toBe(admin.userId);
+		expect(entry?.targetType).toBe('report');
+		expect(entry?.targetId.toString()).toBe(seeded.reportId.toString());
+		expect(entry?.auditLogReason).toBe(AUDIT_REASON);
+		expect(Object.fromEntries(entry?.metadata ?? [])).toEqual({
+			report_id: seeded.reportId.toString(),
+			report_type: '0',
+			status: '1',
+			objects_deleted: '2',
+			shared_objects_kept: '0',
+		});
+	});
+
+	test('keeps an evidence copy another report still uses', async () => {
+		const shared = newEvidence();
+		const deleted = await seedMessageReport(shared);
+		const other = await seedMessageReport(shared);
+
+		await deleteRequest(admin, deleted.reportId).expect(HTTP_STATUS.NO_CONTENT).execute();
+
+		expect(await repository.getReport(deleted.reportId)).toBeNull();
+		expect(harness.storageService.hasObject(REPORTS_BUCKET, deleted.profileKey)).toBe(false);
+		expect(harness.storageService.hasObject(REPORTS_BUCKET, evidenceKey(shared))).toBe(true);
+		expect(await repository.getReport(other.reportId)).not.toBeNull();
+		expect(harness.storageService.hasObject(REPORTS_BUCKET, other.profileKey)).toBe(true);
+		expect(await indexedIds()).toContain(other.reportId.toString());
+		expect(await reservationIsFree(other)).toBe(false);
+		const [entry] = await auditEntries('delete_report');
+		expect(Object.fromEntries(entry?.metadata ?? [])).toMatchObject({objects_deleted: '1', shared_objects_kept: '1'});
+	});
+
+	test('refuses a report under a legal hold and changes nothing', async () => {
+		const seeded = await seedMessageReport(newEvidence(), {legalHoldUntil: new Date(Date.now() + ms('30 days'))});
+
+		const response = await deleteRequest(admin, seeded.reportId).expect(HTTP_STATUS.CONFLICT).execute();
+
+		expect(response.code).toBe(APIErrorCodes.REPORT_UNDER_LEGAL_HOLD);
+		expect(await repository.getReport(seeded.reportId)).not.toBeNull();
+		expect(harness.storageService.hasObject(REPORTS_BUCKET, seeded.attachmentKey)).toBe(true);
+		expect(harness.storageService.hasObject(REPORTS_BUCKET, seeded.profileKey)).toBe(true);
+		expect(await indexedIds()).toContain(seeded.reportId.toString());
+		expect(await reservationIsFree(seeded)).toBe(false);
+		expect(await auditEntries('delete_report')).toEqual([]);
+	});
+
+	test('deletes a report whose legal hold has ended', async () => {
+		const seeded = await seedMessageReport(newEvidence(), {legalHoldUntil: new Date(Date.now() - ms('1 minute'))});
+
+		await deleteRequest(admin, seeded.reportId).expect(HTTP_STATUS.NO_CONTENT).execute();
+
+		expect(await repository.getReport(seeded.reportId)).toBeNull();
+	});
+
+	test('an unknown report gives UNKNOWN_REPORT and records nothing', async () => {
+		await deleteRequest(admin, '1499999999999999999')
+			.expect(HTTP_STATUS.NOT_FOUND, APIErrorCodes.UNKNOWN_REPORT)
+			.execute();
+
+		expect(await auditEntries('delete_report')).toEqual([]);
+	});
+
+	test('needs REPORT_DELETE, which REPORT_RESOLVE does not grant', async () => {
+		const seeded = await seedMessageReport(newEvidence());
+		const resolver = await setUserACLs(harness, await createTestAccount(harness), [
+			'admin:authenticate',
+			AdminACLs.REPORT_VIEW,
+			AdminACLs.REPORT_RESOLVE,
+		]);
+
+		await deleteRequest(resolver, seeded.reportId).expect(HTTP_STATUS.FORBIDDEN, 'MISSING_ACL').execute();
+
+		expect(await repository.getReport(seeded.reportId)).not.toBeNull();
+		expect(harness.storageService.hasObject(REPORTS_BUCKET, seeded.attachmentKey)).toBe(true);
+		expect(await auditEntries('delete_report')).toEqual([]);
+	});
+});

fluxer_api/src/api/admin/tests/AdminReportLegalHold.test.ts

@@ -70,7 +70,7 @@ describe('report legal hold', () => {
 	async function runRetention() {
 		return processReportRetention(
 			{reportRepository: repository, storageService: new MockStorageService(), reportSearchService: null},
-			{now: new Date(), dryRun: false},
+			{now: new Date(), dryRun: false, policy: {retentionDays: 365, resolvedRetentionDays: null}},
 		);
 	}
 

fluxer_api/src/api/admin/tests/audit_coverage/ReportAdminAuditCases.ts

@@ -277,6 +277,28 @@ export const ReportAdminAuditCases: ReadonlyArray<AdminAuditCoverageCase> = [
 			};
 		},
 	},
+	{
+		method: 'DELETE',
+		route: '/admin/reports/:report_id',
+		async prepare(context) {
+			const {reportId} = await fileUserReport(context);
+			return {
+				request: {path: `/admin/reports/${reportId}`, expectStatus: 204},
+				expected: {
+					action: 'delete_report',
+					targetType: 'report',
+					targetId: reportId,
+					metadata: {
+						report_id: reportId,
+						report_type: '1',
+						status: '0',
+						objects_deleted: '0',
+						shared_objects_kept: '0',
+					},
+				},
+			};
+		},
+	},
 	{
 		method: 'POST',
 		route: '/admin/reports/:report_id/legal-hold',