mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-17 04:55:42 +09:00
feat(reports): delete reports and configure retention (#3338)
This commit is contained in:
1 parent
93d1f8af46
commit
82abd03a37
106 files changed
+1457
-170
No files matched your search
@@ -448,6 +448,11 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
|
||||
attachmentDecayEnabled: master.attachment_decay_enabled,
|
||||
deletionGracePeriodHours: master.dev.test_mode_enabled ? 0.01 : master.deletion_grace_period_hours,
|
||||
inactivityDeletionThresholdDays: master.inactivity_deletion_threshold_days,
|
||||
reportRetention: {
|
||||
days: master.report_retention.days,
|
||||
resolvedDays: master.report_retention.resolved_days,
|
||||
dryRun: master.report_retention.dry_run,
|
||||
},
|
||||
push: {
|
||||
publicVapidKey: master.auth.vapid.public_key,
|
||||
apns: {
|
||||
|
||||
@@ -8,6 +8,8 @@ import {RateLimitMiddleware} from '@app/api/middleware/RateLimitMiddleware';
|
||||
import {OpenAPI} from '@app/api/middleware/ResponseTypeMiddleware';
|
||||
import {getReportRepository} from '@app/api/middleware/ServiceSingletons';
|
||||
import {RateLimitConfigs} from '@app/api/RateLimitConfig';
|
||||
import {deleteReportNow} from '@app/api/report/ReportDeletion';
|
||||
import {getReportSearchService} from '@app/api/SearchFactory';
|
||||
import type {HonoApp} from '@app/api/types/HonoEnv';
|
||||
import {Validator} from '@app/api/Validator';
|
||||
import {AdminACLs} from '@fluxer/constants/src/AdminACLs';
|
||||
@@ -241,7 +243,7 @@ export function ReportAdminController(app: HonoApp) {
|
||||
operationId: 'set_admin_report_legal_hold',
|
||||
summary: 'Set report legal hold',
|
||||
description:
|
||||
'Places or clears a legal hold on a report. Reports are deleted 365 days after they were filed, together with their stored evidence, unless a hold that ends later is in place. A hold needs an end time in the future and a reason. A null legal_hold_until clears the hold. Creates an audit log entry. Requires REPORT_RESOLVE permission.',
|
||||
'Places or clears a legal hold on a report. A report is deleted together with its stored evidence when its retention period ends, unless a hold that ends later is in place, and a held report cannot be deleted through DELETE /admin/reports/{report_id} either. A hold needs an end time in the future and a reason. A null legal_hold_until clears the hold. Creates an audit log entry. Requires REPORT_RESOLVE permission.',
|
||||
responseSchema: ReportLegalHoldResponse,
|
||||
statusCode: 200,
|
||||
security: 'adminApiKey',
|
||||
@@ -289,4 +291,48 @@ export function ReportAdminController(app: HonoApp) {
|
||||
});
|
||||
},
|
||||
);
|
||||
app.delete(
|
||||
'/admin/reports/:report_id',
|
||||
RateLimitMiddleware(RateLimitConfigs.ADMIN_LOOKUP),
|
||||
requireAdminACL(AdminACLs.REPORT_DELETE),
|
||||
Validator('param', ReportIdParam),
|
||||
OpenAPI({
|
||||
operationId: 'delete_admin_report',
|
||||
summary: 'Delete report',
|
||||
description:
|
||||
'Deletes a report now: its record with the captured message context and profile snapshot, its stored evidence copies, its search index entry, and the duplicate-report reservation it holds for the reporter. An evidence copy that another report also uses is kept. A report under a legal hold that has not ended is refused with REPORT_UNDER_LEGAL_HOLD. This cannot be undone. Creates an audit log entry. Requires REPORT_DELETE permission.',
|
||||
responseSchema: null,
|
||||
statusCode: 204,
|
||||
security: 'adminApiKey',
|
||||
tags: 'Admin',
|
||||
}),
|
||||
async (ctx) => {
|
||||
const {report_id} = ctx.req.valid('param');
|
||||
const reportId = createReportID(report_id);
|
||||
const deleted = await deleteReportNow(
|
||||
{
|
||||
reportRepository: getReportRepository(),
|
||||
storageService: ctx.get('storageService'),
|
||||
reportSearchService: getReportSearchService(),
|
||||
},
|
||||
reportId,
|
||||
new Date(),
|
||||
);
|
||||
await ctx.get('adminService').auditService.createAuditLog({
|
||||
adminUserId: ctx.get('adminUserId'),
|
||||
targetType: 'report',
|
||||
targetId: BigInt(reportId),
|
||||
action: 'delete_report',
|
||||
auditLogReason: ctx.get('auditLogReason'),
|
||||
metadata: new Map([
|
||||
['report_id', reportId.toString()],
|
||||
['report_type', deleted.report.reportType.toString()],
|
||||
['status', deleted.report.status.toString()],
|
||||
['objects_deleted', deleted.objectsDeleted.toString()],
|
||||
['shared_objects_kept', deleted.sharedObjectsKept.toString()],
|
||||
]),
|
||||
});
|
||||
return ctx.body(null, 204);
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -32,6 +32,7 @@ const adminEndpoints: Array<AdminEndpointCase> = [
|
||||
{method: 'GET', path: '/admin/reports', requiredACL: 'report:view'},
|
||||
{method: 'GET', path: '/admin/reports/1', requiredACL: 'report:view'},
|
||||
{method: 'PATCH', path: '/admin/reports/1', requiredACL: 'report:resolve'},
|
||||
{method: 'DELETE', path: '/admin/reports/1', requiredACL: 'report:delete'},
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/admin/bulk-jobs',
|
||||
|
||||
@@ -0,0 +1,296 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {AdminAuditLog} from '@app/api/admin/IAdminRepository';
|
||||
import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';
|
||||
import {
|
||||
createAttachmentID,
|
||||
createChannelID,
|
||||
createMessageID,
|
||||
createReportID,
|
||||
createUserID,
|
||||
type ReportID,
|
||||
} from '@app/api/BrandedTypes';
|
||||
import {Config} from '@app/api/Config';
|
||||
import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';
|
||||
import {
|
||||
IAR_SUBMISSION_COLUMNS,
|
||||
type IARMessageContextRow,
|
||||
type IARSubmissionRow,
|
||||
} from '@app/api/database/types/ReportTypes';
|
||||
import {getAdminRepository} from '@app/api/middleware/ServiceSingletons';
|
||||
import {ReportRepository} from '@app/api/report/ReportRepository';
|
||||
import {getReportSearchService} from '@app/api/SearchFactory';
|
||||
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {HTTP_STATUS} from '@app/api/test/TestConstants';
|
||||
import {createBuilder} from '@app/api/test/TestRequestBuilder';
|
||||
import {AdminACLs} from '@fluxer/constants/src/AdminACLs';
|
||||
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
|
||||
import {serializeReportProfileSnapshot} from '@fluxer/schema/src/domains/report/ReportProfileSnapshotSchemas';
|
||||
import {ms} from 'itty-time';
|
||||
import {afterEach, beforeEach, describe, expect, test} from 'vitest';
|
||||
|
||||
const REPORTS_BUCKET = Config.s3.buckets.reports;
|
||||
const AUDIT_REASON = 'Erasure request 77';
|
||||
let sequence = 1_498_000_000_000_000_000n;
|
||||
|
||||
function nextId(): bigint {
|
||||
sequence += 1n;
|
||||
return sequence;
|
||||
}
|
||||
|
||||
interface Evidence {
|
||||
channelId: bigint;
|
||||
attachmentId: bigint;
|
||||
filename: string;
|
||||
}
|
||||
|
||||
interface SeededReport {
|
||||
reportId: ReportID;
|
||||
reporterId: bigint;
|
||||
channelId: bigint;
|
||||
messageId: bigint;
|
||||
attachmentKey: string;
|
||||
profileKey: string;
|
||||
}
|
||||
|
||||
function evidenceKey(evidence: Evidence): string {
|
||||
return makeAttachmentCdnKey(createChannelID(evidence.channelId), evidence.attachmentId, evidence.filename);
|
||||
}
|
||||
|
||||
function contextRow(evidence: Evidence, authorId: bigint): IARMessageContextRow {
|
||||
return {
|
||||
message_id: nextId(),
|
||||
channel_id: evidence.channelId,
|
||||
author_id: authorId,
|
||||
webhook_id: null,
|
||||
author_username: 'reported',
|
||||
author_discriminator: 1,
|
||||
author_avatar_hash: null,
|
||||
content: 'the reported message',
|
||||
timestamp: new Date(),
|
||||
edited_timestamp: null,
|
||||
type: 0,
|
||||
flags: 0,
|
||||
mention_everyone: false,
|
||||
mention_users: null,
|
||||
mention_roles: null,
|
||||
mention_channels: null,
|
||||
attachments: [
|
||||
{
|
||||
attachment_id: createAttachmentID(evidence.attachmentId),
|
||||
filename: evidence.filename,
|
||||
size: 1024n,
|
||||
title: null,
|
||||
description: null,
|
||||
width: 64,
|
||||
height: 64,
|
||||
content_type: 'image/png',
|
||||
content_hash: null,
|
||||
placeholder: null,
|
||||
flags: 0,
|
||||
duration: null,
|
||||
nsfw: false,
|
||||
waveform: null,
|
||||
},
|
||||
],
|
||||
embeds: null,
|
||||
sticker_items: null,
|
||||
};
|
||||
}
|
||||
|
||||
describe('DELETE /admin/reports/:report_id', () => {
|
||||
let harness: ApiTestHarness;
|
||||
let admin: TestAccount;
|
||||
let repository: ReportRepository;
|
||||
|
||||
beforeEach(async () => {
|
||||
harness = await createApiTestHarness({search: 'enabled'});
|
||||
admin = await setUserACLs(harness, await createTestAccount(harness), [
|
||||
'admin:authenticate',
|
||||
AdminACLs.REPORT_VIEW,
|
||||
AdminACLs.REPORT_DELETE,
|
||||
]);
|
||||
repository = new ReportRepository();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await harness?.shutdown();
|
||||
});
|
||||
|
||||
async function seedMessageReport(
|
||||
evidence: Evidence,
|
||||
options: {legalHoldUntil?: Date | null} = {},
|
||||
): Promise<SeededReport> {
|
||||
const id = nextId();
|
||||
const reportId = createReportID(id);
|
||||
const reporterId = nextId();
|
||||
const reportedUserId = nextId();
|
||||
const messageId = nextId();
|
||||
const attachmentKey = evidenceKey(evidence);
|
||||
const profileKey = `reports/${reportId}/profile/user_avatar/abc${id}`;
|
||||
await harness.storageService.uploadObject({bucket: REPORTS_BUCKET, key: attachmentKey, body: new Uint8Array([1])});
|
||||
await harness.storageService.uploadObject({bucket: REPORTS_BUCKET, key: profileKey, body: new Uint8Array([2])});
|
||||
const report = await repository.createReport({
|
||||
...Object.fromEntries(IAR_SUBMISSION_COLUMNS.map((column) => [column, null])),
|
||||
report_id: id,
|
||||
reporter_id: reporterId,
|
||||
reported_at: new Date(Date.now() - ms('1 day')),
|
||||
status: 1,
|
||||
resolved_at: new Date(),
|
||||
report_type: 0,
|
||||
category: 'spam',
|
||||
reported_user_id: reportedUserId,
|
||||
reported_channel_id: evidence.channelId,
|
||||
reported_message_id: messageId,
|
||||
message_context: [contextRow(evidence, reportedUserId)],
|
||||
reported_profile_snapshot: serializeReportProfileSnapshot({
|
||||
captured_at: new Date().toISOString(),
|
||||
user: {
|
||||
id: reportedUserId.toString(),
|
||||
username: 'reported',
|
||||
discriminator: 1,
|
||||
global_name: null,
|
||||
bio: null,
|
||||
pronouns: null,
|
||||
avatar: {hash: `abc${id}`, key: profileKey},
|
||||
banner: null,
|
||||
},
|
||||
member: null,
|
||||
guild: null,
|
||||
}),
|
||||
legal_hold_until: options.legalHoldUntil ?? null,
|
||||
legal_hold_reason: options.legalHoldUntil ? 'Preservation request' : null,
|
||||
} as IARSubmissionRow);
|
||||
await getReportSearchService()!.indexReport(report);
|
||||
const reserved = await repository.reserveMessageReportByReporter({
|
||||
reporter_id: reporterId,
|
||||
channel_id: evidence.channelId,
|
||||
message_id: messageId,
|
||||
report_id: id,
|
||||
reported_at: report.reportedAt,
|
||||
});
|
||||
expect(reserved).toBe(true);
|
||||
return {reportId, reporterId, channelId: evidence.channelId, messageId, attachmentKey, profileKey};
|
||||
}
|
||||
|
||||
function newEvidence(): Evidence {
|
||||
return {channelId: nextId(), attachmentId: nextId(), filename: 'evidence.png'};
|
||||
}
|
||||
|
||||
async function indexedIds(): Promise<Array<string>> {
|
||||
const result = await getReportSearchService()!.searchReports('', {}, {limit: 1000});
|
||||
return result.hits.map((hit) => hit.id);
|
||||
}
|
||||
|
||||
function reservationIsFree(seeded: SeededReport): Promise<boolean> {
|
||||
return repository.reserveMessageReportByReporter({
|
||||
reporter_id: createUserID(seeded.reporterId),
|
||||
channel_id: createChannelID(seeded.channelId),
|
||||
message_id: createMessageID(seeded.messageId),
|
||||
report_id: createReportID(nextId()),
|
||||
reported_at: new Date(),
|
||||
});
|
||||
}
|
||||
|
||||
function deleteRequest(account: TestAccount, reportId: ReportID | string) {
|
||||
return createBuilder<{code?: string}>(harness, account.token)
|
||||
.delete(`/admin/reports/${reportId}`)
|
||||
.header('X-Audit-Log-Reason', AUDIT_REASON);
|
||||
}
|
||||
|
||||
async function auditEntries(action: string): Promise<Array<AdminAuditLog>> {
|
||||
const logs = await getAdminRepository().listAllAuditLogsPaginated(10000);
|
||||
return logs.filter((log) => log.action === action);
|
||||
}
|
||||
|
||||
test('deletes the record, message context, evidence, search entry and reservation, and records the reason', async () => {
|
||||
const seeded = await seedMessageReport(newEvidence());
|
||||
const report = await repository.getReport(seeded.reportId);
|
||||
expect(report?.messageContext).toHaveLength(1);
|
||||
expect(await indexedIds()).toContain(seeded.reportId.toString());
|
||||
|
||||
await deleteRequest(admin, seeded.reportId).expect(HTTP_STATUS.NO_CONTENT).execute();
|
||||
|
||||
expect(await repository.getReport(seeded.reportId)).toBeNull();
|
||||
expect(harness.storageService.hasObject(REPORTS_BUCKET, seeded.attachmentKey)).toBe(false);
|
||||
expect(harness.storageService.hasObject(REPORTS_BUCKET, seeded.profileKey)).toBe(false);
|
||||
expect(await indexedIds()).not.toContain(seeded.reportId.toString());
|
||||
expect(await reservationIsFree(seeded)).toBe(true);
|
||||
const [entry, ...rest] = await auditEntries('delete_report');
|
||||
expect(rest).toEqual([]);
|
||||
expect(entry?.adminUserId.toString()).toBe(admin.userId);
|
||||
expect(entry?.targetType).toBe('report');
|
||||
expect(entry?.targetId.toString()).toBe(seeded.reportId.toString());
|
||||
expect(entry?.auditLogReason).toBe(AUDIT_REASON);
|
||||
expect(Object.fromEntries(entry?.metadata ?? [])).toEqual({
|
||||
report_id: seeded.reportId.toString(),
|
||||
report_type: '0',
|
||||
status: '1',
|
||||
objects_deleted: '2',
|
||||
shared_objects_kept: '0',
|
||||
});
|
||||
});
|
||||
|
||||
test('keeps an evidence copy another report still uses', async () => {
|
||||
const shared = newEvidence();
|
||||
const deleted = await seedMessageReport(shared);
|
||||
const other = await seedMessageReport(shared);
|
||||
|
||||
await deleteRequest(admin, deleted.reportId).expect(HTTP_STATUS.NO_CONTENT).execute();
|
||||
|
||||
expect(await repository.getReport(deleted.reportId)).toBeNull();
|
||||
expect(harness.storageService.hasObject(REPORTS_BUCKET, deleted.profileKey)).toBe(false);
|
||||
expect(harness.storageService.hasObject(REPORTS_BUCKET, evidenceKey(shared))).toBe(true);
|
||||
expect(await repository.getReport(other.reportId)).not.toBeNull();
|
||||
expect(harness.storageService.hasObject(REPORTS_BUCKET, other.profileKey)).toBe(true);
|
||||
expect(await indexedIds()).toContain(other.reportId.toString());
|
||||
expect(await reservationIsFree(other)).toBe(false);
|
||||
const [entry] = await auditEntries('delete_report');
|
||||
expect(Object.fromEntries(entry?.metadata ?? [])).toMatchObject({objects_deleted: '1', shared_objects_kept: '1'});
|
||||
});
|
||||
|
||||
test('refuses a report under a legal hold and changes nothing', async () => {
|
||||
const seeded = await seedMessageReport(newEvidence(), {legalHoldUntil: new Date(Date.now() + ms('30 days'))});
|
||||
|
||||
const response = await deleteRequest(admin, seeded.reportId).expect(HTTP_STATUS.CONFLICT).execute();
|
||||
|
||||
expect(response.code).toBe(APIErrorCodes.REPORT_UNDER_LEGAL_HOLD);
|
||||
expect(await repository.getReport(seeded.reportId)).not.toBeNull();
|
||||
expect(harness.storageService.hasObject(REPORTS_BUCKET, seeded.attachmentKey)).toBe(true);
|
||||
expect(harness.storageService.hasObject(REPORTS_BUCKET, seeded.profileKey)).toBe(true);
|
||||
expect(await indexedIds()).toContain(seeded.reportId.toString());
|
||||
expect(await reservationIsFree(seeded)).toBe(false);
|
||||
expect(await auditEntries('delete_report')).toEqual([]);
|
||||
});
|
||||
|
||||
test('deletes a report whose legal hold has ended', async () => {
|
||||
const seeded = await seedMessageReport(newEvidence(), {legalHoldUntil: new Date(Date.now() - ms('1 minute'))});
|
||||
|
||||
await deleteRequest(admin, seeded.reportId).expect(HTTP_STATUS.NO_CONTENT).execute();
|
||||
|
||||
expect(await repository.getReport(seeded.reportId)).toBeNull();
|
||||
});
|
||||
|
||||
test('an unknown report gives UNKNOWN_REPORT and records nothing', async () => {
|
||||
await deleteRequest(admin, '1499999999999999999')
|
||||
.expect(HTTP_STATUS.NOT_FOUND, APIErrorCodes.UNKNOWN_REPORT)
|
||||
.execute();
|
||||
|
||||
expect(await auditEntries('delete_report')).toEqual([]);
|
||||
});
|
||||
|
||||
test('needs REPORT_DELETE, which REPORT_RESOLVE does not grant', async () => {
|
||||
const seeded = await seedMessageReport(newEvidence());
|
||||
const resolver = await setUserACLs(harness, await createTestAccount(harness), [
|
||||
'admin:authenticate',
|
||||
AdminACLs.REPORT_VIEW,
|
||||
AdminACLs.REPORT_RESOLVE,
|
||||
]);
|
||||
|
||||
await deleteRequest(resolver, seeded.reportId).expect(HTTP_STATUS.FORBIDDEN, 'MISSING_ACL').execute();
|
||||
|
||||
expect(await repository.getReport(seeded.reportId)).not.toBeNull();
|
||||
expect(harness.storageService.hasObject(REPORTS_BUCKET, seeded.attachmentKey)).toBe(true);
|
||||
expect(await auditEntries('delete_report')).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -70,7 +70,7 @@ describe('report legal hold', () => {
|
||||
async function runRetention() {
|
||||
return processReportRetention(
|
||||
{reportRepository: repository, storageService: new MockStorageService(), reportSearchService: null},
|
||||
{now: new Date(), dryRun: false},
|
||||
{now: new Date(), dryRun: false, policy: {retentionDays: 365, resolvedRetentionDays: null}},
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -277,6 +277,28 @@ export const ReportAdminAuditCases: ReadonlyArray<AdminAuditCoverageCase> = [
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
method: 'DELETE',
|
||||
route: '/admin/reports/:report_id',
|
||||
async prepare(context) {
|
||||
const {reportId} = await fileUserReport(context);
|
||||
return {
|
||||
request: {path: `/admin/reports/${reportId}`, expectStatus: 204},
|
||||
expected: {
|
||||
action: 'delete_report',
|
||||
targetType: 'report',
|
||||
targetId: reportId,
|
||||
metadata: {
|
||||
report_id: reportId,
|
||||
report_type: '1',
|
||||
status: '0',
|
||||
objects_deleted: '0',
|
||||
shared_objects_kept: '0',
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
method: 'POST',
|
||||
route: '/admin/reports/:report_id/legal-hold',
|
||||
|
||||
@@ -312,6 +312,11 @@ export interface APIConfig {
|
||||
presignedHarvestDownloadsEnabled: boolean;
|
||||
attachmentDecayEnabled: boolean;
|
||||
deletionGracePeriodHours: number;
|
||||
reportRetention: {
|
||||
days: number;
|
||||
resolvedDays: number | null;
|
||||
dryRun: boolean;
|
||||
};
|
||||
inactivityDeletionThresholdDays?: number;
|
||||
push: {
|
||||
publicVapidKey?: string;
|
||||
|
||||
@@ -39762,6 +39762,7 @@
|
||||
"REPORT_ALREADY_RESOLVED",
|
||||
"REPORT_BANNED",
|
||||
"REPORT_FLOW_OUTDATED",
|
||||
"REPORT_UNDER_LEGAL_HOLD",
|
||||
"RESPONSE_VALIDATION_ERROR",
|
||||
"RESOURCE_LOCKED",
|
||||
"SERVICE_UNAVAILABLE",
|
||||
@@ -40057,6 +40058,7 @@
|
||||
"Report already resolved.",
|
||||
"You've been banned from submitting reports.",
|
||||
"The report form changed while you were filling it in. Start the report again.",
|
||||
"This report is under a legal hold and can't be deleted.",
|
||||
"Response validation failed: {errors}.",
|
||||
"This resource is being modified. Please try again shortly.",
|
||||
"Service unavailable.",
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {ReportID} from '@app/api/BrandedTypes';
|
||||
import {Config} from '@app/api/Config';
|
||||
import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';
|
||||
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
|
||||
import type {IARSubmission, IReportRepository} from '@app/api/report/IReportRepository';
|
||||
import type {IReportSearchService} from '@app/api/search/IReportSearchService';
|
||||
import {ReportUnderLegalHoldError} from '@fluxer/errors/src/domains/moderation/ReportUnderLegalHoldError';
|
||||
import {UnknownReportError} from '@fluxer/errors/src/domains/moderation/UnknownReportError';
|
||||
import {listReportProfileSnapshotAssets} from '@fluxer/schema/src/domains/report/ReportProfileSnapshotSchemas';
|
||||
|
||||
const SCAN_PAGE_SIZE = 500;
|
||||
|
||||
export interface ReportDeletionDeps {
|
||||
reportRepository: IReportRepository;
|
||||
storageService: IStorageService;
|
||||
reportSearchService: IReportSearchService | null;
|
||||
}
|
||||
|
||||
export interface ReportEvidenceLedger {
|
||||
retainedKeys: Set<string>;
|
||||
deletedKeys: Set<string>;
|
||||
}
|
||||
|
||||
export interface ReportDeletionResult {
|
||||
objectsDeleted: number;
|
||||
sharedObjectsKept: number;
|
||||
}
|
||||
|
||||
export function isReportUnderLegalHold(report: IARSubmission, now: Date): boolean {
|
||||
return report.legalHoldUntil != null && report.legalHoldUntil.getTime() > now.getTime();
|
||||
}
|
||||
|
||||
export function referencedObjectKeys(report: IARSubmission): Array<string> {
|
||||
const keys: Array<string> = [];
|
||||
for (const message of report.messageContext ?? []) {
|
||||
const channelId = message.channelId ?? report.reportedChannelId;
|
||||
if (!channelId) continue;
|
||||
for (const attachment of message.attachments) {
|
||||
if (attachment.attachment_id == null || !attachment.filename) continue;
|
||||
keys.push(makeAttachmentCdnKey(channelId, attachment.attachment_id, String(attachment.filename)));
|
||||
}
|
||||
}
|
||||
for (const asset of listReportProfileSnapshotAssets(report.reportedProfileSnapshot)) {
|
||||
if (asset.key) {
|
||||
keys.push(asset.key);
|
||||
}
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
export async function forEachStoredReport(
|
||||
reportRepository: IReportRepository,
|
||||
visit: (report: IARSubmission) => void,
|
||||
pageSize = SCAN_PAGE_SIZE,
|
||||
): Promise<void> {
|
||||
let cursor: ReportID | undefined;
|
||||
while (true) {
|
||||
const page = await reportRepository.listAllReportsPaginated(pageSize, cursor);
|
||||
for (const report of page) {
|
||||
visit(report);
|
||||
}
|
||||
if (page.length < pageSize) return;
|
||||
cursor = page[page.length - 1]!.reportId;
|
||||
}
|
||||
}
|
||||
|
||||
async function storedObjectKeys(storageService: IStorageService, report: IARSubmission): Promise<Set<string>> {
|
||||
const keys = new Set(referencedObjectKeys(report));
|
||||
const listed = await storageService.listObjects({
|
||||
bucket: Config.s3.buckets.reports,
|
||||
prefix: `reports/${report.reportId}/profile/`,
|
||||
});
|
||||
for (const object of listed) {
|
||||
keys.add(object.key);
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
export async function deleteReportWithEvidence(
|
||||
deps: ReportDeletionDeps,
|
||||
report: IARSubmission,
|
||||
ledger: ReportEvidenceLedger,
|
||||
options: {now: Date; dryRun: boolean},
|
||||
): Promise<ReportDeletionResult> {
|
||||
const {reportRepository, storageService, reportSearchService} = deps;
|
||||
const result: ReportDeletionResult = {objectsDeleted: 0, sharedObjectsKept: 0};
|
||||
for (const key of await storedObjectKeys(storageService, report)) {
|
||||
if (ledger.retainedKeys.has(key)) {
|
||||
result.sharedObjectsKept++;
|
||||
continue;
|
||||
}
|
||||
if (ledger.deletedKeys.has(key)) continue;
|
||||
const metadata = await storageService.getObjectMetadata(Config.s3.buckets.reports, key);
|
||||
if (metadata?.lastModified && metadata.lastModified.getTime() >= options.now.getTime()) {
|
||||
ledger.retainedKeys.add(key);
|
||||
result.sharedObjectsKept++;
|
||||
continue;
|
||||
}
|
||||
if (!options.dryRun) {
|
||||
await storageService.deleteObject(Config.s3.buckets.reports, key);
|
||||
}
|
||||
ledger.deletedKeys.add(key);
|
||||
result.objectsDeleted++;
|
||||
}
|
||||
if (!options.dryRun) {
|
||||
await reportSearchService?.deleteReport(report.reportId);
|
||||
await reportRepository.deleteReport(report.reportId);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function loadDeletableReport(
|
||||
reportRepository: IReportRepository,
|
||||
reportId: ReportID,
|
||||
now: Date,
|
||||
): Promise<IARSubmission> {
|
||||
const report = await reportRepository.getReport(reportId);
|
||||
if (!report) {
|
||||
throw new UnknownReportError();
|
||||
}
|
||||
if (isReportUnderLegalHold(report, now)) {
|
||||
throw new ReportUnderLegalHoldError();
|
||||
}
|
||||
return report;
|
||||
}
|
||||
|
||||
export async function deleteReportNow(
|
||||
deps: ReportDeletionDeps,
|
||||
reportId: ReportID,
|
||||
now: Date,
|
||||
): Promise<{report: IARSubmission} & ReportDeletionResult> {
|
||||
const {reportRepository} = deps;
|
||||
await loadDeletableReport(reportRepository, reportId, now);
|
||||
const retainedKeys = new Set<string>();
|
||||
await forEachStoredReport(reportRepository, (other) => {
|
||||
if (other.reportId === reportId) return;
|
||||
for (const key of referencedObjectKeys(other)) {
|
||||
retainedKeys.add(key);
|
||||
}
|
||||
});
|
||||
const report = await loadDeletableReport(reportRepository, reportId, now);
|
||||
const result = await deleteReportWithEvidence(
|
||||
deps,
|
||||
report,
|
||||
{retainedKeys, deletedKeys: new Set()},
|
||||
{now, dryRun: false},
|
||||
);
|
||||
return {report, ...result};
|
||||
}
|
||||
@@ -27,6 +27,7 @@ import {
|
||||
type IARSubmissionRow,
|
||||
type IReportRepository,
|
||||
ReportStatus,
|
||||
ReportType,
|
||||
} from '@app/api/report/IReportRepository';
|
||||
import {
|
||||
DSAReportEmailVerifications,
|
||||
@@ -196,11 +197,31 @@ export class ReportRepository implements IReportRepository {
|
||||
if (!report) {
|
||||
return;
|
||||
}
|
||||
await this.releaseReporterReservation(report);
|
||||
await executeConditional(
|
||||
IARSubmissions.conditionalDeleteByPk({report_id: reportId}, {report_type: report.reportType}),
|
||||
);
|
||||
}
|
||||
|
||||
private async releaseReporterReservation(report: IARSubmission): Promise<void> {
|
||||
const {reporterId, reportId, reportedAt} = report;
|
||||
if (!reporterId) {
|
||||
return;
|
||||
}
|
||||
const base = {reporter_id: reporterId, report_id: reportId, reported_at: reportedAt};
|
||||
if (report.reportType === ReportType.MESSAGE && report.reportedChannelId && report.reportedMessageId) {
|
||||
await this.releaseMessageReportByReporter({
|
||||
...base,
|
||||
channel_id: report.reportedChannelId,
|
||||
message_id: report.reportedMessageId,
|
||||
});
|
||||
} else if (report.reportType === ReportType.USER && report.reportedUserId) {
|
||||
await this.releaseUserReportByReporter({...base, reported_user_id: report.reportedUserId});
|
||||
} else if (report.reportType === ReportType.GUILD && report.reportedGuildId) {
|
||||
await this.releaseGuildReportByReporter({...base, reported_guild_id: report.reportedGuildId});
|
||||
}
|
||||
}
|
||||
|
||||
private mapRowToSubmission(row: IARSubmissionRow): IARSubmission {
|
||||
return {
|
||||
reportId: createReportID(row.report_id),
|
||||
|
||||
@@ -2,31 +2,29 @@
|
||||
|
||||
import type {ReportID} from '@app/api/BrandedTypes';
|
||||
import {Config} from '@app/api/Config';
|
||||
import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';
|
||||
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
|
||||
import {Logger} from '@app/api/Logger';
|
||||
import type {IARSubmission, IReportRepository} from '@app/api/report/IReportRepository';
|
||||
import {type IARSubmission, ReportStatus} from '@app/api/report/IReportRepository';
|
||||
import {
|
||||
deleteReportWithEvidence,
|
||||
forEachStoredReport,
|
||||
isReportUnderLegalHold,
|
||||
type ReportDeletionDeps,
|
||||
referencedObjectKeys,
|
||||
} from '@app/api/report/ReportDeletion';
|
||||
import {getReportSearchService} from '@app/api/SearchFactory';
|
||||
import type {IReportSearchService} from '@app/api/search/IReportSearchService';
|
||||
import {readOptionalEnv} from '@app/api/utils/IntegerOptions';
|
||||
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
|
||||
import {listReportProfileSnapshotAssets} from '@fluxer/schema/src/domains/report/ReportProfileSnapshotSchemas';
|
||||
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
|
||||
import {ms} from 'itty-time';
|
||||
|
||||
const REPORT_RETENTION_DAYS = 365;
|
||||
const REPORT_RETENTION_MS = REPORT_RETENTION_DAYS * ms('1 day');
|
||||
const DEFAULT_SCAN_PAGE_SIZE = 500;
|
||||
|
||||
export interface ReportRetentionDeps {
|
||||
reportRepository: IReportRepository;
|
||||
storageService: IStorageService;
|
||||
reportSearchService: IReportSearchService | null;
|
||||
export interface ReportRetentionPolicy {
|
||||
retentionDays: number;
|
||||
resolvedRetentionDays: number | null;
|
||||
}
|
||||
|
||||
export interface ReportRetentionOptions {
|
||||
now: Date;
|
||||
dryRun: boolean;
|
||||
policy: ReportRetentionPolicy;
|
||||
pageSize?: number;
|
||||
}
|
||||
|
||||
@@ -43,63 +41,36 @@ export interface ReportRetentionSummary {
|
||||
|
||||
type ReportRetentionState = 'expired' | 'held' | 'retained';
|
||||
|
||||
export function resolveReportRetentionDryRun(raw: string | undefined): boolean {
|
||||
const value = raw?.trim().toLowerCase();
|
||||
return value !== 'false' && value !== '0';
|
||||
function reportExpiresAt(report: IARSubmission, policy: ReportRetentionPolicy): number {
|
||||
const reportedAt = report.reportedAt instanceof Date ? report.reportedAt.getTime() : Number.NaN;
|
||||
const byAge = reportedAt + policy.retentionDays * ms('1 day');
|
||||
if (policy.resolvedRetentionDays === null || report.status !== ReportStatus.RESOLVED) {
|
||||
return byAge;
|
||||
}
|
||||
const resolvedAt = report.resolvedAt instanceof Date ? report.resolvedAt.getTime() : Number.NaN;
|
||||
if (Number.isNaN(resolvedAt)) {
|
||||
return byAge;
|
||||
}
|
||||
return Math.min(byAge, resolvedAt + policy.resolvedRetentionDays * ms('1 day'));
|
||||
}
|
||||
|
||||
function reportRetentionState(report: IARSubmission, now: Date): ReportRetentionState {
|
||||
const reportedAt = report.reportedAt instanceof Date ? report.reportedAt.getTime() : Number.NaN;
|
||||
if (Number.isNaN(reportedAt) || reportedAt > now.getTime() - REPORT_RETENTION_MS) {
|
||||
function reportRetentionState(report: IARSubmission, now: Date, policy: ReportRetentionPolicy): ReportRetentionState {
|
||||
const expiresAt = reportExpiresAt(report, policy);
|
||||
if (Number.isNaN(expiresAt) || expiresAt > now.getTime()) {
|
||||
return 'retained';
|
||||
}
|
||||
if (report.legalHoldUntil && report.legalHoldUntil.getTime() > now.getTime()) {
|
||||
if (isReportUnderLegalHold(report, now)) {
|
||||
return 'held';
|
||||
}
|
||||
return 'expired';
|
||||
}
|
||||
|
||||
function reportProfilePrefix(reportId: ReportID): string {
|
||||
return `reports/${reportId}/profile/`;
|
||||
}
|
||||
|
||||
function referencedObjectKeys(report: IARSubmission): Array<string> {
|
||||
const keys: Array<string> = [];
|
||||
for (const message of report.messageContext ?? []) {
|
||||
const channelId = message.channelId ?? report.reportedChannelId;
|
||||
if (!channelId) continue;
|
||||
for (const attachment of message.attachments) {
|
||||
if (attachment.attachment_id == null || !attachment.filename) continue;
|
||||
keys.push(makeAttachmentCdnKey(channelId, attachment.attachment_id, String(attachment.filename)));
|
||||
}
|
||||
}
|
||||
for (const asset of listReportProfileSnapshotAssets(report.reportedProfileSnapshot)) {
|
||||
if (asset.key) {
|
||||
keys.push(asset.key);
|
||||
}
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
async function storedObjectKeys(storageService: IStorageService, report: IARSubmission): Promise<Set<string>> {
|
||||
const keys = new Set(referencedObjectKeys(report));
|
||||
const listed = await storageService.listObjects({
|
||||
bucket: Config.s3.buckets.reports,
|
||||
prefix: reportProfilePrefix(report.reportId),
|
||||
});
|
||||
for (const object of listed) {
|
||||
keys.add(object.key);
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
export async function processReportRetention(
|
||||
deps: ReportRetentionDeps,
|
||||
deps: ReportDeletionDeps,
|
||||
options: ReportRetentionOptions,
|
||||
): Promise<ReportRetentionSummary> {
|
||||
const {reportRepository, storageService, reportSearchService} = deps;
|
||||
const {now, dryRun} = options;
|
||||
const pageSize = options.pageSize ?? DEFAULT_SCAN_PAGE_SIZE;
|
||||
const {reportRepository} = deps;
|
||||
const {now, dryRun, policy} = options;
|
||||
const summary: ReportRetentionSummary = {
|
||||
dryRun,
|
||||
scanned: 0,
|
||||
@@ -112,15 +83,14 @@ export async function processReportRetention(
|
||||
};
|
||||
const retainedKeys = new Set<string>();
|
||||
const expiredIds: Array<ReportID> = [];
|
||||
let cursor: ReportID | undefined;
|
||||
while (true) {
|
||||
const page = await reportRepository.listAllReportsPaginated(pageSize, cursor);
|
||||
for (const report of page) {
|
||||
await forEachStoredReport(
|
||||
reportRepository,
|
||||
(report) => {
|
||||
summary.scanned++;
|
||||
const state = reportRetentionState(report, now);
|
||||
const state = reportRetentionState(report, now, policy);
|
||||
if (state === 'expired') {
|
||||
expiredIds.push(report.reportId);
|
||||
continue;
|
||||
return;
|
||||
}
|
||||
if (state === 'held') {
|
||||
summary.held++;
|
||||
@@ -128,59 +98,47 @@ export async function processReportRetention(
|
||||
for (const key of referencedObjectKeys(report)) {
|
||||
retainedKeys.add(key);
|
||||
}
|
||||
}
|
||||
if (page.length < pageSize) break;
|
||||
cursor = page[page.length - 1]!.reportId;
|
||||
}
|
||||
},
|
||||
options.pageSize,
|
||||
);
|
||||
summary.expired = expiredIds.length;
|
||||
const deletedKeys = new Set<string>();
|
||||
const ledger = {retainedKeys, deletedKeys: new Set<string>()};
|
||||
for (const reportId of expiredIds) {
|
||||
try {
|
||||
const report = await reportRepository.getReport(reportId);
|
||||
if (!report) continue;
|
||||
if (reportRetentionState(report, now) !== 'expired') {
|
||||
if (reportRetentionState(report, now, policy) !== 'expired') {
|
||||
for (const key of referencedObjectKeys(report)) {
|
||||
retainedKeys.add(key);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
for (const key of await storedObjectKeys(storageService, report)) {
|
||||
if (retainedKeys.has(key)) {
|
||||
summary.sharedObjectsKept++;
|
||||
continue;
|
||||
}
|
||||
if (deletedKeys.has(key)) continue;
|
||||
const metadata = await storageService.getObjectMetadata(Config.s3.buckets.reports, key);
|
||||
if (metadata?.lastModified && metadata.lastModified.getTime() >= now.getTime()) {
|
||||
retainedKeys.add(key);
|
||||
summary.sharedObjectsKept++;
|
||||
continue;
|
||||
}
|
||||
if (!dryRun) {
|
||||
await storageService.deleteObject(Config.s3.buckets.reports, key);
|
||||
}
|
||||
deletedKeys.add(key);
|
||||
summary.objectsDeleted++;
|
||||
}
|
||||
if (!dryRun) {
|
||||
await reportSearchService?.deleteReport(reportId);
|
||||
await reportRepository.deleteReport(reportId);
|
||||
}
|
||||
const result = await deleteReportWithEvidence(deps, report, ledger, {now, dryRun});
|
||||
summary.objectsDeleted += result.objectsDeleted;
|
||||
summary.sharedObjectsKept += result.sharedObjectsKept;
|
||||
summary.deleted++;
|
||||
} catch (error) {
|
||||
summary.failed++;
|
||||
Logger.error({error, reportId: reportId.toString()}, 'Failed to delete an expired report');
|
||||
}
|
||||
}
|
||||
Logger.info({...summary, retentionDays: REPORT_RETENTION_DAYS}, 'Processed report retention');
|
||||
Logger.info(
|
||||
{
|
||||
...summary,
|
||||
retentionDays: policy.retentionDays,
|
||||
resolvedRetentionDays: policy.resolvedRetentionDays,
|
||||
},
|
||||
'Processed report retention',
|
||||
);
|
||||
return summary;
|
||||
}
|
||||
|
||||
const expireReportSnapshots: WorkerTaskHandler = async () => {
|
||||
const {reportRepository, storageService} = getWorkerDependencies();
|
||||
const {days, resolvedDays, dryRun} = Config.reportRetention;
|
||||
await processReportRetention(
|
||||
{reportRepository, storageService, reportSearchService: getReportSearchService()},
|
||||
{now: new Date(), dryRun: resolveReportRetentionDryRun(readOptionalEnv('FLUXER_REPORT_RETENTION_DRY_RUN'))},
|
||||
{now: new Date(), dryRun, policy: {retentionDays: days, resolvedRetentionDays: resolvedDays}},
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {createAttachmentID, createChannelID, createReportID, type ReportID} from '@app/api/BrandedTypes';
|
||||
import {createAttachmentID, createChannelID, createReportID, createUserID, type ReportID} from '@app/api/BrandedTypes';
|
||||
import {Config} from '@app/api/Config';
|
||||
import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';
|
||||
import type {MessageAttachment} from '@app/api/database/types/MessageTypes';
|
||||
@@ -10,22 +10,28 @@ import {
|
||||
type IARSubmissionRow,
|
||||
} from '@app/api/database/types/ReportTypes';
|
||||
import type {IARSubmission} from '@app/api/report/IReportRepository';
|
||||
import type {ReportDeletionDeps} from '@app/api/report/ReportDeletion';
|
||||
import {ReportRepository} from '@app/api/report/ReportRepository';
|
||||
import type {IReportSearchService} from '@app/api/search/IReportSearchService';
|
||||
import {MockStorageService} from '@app/api/test/mocks/MockStorageService';
|
||||
import {NoopLogger} from '@app/api/test/mocks/NoopLogger';
|
||||
import {InMemorySearchProvider} from '@app/api/test/search/InMemorySearchProvider';
|
||||
import {
|
||||
import expireReportSnapshots, {
|
||||
processReportRetention,
|
||||
type ReportRetentionDeps,
|
||||
resolveReportRetentionDryRun,
|
||||
type ReportRetentionOptions,
|
||||
type ReportRetentionPolicy,
|
||||
} from '@app/api/worker/tasks/ExpireReportSnapshots';
|
||||
import {clearWorkerDependencies, setWorkerDependenciesForTest} from '@app/api/worker/WorkerContext';
|
||||
import {serializeReportProfileSnapshot} from '@fluxer/schema/src/domains/report/ReportProfileSnapshotSchemas';
|
||||
import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
|
||||
import {ms} from 'itty-time';
|
||||
import {beforeEach, describe, expect, it} from 'vitest';
|
||||
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
|
||||
|
||||
const NOW = new Date('2027-06-01T12:00:00.000Z');
|
||||
const DAY_MS = ms('1 day');
|
||||
const REPORTS_BUCKET = Config.s3.buckets.reports;
|
||||
const WORKER_HELPERS = {logger: new NoopLogger()} as unknown as WorkerTaskHelpers;
|
||||
const DEFAULT_POLICY: ReportRetentionPolicy = {retentionDays: 365, resolvedRetentionDays: null};
|
||||
let sequence = 1_480_000_000_000_000_000n;
|
||||
|
||||
function nextId(): bigint {
|
||||
@@ -92,6 +98,7 @@ interface SeedOptions {
|
||||
profile?: boolean;
|
||||
legalHoldUntil?: Date | null;
|
||||
status?: number;
|
||||
resolvedDaysAgo?: number;
|
||||
snapshotJson?: string;
|
||||
}
|
||||
|
||||
@@ -113,7 +120,7 @@ describe('report retention', () => {
|
||||
let repository: ReportRepository;
|
||||
let storage: MockStorageService;
|
||||
let search: IReportSearchService;
|
||||
let deps: ReportRetentionDeps;
|
||||
let deps: ReportDeletionDeps;
|
||||
|
||||
beforeEach(() => {
|
||||
repository = new ReportRepository();
|
||||
@@ -158,7 +165,8 @@ describe('report retention', () => {
|
||||
...Object.fromEntries(IAR_SUBMISSION_COLUMNS.map((column) => [column, null])),
|
||||
report_id: id,
|
||||
reported_at: daysAgo(options.ageDays),
|
||||
status: options.status ?? 0,
|
||||
status: options.status ?? (options.resolvedDaysAgo === undefined ? 0 : 1),
|
||||
resolved_at: options.resolvedDaysAgo === undefined ? null : daysAgo(options.resolvedDaysAgo),
|
||||
report_type: 0,
|
||||
category: 'other',
|
||||
reported_channel_id: options.attachment?.channelId ?? null,
|
||||
@@ -195,7 +203,7 @@ describe('report retention', () => {
|
||||
const expired = await seed({ageDays: 365, attachment: newAttachment(), profile: true});
|
||||
const kept = await seed({ageDays: 364, attachment: newAttachment(), profile: true});
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false});
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
|
||||
await expectGone(expired);
|
||||
expect(storage.hasObject(REPORTS_BUCKET, expired.attachmentKey!)).toBe(false);
|
||||
@@ -216,7 +224,7 @@ describe('report retention', () => {
|
||||
const pending = await seed({ageDays: 400, status: 0});
|
||||
const resolved = await seed({ageDays: 400, status: 1});
|
||||
|
||||
await processReportRetention(deps, {now: NOW, dryRun: false});
|
||||
await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
|
||||
await expectGone(pending);
|
||||
await expectGone(resolved);
|
||||
@@ -227,7 +235,7 @@ describe('report retention', () => {
|
||||
const expired = await seed({ageDays: 500, attachment: shared});
|
||||
const retained = await seed({ageDays: 10, attachment: shared});
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false});
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
|
||||
await expectGone(expired);
|
||||
await expectKept(retained);
|
||||
@@ -241,7 +249,7 @@ describe('report retention', () => {
|
||||
const first = await seed({ageDays: 500, attachment: shared});
|
||||
const second = await seed({ageDays: 450, attachment: shared});
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false});
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
|
||||
await expectGone(first);
|
||||
await expectGone(second);
|
||||
@@ -261,7 +269,7 @@ describe('report retention', () => {
|
||||
const holdEnded = await seed({ageDays: 500, profile: true, legalHoldUntil: daysAgo(1)});
|
||||
const sharingWithHeld = await seed({ageDays: 500, attachment: shared});
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false});
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
|
||||
await expectKept(held);
|
||||
expect((await repository.getReport(held.reportId))?.legalHoldReason).toBe('Preservation request');
|
||||
@@ -275,7 +283,7 @@ describe('report retention', () => {
|
||||
const expired = await seed({ageDays: 366, attachment: newAttachment(), profile: true});
|
||||
const kept = await seed({ageDays: 1, profile: true});
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: true});
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: true, policy: DEFAULT_POLICY});
|
||||
|
||||
await expectKept(expired);
|
||||
await expectKept(kept);
|
||||
@@ -305,7 +313,10 @@ describe('report retention', () => {
|
||||
};
|
||||
|
||||
await expect(
|
||||
processReportRetention({...deps, reportRepository: failingRepository}, {now: NOW, dryRun: false, pageSize: 2}),
|
||||
processReportRetention(
|
||||
{...deps, reportRepository: failingRepository},
|
||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY, pageSize: 2},
|
||||
),
|
||||
).rejects.toThrow('read timeout');
|
||||
|
||||
await expectKept(expired);
|
||||
@@ -321,7 +332,7 @@ describe('report retention', () => {
|
||||
kept.push(await seed({ageDays: index, profile: true}));
|
||||
}
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, pageSize: 3});
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY, pageSize: 3});
|
||||
|
||||
expect(summary).toMatchObject({scanned: 14, expired: 7, deleted: 7, objectsDeleted: 7, failed: 0});
|
||||
for (const report of expired) await expectGone(report);
|
||||
@@ -332,14 +343,14 @@ describe('report retention', () => {
|
||||
const expired = await seed({ageDays: 400, attachment: newAttachment(), profile: true});
|
||||
storage.configure({shouldFailDelete: true});
|
||||
|
||||
const failed = await processReportRetention(deps, {now: NOW, dryRun: false});
|
||||
const failed = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
|
||||
expect(failed).toMatchObject({expired: 1, deleted: 0, failed: 1});
|
||||
expect(await repository.getReport(expired.reportId)).not.toBeNull();
|
||||
expect(await indexedIds()).toContain(expired.reportId.toString());
|
||||
|
||||
storage.configure({shouldFailDelete: false});
|
||||
const retried = await processReportRetention(deps, {now: NOW, dryRun: false});
|
||||
const retried = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
|
||||
expect(retried).toMatchObject({expired: 1, deleted: 1, failed: 0});
|
||||
await expectGone(expired);
|
||||
@@ -355,7 +366,7 @@ describe('report retention', () => {
|
||||
|
||||
const summary = await processReportRetention(
|
||||
{...deps, reportSearchService: failingSearch},
|
||||
{now: NOW, dryRun: false},
|
||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
|
||||
);
|
||||
|
||||
expect(summary).toMatchObject({deleted: 0, failed: 1});
|
||||
@@ -365,7 +376,10 @@ describe('report retention', () => {
|
||||
it('deletes rows when search is not configured', async () => {
|
||||
const expired = await seed({ageDays: 400, profile: true});
|
||||
|
||||
const summary = await processReportRetention({...deps, reportSearchService: null}, {now: NOW, dryRun: false});
|
||||
const summary = await processReportRetention(
|
||||
{...deps, reportSearchService: null},
|
||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
|
||||
);
|
||||
|
||||
expect(summary).toMatchObject({deleted: 1, failed: 0});
|
||||
expect(await repository.getReport(expired.reportId)).toBeNull();
|
||||
@@ -394,7 +408,7 @@ describe('report retention', () => {
|
||||
|
||||
const summary = await processReportRetention(
|
||||
{...deps, reportRepository: holdingRepository},
|
||||
{now: NOW, dryRun: false},
|
||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
|
||||
);
|
||||
|
||||
await expectKept(first);
|
||||
@@ -417,7 +431,7 @@ describe('report retention', () => {
|
||||
|
||||
const summary = await processReportRetention(
|
||||
{...deps, storageService: rewritingStorage},
|
||||
{now: NOW, dryRun: false},
|
||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
|
||||
);
|
||||
|
||||
await expectGone(expired);
|
||||
@@ -428,22 +442,156 @@ describe('report retention', () => {
|
||||
it('deletes profile images stored for a report whose snapshot cannot be read', async () => {
|
||||
const expired = await seed({ageDays: 400, profile: true, snapshotJson: '{"not":"a snapshot"'});
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false});
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
|
||||
expect(summary).toMatchObject({deleted: 1, objectsDeleted: 1});
|
||||
await expectGone(expired);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[undefined, true],
|
||||
['', true],
|
||||
['true', true],
|
||||
['1', true],
|
||||
['yes', true],
|
||||
['false', false],
|
||||
['FALSE', false],
|
||||
[' 0 ', false],
|
||||
])('FLUXER_REPORT_RETENTION_DRY_RUN=%j gives a dry run of %s', (raw, expected) => {
|
||||
expect(resolveReportRetentionDryRun(raw)).toBe(expected);
|
||||
function run(options: Partial<ReportRetentionOptions> = {}) {
|
||||
return processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY, ...options});
|
||||
}
|
||||
|
||||
it('deletes earlier when the retention period is shorter', async () => {
|
||||
const expired = await seed({ageDays: 30, profile: true});
|
||||
const kept = await seed({ageDays: 29, profile: true});
|
||||
|
||||
const summary = await run({policy: {retentionDays: 30, resolvedRetentionDays: null}});
|
||||
|
||||
await expectGone(expired);
|
||||
await expectKept(kept);
|
||||
expect(summary).toMatchObject({scanned: 2, expired: 1, deleted: 1});
|
||||
});
|
||||
|
||||
it('deletes a resolved report once the resolved-report period has passed since it was resolved', async () => {
|
||||
const policy = {retentionDays: 365, resolvedRetentionDays: 30};
|
||||
const resolvedLongAgo = await seed({ageDays: 100, resolvedDaysAgo: 30, attachment: newAttachment()});
|
||||
const resolvedRecently = await seed({ageDays: 100, resolvedDaysAgo: 29, profile: true});
|
||||
const pending = await seed({ageDays: 100, profile: true});
|
||||
const resolvedWithoutTime = await seed({ageDays: 100, status: 1, profile: true});
|
||||
|
||||
const summary = await run({policy});
|
||||
|
||||
await expectGone(resolvedLongAgo);
|
||||
expect(storage.hasObject(REPORTS_BUCKET, resolvedLongAgo.attachmentKey!)).toBe(false);
|
||||
await expectKept(resolvedRecently);
|
||||
await expectKept(pending);
|
||||
await expectKept(resolvedWithoutTime);
|
||||
expect(summary).toMatchObject({scanned: 4, expired: 1, deleted: 1, held: 0});
|
||||
});
|
||||
|
||||
it('applies whichever rule ends first', async () => {
|
||||
const policy = {retentionDays: 100, resolvedRetentionDays: 300};
|
||||
const byAge = await seed({ageDays: 100, resolvedDaysAgo: 10});
|
||||
const neither = await seed({ageDays: 99, resolvedDaysAgo: 90});
|
||||
|
||||
await run({policy});
|
||||
|
||||
await expectGone(byAge);
|
||||
await expectKept(neither);
|
||||
});
|
||||
|
||||
it('keeps a held resolved report past the resolved-report period and keeps its evidence', async () => {
|
||||
const shared = newAttachment();
|
||||
const held = await seed({
|
||||
ageDays: 50,
|
||||
resolvedDaysAgo: 40,
|
||||
attachment: shared,
|
||||
profile: true,
|
||||
legalHoldUntil: new Date(NOW.getTime() + DAY_MS),
|
||||
});
|
||||
const sharingWithHeld = await seed({ageDays: 50, resolvedDaysAgo: 40, attachment: shared});
|
||||
|
||||
const summary = await run({policy: {retentionDays: 365, resolvedRetentionDays: 7}});
|
||||
|
||||
await expectKept(held);
|
||||
await expectGone(sharingWithHeld);
|
||||
expect(storage.hasObject(REPORTS_BUCKET, attachmentKey(shared))).toBe(true);
|
||||
expect(summary).toMatchObject({expired: 1, held: 1, deleted: 1, sharedObjectsKept: 1});
|
||||
});
|
||||
|
||||
it('frees the reporter to report the same account again once the report is deleted', async () => {
|
||||
const reporterId = nextId();
|
||||
const reportedUserId = nextId();
|
||||
const reportId = createReportID(nextId());
|
||||
await repository.createReport({
|
||||
...Object.fromEntries(IAR_SUBMISSION_COLUMNS.map((column) => [column, null])),
|
||||
report_id: BigInt(reportId),
|
||||
reporter_id: reporterId,
|
||||
reported_user_id: reportedUserId,
|
||||
reported_at: daysAgo(400),
|
||||
status: 0,
|
||||
report_type: 1,
|
||||
category: 'harassment',
|
||||
} as IARSubmissionRow);
|
||||
const reservation = {
|
||||
reporter_id: createUserID(reporterId),
|
||||
reported_user_id: createUserID(reportedUserId),
|
||||
report_id: reportId,
|
||||
reported_at: daysAgo(400),
|
||||
};
|
||||
expect(await repository.reserveUserReportByReporter(reservation)).toBe(true);
|
||||
|
||||
await run();
|
||||
|
||||
expect(await repository.getReport(reportId)).toBeNull();
|
||||
expect(await repository.reserveUserReportByReporter({...reservation, report_id: createReportID(nextId())})).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
describe('the daily task', () => {
|
||||
const configured = {...Config.reportRetention};
|
||||
|
||||
beforeEach(() => {
|
||||
setWorkerDependenciesForTest({reportRepository: repository, storageService: storage});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
Object.assign(Config.reportRetention, configured);
|
||||
clearWorkerDependencies();
|
||||
});
|
||||
|
||||
async function runDailyTask() {
|
||||
vi.useFakeTimers({now: NOW, toFake: ['Date']});
|
||||
await expireReportSnapshots({}, WORKER_HELPERS);
|
||||
}
|
||||
|
||||
it('deletes for real with the default settings', async () => {
|
||||
expect(configured).toEqual({days: 365, resolvedDays: null, dryRun: false});
|
||||
const expired = await seed({ageDays: 366, attachment: newAttachment(), profile: true});
|
||||
const kept = await seed({ageDays: 364, profile: true});
|
||||
|
||||
await runDailyTask();
|
||||
|
||||
expect(await repository.getReport(expired.reportId)).toBeNull();
|
||||
expect(storage.hasObject(REPORTS_BUCKET, expired.attachmentKey!)).toBe(false);
|
||||
expect(storage.hasObject(REPORTS_BUCKET, expired.profileKey!)).toBe(false);
|
||||
expect(await repository.getReport(kept.reportId)).not.toBeNull();
|
||||
});
|
||||
|
||||
it('only counts what it would delete when FLUXER_REPORT_RETENTION_DRY_RUN is true', async () => {
|
||||
Object.assign(Config.reportRetention, {dryRun: true});
|
||||
const expired = await seed({ageDays: 366, attachment: newAttachment(), profile: true});
|
||||
|
||||
await runDailyTask();
|
||||
|
||||
expect(await repository.getReport(expired.reportId)).not.toBeNull();
|
||||
expect(storage.deleteObjectSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('reads both day settings from the config', async () => {
|
||||
Object.assign(Config.reportRetention, {days: 60, resolvedDays: 5});
|
||||
const byAge = await seed({ageDays: 60});
|
||||
const byResolution = await seed({ageDays: 20, resolvedDaysAgo: 5});
|
||||
const kept = await seed({ageDays: 20, resolvedDaysAgo: 4});
|
||||
|
||||
await runDailyTask();
|
||||
|
||||
expect(await repository.getReport(byAge.reportId)).toBeNull();
|
||||
expect(await repository.getReport(byResolution.reportId)).toBeNull();
|
||||
expect(await repository.getReport(kept.reportId)).not.toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user