feat(reports): keep reports until resolved, then by outcome (#3493)

This commit is contained in:
Hampus authored and GitHub committed 2026-10-15 16:40:51 +02:00
1 parent f5c30fdf8f
commit 4f88872a3c
18 files changed
+673 -238

No files matched your search

+3 -5
View File
@@ -353,12 +353,10 @@ FLUXER_DISCOVERY_ENABLED=true
#FLUXER_AUTO_JOIN_INVITE_CODE=
#FLUXER_DELETION_GRACE_PERIOD_HOURS=336
# Report retention. A daily job deletes each report this many days after it was
# filed, with its evidence copies and search entry, unless a legal hold is set.
# Report retention. A daily job deletes each resolved report this many days
# after it was resolved, with its evidence copies and search entry, unless a
# legal hold is set. Pending reports are kept until they are resolved.
#FLUXER_REPORT_RETENTION_DAYS=365
# Delete resolved reports this many days after they were resolved, when that
# comes first. Unset leaves them to FLUXER_REPORT_RETENTION_DAYS.
#FLUXER_RESOLVED_REPORT_RETENTION_DAYS=
# true only logs what the job would delete. Deleted evidence cannot be
# restored, so check the "Processed report retention" log line of the worker
# in a dry run first if you are unsure.
-1
View File
@@ -159,7 +159,6 @@ x-fluxer-env: &fluxer-env
FLUXER_DISCOVERY_MIN_MEMBER_COUNT: ${FLUXER_DISCOVERY_MIN_MEMBER_COUNT:-}
FLUXER_DELETION_GRACE_PERIOD_HOURS: ${FLUXER_DELETION_GRACE_PERIOD_HOURS:-}
FLUXER_REPORT_RETENTION_DAYS: ${FLUXER_REPORT_RETENTION_DAYS:-}
FLUXER_RESOLVED_REPORT_RETENTION_DAYS: ${FLUXER_RESOLVED_REPORT_RETENTION_DAYS:-}
FLUXER_REPORT_RETENTION_DRY_RUN: ${FLUXER_REPORT_RETENTION_DRY_RUN:-}
FLUXER_API_WORKER_LANE_CONCURRENCY_OVERRIDES: ${FLUXER_API_WORKER_LANE_CONCURRENCY_OVERRIDES:-}
-1
View File
@@ -449,7 +449,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
inactivityDeletionThresholdDays: master.inactivity_deletion_threshold_days,
reportRetention: {
days: master.report_retention.days,
resolvedDays: master.report_retention.resolved_days,
dryRun: master.report_retention.dry_run,
},
push: {
@@ -3,6 +3,7 @@
import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';
import {createReportID, type ReportID} from '@app/api/BrandedTypes';
import {IAR_SUBMISSION_COLUMNS, type IARSubmissionRow} from '@app/api/database/types/ReportTypes';
import {EnforcementDecisionRepository} from '@app/api/enforcement/EnforcementDecisionRepository';
import {ReportRepository} from '@app/api/report/ReportRepository';
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
import {MockStorageService} from '@app/api/test/mocks/MockStorageService';
@@ -47,6 +48,7 @@ describe('report legal hold', () => {
...Object.fromEntries(IAR_SUBMISSION_COLUMNS.map((column) => [column, null])),
report_id: sequence,
reported_at: new Date(Date.now() - ageDays * ms('1 day')),
resolved_at: new Date(Date.now() - ageDays * ms('1 day')),
status: 1,
report_type: 1,
category: 'harassment',
@@ -69,8 +71,13 @@ describe('report legal hold', () => {
async function runRetention() {
return processReportRetention(
{reportRepository: repository, storageService: new MockStorageService(), reportSearchService: null},
{now: new Date(), dryRun: false, policy: {retentionDays: 365, resolvedRetentionDays: null}},
{
reportRepository: repository,
storageService: new MockStorageService(),
reportSearchService: null,
decisionRepository: new EnforcementDecisionRepository(),
},
{now: new Date(), dryRun: false, policy: {retentionDays: 365}},
);
}
-1
View File
@@ -308,7 +308,6 @@ export interface APIConfig {
deletionGracePeriodHours: number;
reportRetention: {
days: number;
resolvedDays: number | null;
dryRun: boolean;
};
inactivityDeletionThresholdDays?: number;
@@ -40,6 +40,16 @@ const LIST_NOTES_QUERY = EnforcementDecisionNotes.select({
orderBy: {col: 'note_id', direction: 'ASC'},
});
type DecisionReportLink = Pick<EnforcementDecisionRow, 'decision_id' | 'report_ids' | 'created_at'>;
function listReportLinksQuery(limit: number, paged: boolean) {
return EnforcementDecisions.select({
columns: ['decision_id', 'report_ids', 'created_at'],
...(paged ? {where: EnforcementDecisions.where.tokenGt('decision_id', 'after')} : {}),
limit,
});
}
const REVIEW_QUEUE_BUCKET = 0;
const LIST_REVIEW_QUEUE_QUERY = EnforcementReviewQueue.select({
columns: ['decision_id'],
@@ -162,6 +172,12 @@ export class EnforcementDecisionRepository {
return this.getMany(keys.map((key) => key.decision_id));
}
async listReportLinks(limit: number, after?: bigint): Promise<Array<DecisionReportLink>> {
return fetchMany<DecisionReportLink>(
listReportLinksQuery(limit, after !== undefined).bind(after !== undefined ? {after} : {}),
);
}
async end(decisionId: bigint, end: EnforcementDecisionEnd): Promise<void> {
await upsertOne(EnforcementDecisions.patchByPk({decision_id: decisionId}, endPatch(end)));
}
+32 -3
View File
@@ -79,13 +79,12 @@ async function storedObjectKeys(storageService: IStorageService, report: IARSubm
return keys;
}
export async function deleteReportWithEvidence(
deps: ReportDeletionDeps,
async function deleteReportEvidence(
storageService: IStorageService,
report: IARSubmission,
ledger: ReportEvidenceLedger,
options: {now: Date; dryRun: boolean},
): Promise<ReportDeletionResult> {
const {reportRepository, storageService, reportSearchService} = deps;
const result: ReportDeletionResult = {objectsDeleted: 0, sharedObjectsKept: 0};
for (const key of await storedObjectKeys(storageService, report)) {
if (ledger.retainedKeys.has(key)) {
@@ -105,6 +104,17 @@ export async function deleteReportWithEvidence(
ledger.deletedKeys.add(key);
result.objectsDeleted++;
}
return result;
}
export async function deleteReportWithEvidence(
deps: ReportDeletionDeps,
report: IARSubmission,
ledger: ReportEvidenceLedger,
options: {now: Date; dryRun: boolean},
): Promise<ReportDeletionResult> {
const {reportRepository, storageService, reportSearchService} = deps;
const result = await deleteReportEvidence(storageService, report, ledger, options);
if (!options.dryRun) {
await reportSearchService?.deleteReport(report.reportId);
await reportRepository.deleteReport(report.reportId);
@@ -112,6 +122,25 @@ export async function deleteReportWithEvidence(
return result;
}
export async function clearCopiedContextWithEvidence(
deps: ReportDeletionDeps,
report: IARSubmission,
ledger: ReportEvidenceLedger,
options: {now: Date; dryRun: boolean},
): Promise<ReportDeletionResult> {
const {reportRepository, storageService, reportSearchService} = deps;
const result = await deleteReportEvidence(storageService, report, ledger, options);
if (!options.dryRun && (await reportRepository.clearCopiedContext(report.reportId, report.reportType))) {
await reportSearchService?.updateReport({
...report,
messageContext: null,
reportedProfileSnapshot: null,
additionalInfo: null,
});
}
return result;
}
async function loadDeletableReport(
reportRepository: ReportRepository,
reportId: ReportID,
@@ -186,6 +186,20 @@ export class ReportRepository {
return {...report, legalHoldUntil, legalHoldReason};
}
async clearCopiedContext(reportId: ReportID, reportType: number): Promise<boolean> {
return executeConditional(
IARSubmissions.conditionalPatchByPk(
{report_id: reportId},
{
message_context: Db.set(null),
reported_profile_snapshot: Db.set(null),
additional_info: Db.set(null),
},
{report_type: reportType},
),
);
}
async clearReporterEmail(reportId: ReportID, reporterId: UserID): Promise<boolean> {
return executeConditional(
IARSubmissions.conditionalPatchByPk(
@@ -2,23 +2,36 @@
import type {ReportID} from '@app/api/BrandedTypes';
import {Config} from '@app/api/Config';
import {EnforcementDecisionRepository} from '@app/api/enforcement/EnforcementDecisionRepository';
import {Logger} from '@app/api/Logger';
import {
clearCopiedContextWithEvidence,
deleteReportWithEvidence,
forEachStoredReport,
isReportUnderLegalHold,
type ReportDeletionDeps,
type ReportEvidenceLedger,
referencedObjectKeys,
} from '@app/api/report/ReportDeletion';
import {type IARSubmission, ReportStatus} from '@app/api/report/ReportModels';
import {getReportSearchService} from '@app/api/SearchFactory';
import {EnforcementDecisionEvidence} from '@app/api/Tables';
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
import {ms} from 'itty-time';
export const REPORT_COMPLAINT_WINDOW_DAYS = 184;
export const CHILD_SAFETY_REPORT_RETENTION_DAYS = 365;
const CHILD_SAFETY_CATEGORY = 'child_safety';
const DECISION_EVIDENCE_RETENTION_MS = EnforcementDecisionEvidence.defaultTtlSeconds! * 1000;
const DEFAULT_SCAN_PAGE_SIZE = 500;
export interface ReportRetentionPolicy {
retentionDays: number;
resolvedRetentionDays: number | null;
}
export interface ReportRetentionDeps extends ReportDeletionDeps {
decisionRepository: Pick<EnforcementDecisionRepository, 'listReportLinks'>;
}
export interface ReportRetentionOptions {
@@ -31,65 +44,136 @@ export interface ReportRetentionOptions {
export interface ReportRetentionSummary {
dryRun: boolean;
scanned: number;
pending: number;
expired: number;
contextExpired: number;
held: number;
deleted: number;
contextCleared: number;
objectsDeleted: number;
sharedObjectsKept: number;
failed: number;
}
type ReportRetentionState = 'expired' | 'held' | 'retained';
function reportExpiresAt(report: IARSubmission, policy: ReportRetentionPolicy): number {
const reportedAt = report.reportedAt instanceof Date ? report.reportedAt.getTime() : Number.NaN;
const byAge = reportedAt + policy.retentionDays * ms('1 day');
if (policy.resolvedRetentionDays === null || report.status !== ReportStatus.RESOLVED) {
return byAge;
}
const resolvedAt = report.resolvedAt instanceof Date ? report.resolvedAt.getTime() : Number.NaN;
if (Number.isNaN(resolvedAt)) {
return byAge;
}
return Math.min(byAge, resolvedAt + policy.resolvedRetentionDays * ms('1 day'));
export interface ReportRetentionDates {
deleteAt: number | null;
contextExpiresAt: number | null;
}
function reportRetentionState(report: IARSubmission, now: Date, policy: ReportRetentionPolicy): ReportRetentionState {
const expiresAt = reportExpiresAt(report, policy);
if (Number.isNaN(expiresAt) || expiresAt > now.getTime()) {
type ReportRetentionState = 'expired' | 'context_expired' | 'held' | 'retained';
const KEEP_UNTIL_RESOLVED: ReportRetentionDates = {deleteAt: null, contextExpiresAt: null};
export function reportRetentionDates(
report: IARSubmission,
policy: ReportRetentionPolicy,
decidedAt: Date | null,
): ReportRetentionDates {
if (report.status !== ReportStatus.RESOLVED) {
return KEEP_UNTIL_RESOLVED;
}
const resolvedAt = (report.resolvedAt ?? report.reportedAt).getTime();
if (Number.isNaN(resolvedAt)) {
return KEEP_UNTIL_RESOLVED;
}
const childSafety = report.category === CHILD_SAFETY_CATEGORY;
const ends = [resolvedAt + policy.retentionDays * ms('1 day')];
if (childSafety) {
ends.push(resolvedAt + CHILD_SAFETY_REPORT_RETENTION_DAYS * ms('1 day'));
}
if (decidedAt) {
ends.push(decidedAt.getTime() + DECISION_EVIDENCE_RETENTION_MS);
}
const dropsContextEarly = report.reporterOutcome === 'no_violation' && !childSafety && !decidedAt;
return {
deleteAt: Math.max(...ends),
contextExpiresAt: dropsContextEarly ? resolvedAt + REPORT_COMPLAINT_WINDOW_DAYS * ms('1 day') : null,
};
}
function hasCopiedContext(report: IARSubmission): boolean {
return (
(report.messageContext?.length ?? 0) > 0 ||
report.reportedProfileSnapshot !== null ||
report.additionalInfo !== null
);
}
function reportRetentionState(
report: IARSubmission,
now: Date,
policy: ReportRetentionPolicy,
decidedAt: Date | null,
): ReportRetentionState {
const {deleteAt, contextExpiresAt} = reportRetentionDates(report, policy, decidedAt);
const due =
deleteAt !== null && deleteAt <= now.getTime()
? 'expired'
: contextExpiresAt !== null && contextExpiresAt <= now.getTime() && hasCopiedContext(report)
? 'context_expired'
: null;
if (due === null) {
return 'retained';
}
if (isReportUnderLegalHold(report, now)) {
return 'held';
return isReportUnderLegalHold(report, now) ? 'held' : due;
}
async function latestDecisionByReport(
decisionRepository: ReportRetentionDeps['decisionRepository'],
pageSize: number,
): Promise<Map<bigint, Date>> {
const decidedAt = new Map<bigint, Date>();
let cursor: bigint | undefined;
while (true) {
const page = await decisionRepository.listReportLinks(pageSize, cursor);
for (const decision of page) {
for (const reportId of decision.report_ids ?? []) {
const known = decidedAt.get(reportId);
if (!known || known.getTime() < decision.created_at.getTime()) {
decidedAt.set(reportId, decision.created_at);
}
}
}
if (page.length < pageSize) return decidedAt;
cursor = page[page.length - 1]!.decision_id;
}
return 'expired';
}
export async function processReportRetention(
deps: ReportDeletionDeps,
deps: ReportRetentionDeps,
options: ReportRetentionOptions,
): Promise<ReportRetentionSummary> {
const {reportRepository} = deps;
const {now, dryRun, policy} = options;
const pageSize = options.pageSize ?? DEFAULT_SCAN_PAGE_SIZE;
const summary: ReportRetentionSummary = {
dryRun,
scanned: 0,
pending: 0,
expired: 0,
contextExpired: 0,
held: 0,
deleted: 0,
contextCleared: 0,
objectsDeleted: 0,
sharedObjectsKept: 0,
failed: 0,
};
const decidedAt = await latestDecisionByReport(deps.decisionRepository, pageSize);
const stateOf = (report: IARSubmission) =>
reportRetentionState(report, now, policy, decidedAt.get(report.reportId) ?? null);
const retainedKeys = new Set<string>();
const expiredIds: Array<ReportID> = [];
const due: Array<{reportId: ReportID; state: 'expired' | 'context_expired'}> = [];
await forEachStoredReport(
reportRepository,
(report) => {
summary.scanned++;
const state = reportRetentionState(report, now, policy);
if (state === 'expired') {
expiredIds.push(report.reportId);
if (report.status !== ReportStatus.RESOLVED) {
summary.pending++;
}
const state = stateOf(report);
if (state === 'expired' || state === 'context_expired') {
due.push({reportId: report.reportId, state});
return;
}
if (state === 'held') {
@@ -99,35 +183,37 @@ export async function processReportRetention(
retainedKeys.add(key);
}
},
options.pageSize,
pageSize,
);
summary.expired = expiredIds.length;
const ledger = {retainedKeys, deletedKeys: new Set<string>()};
for (const reportId of expiredIds) {
summary.expired = due.filter((entry) => entry.state === 'expired').length;
summary.contextExpired = due.length - summary.expired;
const ledger: ReportEvidenceLedger = {retainedKeys, deletedKeys: new Set<string>()};
for (const {reportId, state} of due) {
try {
const report = await reportRepository.getReport(reportId);
if (!report) continue;
if (reportRetentionState(report, now, policy) !== 'expired') {
if (stateOf(report) !== state) {
for (const key of referencedObjectKeys(report)) {
retainedKeys.add(key);
}
continue;
}
const result = await deleteReportWithEvidence(deps, report, ledger, {now, dryRun});
const apply = state === 'expired' ? deleteReportWithEvidence : clearCopiedContextWithEvidence;
const result = await apply(deps, report, ledger, {now, dryRun});
summary.objectsDeleted += result.objectsDeleted;
summary.sharedObjectsKept += result.sharedObjectsKept;
summary.deleted++;
if (state === 'expired') {
summary.deleted++;
} else {
summary.contextCleared++;
}
} catch (error) {
summary.failed++;
Logger.error({error, reportId: reportId.toString()}, 'Failed to delete an expired report');
Logger.error({error, reportId: reportId.toString(), state}, 'Failed to apply report retention');
}
}
Logger.info(
{
...summary,
retentionDays: policy.retentionDays,
resolvedRetentionDays: policy.resolvedRetentionDays,
},
{...summary, retentionDays: policy.retentionDays, complaintWindowDays: REPORT_COMPLAINT_WINDOW_DAYS},
'Processed report retention',
);
return summary;
@@ -135,10 +221,15 @@ export async function processReportRetention(
const expireReportSnapshots: WorkerTaskHandler = async () => {
const {reportRepository, storageService} = getWorkerDependencies();
const {days, resolvedDays, dryRun} = Config.reportRetention;
const {days, dryRun} = Config.reportRetention;
await processReportRetention(
{reportRepository, storageService, reportSearchService: getReportSearchService()},
{now: new Date(), dryRun, policy: {retentionDays: days, resolvedRetentionDays: resolvedDays}},
{
reportRepository,
storageService,
reportSearchService: getReportSearchService(),
decisionRepository: new EnforcementDecisionRepository(),
},
{now: new Date(), dryRun, policy: {retentionDays: days}},
);
};
@@ -3,13 +3,14 @@
import {createAttachmentID, createChannelID, createReportID, createUserID, type ReportID} from '@app/api/BrandedTypes';
import {Config} from '@app/api/Config';
import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';
import {ENFORCEMENT_DECISION_COLUMNS, type EnforcementDecisionRow} from '@app/api/database/types/EnforcementTypes';
import type {MessageAttachment} from '@app/api/database/types/MessageTypes';
import {
IAR_SUBMISSION_COLUMNS,
type IARMessageContextRow,
type IARSubmissionRow,
} from '@app/api/database/types/ReportTypes';
import type {ReportDeletionDeps} from '@app/api/report/ReportDeletion';
import {EnforcementDecisionRepository} from '@app/api/enforcement/EnforcementDecisionRepository';
import type {IARSubmission} from '@app/api/report/ReportModels';
import {ReportRepository} from '@app/api/report/ReportRepository';
import type {IReportSearchService} from '@app/api/search/IReportSearchService';
@@ -17,11 +18,17 @@ import {MockStorageService} from '@app/api/test/mocks/MockStorageService';
import {NoopLogger} from '@app/api/test/mocks/NoopLogger';
import {InMemorySearchProvider} from '@app/api/test/search/InMemorySearchProvider';
import expireReportSnapshots, {
CHILD_SAFETY_REPORT_RETENTION_DAYS,
processReportRetention,
REPORT_COMPLAINT_WINDOW_DAYS,
type ReportRetentionDeps,
type ReportRetentionOptions,
type ReportRetentionPolicy,
reportRetentionDates,
} from '@app/api/worker/tasks/ExpireReportSnapshots';
import {clearWorkerDependencies, setWorkerDependenciesForTest} from '@app/api/worker/WorkerContext';
import {DeletionReasons} from '@fluxer/constants/src/Core';
import type {ReportItemOutcome} from '@fluxer/schema/src/domains/admin/AdminSchemas';
import {serializeReportProfileSnapshot} from '@fluxer/schema/src/domains/report/ReportProfileSnapshotSchemas';
import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
import {ms} from 'itty-time';
@@ -31,7 +38,7 @@ const NOW = new Date('2027-06-01T12:00:00.000Z');
const DAY_MS = ms('1 day');
const REPORTS_BUCKET = Config.s3.buckets.reports;
const WORKER_HELPERS = {logger: new NoopLogger()} as unknown as WorkerTaskHelpers;
const DEFAULT_POLICY: ReportRetentionPolicy = {retentionDays: 365, resolvedRetentionDays: null};
const DEFAULT_POLICY: ReportRetentionPolicy = {retentionDays: 365};
let sequence = 1_480_000_000_000_000_000n;
function nextId(): bigint {
@@ -93,12 +100,16 @@ interface SharedAttachment {
}
interface SeedOptions {
ageDays: number;
ageDays?: number;
resolvedDaysAgo?: number;
attachment?: SharedAttachment;
profile?: boolean;
legalHoldUntil?: Date | null;
status?: number;
resolvedDaysAgo?: number;
category?: string;
outcome?: ReportItemOutcome | null;
additionalInfo?: string | null;
reporterId?: bigint | null;
snapshotJson?: string;
}
@@ -116,20 +127,96 @@ function attachmentKey(attachment: SharedAttachment): string {
return makeAttachmentCdnKey(createChannelID(attachment.channelId), attachment.attachmentId, attachment.filename);
}
function storedReport(overrides: Partial<IARSubmission>): IARSubmission {
return {
reportId: createReportID(nextId()),
reportedAt: daysAgo(500),
status: 1,
resolvedAt: daysAgo(10),
category: 'harassment',
reporterOutcome: 'removed',
...overrides,
} as IARSubmission;
}
describe('report retention dates', () => {
const day = DAY_MS;
it('keeps a pending report until it is resolved', () => {
expect(reportRetentionDates(storedReport({status: 0, resolvedAt: null}), DEFAULT_POLICY, null)).toEqual({
deleteAt: null,
contextExpiresAt: null,
});
});
it('deletes an actioned report one year after it was resolved and keeps its context until then', () => {
const report = storedReport({});
expect(reportRetentionDates(report, DEFAULT_POLICY, null)).toEqual({
deleteAt: report.resolvedAt!.getTime() + 365 * day,
contextExpiresAt: null,
});
});
it('drops the context of a no violation report once the complaint window has passed', () => {
const report = storedReport({reporterOutcome: 'no_violation'});
const dates = reportRetentionDates(report, DEFAULT_POLICY, null);
expect(dates.contextExpiresAt).toBe(report.resolvedAt!.getTime() + REPORT_COMPLAINT_WINDOW_DAYS * day);
expect(REPORT_COMPLAINT_WINDOW_DAYS).toBeGreaterThanOrEqual(183);
expect(dates.deleteAt).toBe(report.resolvedAt!.getTime() + 365 * day);
});
it('keeps a child safety report and its context for at least a year whatever the configured period', () => {
const report = storedReport({category: 'child_safety', reporterOutcome: 'no_violation'});
expect(reportRetentionDates(report, {retentionDays: 30}, null)).toEqual({
deleteAt: report.resolvedAt!.getTime() + CHILD_SAFETY_REPORT_RETENTION_DAYS * day,
contextExpiresAt: null,
});
expect(CHILD_SAFETY_REPORT_RETENTION_DAYS).toBeGreaterThanOrEqual(365);
});
it('keeps a report linked to a decision as long as the decision evidence and keeps its context', () => {
const report = storedReport({resolvedAt: daysAgo(300), reporterOutcome: 'no_violation'});
const decidedAt = daysAgo(100);
expect(reportRetentionDates(report, DEFAULT_POLICY, decidedAt)).toEqual({
deleteAt: decidedAt.getTime() + 365 * day,
contextExpiresAt: null,
});
});
it('uses the longer period when the configured one is longer than a floor', () => {
const report = storedReport({category: 'child_safety'});
expect(reportRetentionDates(report, {retentionDays: 730}, null).deleteAt).toBe(
report.resolvedAt!.getTime() + 730 * day,
);
});
it('counts from the filing time for a resolved report without a resolution time', () => {
const report = storedReport({resolvedAt: null});
expect(reportRetentionDates(report, DEFAULT_POLICY, null).deleteAt).toBe(report.reportedAt.getTime() + 365 * day);
});
});
describe('report retention', () => {
let repository: ReportRepository;
let decisions: EnforcementDecisionRepository;
let storage: MockStorageService;
let search: IReportSearchService;
let deps: ReportDeletionDeps;
let deps: ReportRetentionDeps;
beforeEach(() => {
repository = new ReportRepository();
decisions = new EnforcementDecisionRepository();
storage = new MockStorageService();
search = new InMemorySearchProvider().getReportSearchService();
deps = {reportRepository: repository, storageService: storage, reportSearchService: search};
deps = {
reportRepository: repository,
storageService: storage,
reportSearchService: search,
decisionRepository: decisions,
};
});
async function seed(options: SeedOptions): Promise<SeededReport> {
async function seed(options: SeedOptions = {}): Promise<SeededReport> {
const id = nextId();
const reportId = createReportID(id);
let storedAttachmentKey: string | null = null;
@@ -161,14 +248,18 @@ describe('report retention', () => {
guild: null,
});
}
const resolved = options.resolvedDaysAgo !== undefined;
const row = {
...Object.fromEntries(IAR_SUBMISSION_COLUMNS.map((column) => [column, null])),
report_id: id,
reported_at: daysAgo(options.ageDays),
status: options.status ?? (options.resolvedDaysAgo === undefined ? 0 : 1),
resolved_at: options.resolvedDaysAgo === undefined ? null : daysAgo(options.resolvedDaysAgo),
reporter_id: options.reporterId ?? null,
reported_at: daysAgo(options.ageDays ?? (options.resolvedDaysAgo ?? 0) + 5),
status: options.status ?? (resolved ? 1 : 0),
resolved_at: resolved ? daysAgo(options.resolvedDaysAgo!) : null,
reporter_outcome: options.outcome === undefined ? (resolved ? 'removed' : null) : options.outcome,
report_type: 0,
category: 'other',
category: options.category ?? 'other',
additional_info: options.additionalInfo ?? null,
reported_channel_id: options.attachment?.channelId ?? null,
message_context: messageContext,
reported_profile_snapshot: snapshot,
@@ -180,6 +271,30 @@ describe('report retention', () => {
return {reportId, attachmentKey: storedAttachmentKey, profileKey};
}
async function linkDecision(reportIds: Array<ReportID>, createdAt: Date): Promise<void> {
const decisionId = nextId();
const decision = {
...Object.fromEntries(ENFORCEMENT_DECISION_COLUMNS.map((column) => [column, null])),
decision_id: decisionId,
subject_type: 'user',
subject_id: nextId(),
kind: 'warning',
reason_code: DeletionReasons.HARASSMENT_OR_BULLYING,
source: 'report',
automation: 'none',
report_ids: new Set<bigint>(reportIds),
decided_by: nextId(),
created_at: createdAt,
status: 'active',
notice_status: 'sent',
} as EnforcementDecisionRow;
await decisions.create({
decision,
evidence: null,
firstNote: {decision_id: decisionId, note_id: nextId(), author_id: nextId(), body: 'note', created_at: createdAt},
});
}
async function indexedIds(): Promise<Array<string>> {
const result = await search.searchReports('', {}, {limit: 1000});
return result.hits.map((hit) => hit.id).sort();
@@ -199,11 +314,27 @@ describe('report retention', () => {
if (seeded.profileKey) expect(storage.hasObject(REPORTS_BUCKET, seeded.profileKey)).toBe(false);
}
it('deletes a report 365 days after it was filed with its objects, search document and row', async () => {
const expired = await seed({ageDays: 365, attachment: newAttachment(), profile: true});
const kept = await seed({ageDays: 364, attachment: newAttachment(), profile: true});
async function expectContextCleared(seeded: SeededReport): Promise<void> {
const report = await repository.getReport(seeded.reportId);
expect(report).not.toBeNull();
expect(report?.messageContext).toBeNull();
expect(report?.reportedProfileSnapshot).toBeNull();
expect(report?.additionalInfo).toBeNull();
expect(await indexedIds()).toContain(seeded.reportId.toString());
for (const key of [seeded.attachmentKey, seeded.profileKey]) {
if (key) expect(storage.hasObject(REPORTS_BUCKET, key)).toBe(false);
}
}
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
function run(options: Partial<ReportRetentionOptions> = {}) {
return processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY, ...options});
}
it('deletes a report 365 days after it was resolved with its objects, search document and row', async () => {
const expired = await seed({resolvedDaysAgo: 365, attachment: newAttachment(), profile: true});
const kept = await seed({resolvedDaysAgo: 364, attachment: newAttachment(), profile: true});
const summary = await run();
await expectGone(expired);
expect(storage.hasObject(REPORTS_BUCKET, expired.attachmentKey!)).toBe(false);
@@ -211,31 +342,175 @@ describe('report retention', () => {
expect(summary).toEqual({
dryRun: false,
scanned: 2,
pending: 0,
expired: 1,
contextExpired: 0,
held: 0,
deleted: 1,
contextCleared: 0,
objectsDeleted: 2,
sharedObjectsKept: 0,
failed: 0,
});
});
it('deletes expired reports whatever their status', async () => {
const pending = await seed({ageDays: 400, status: 0});
const resolved = await seed({ageDays: 400, status: 1});
it('keeps a pending report however old it is', async () => {
const pending = await seed({ageDays: 4000, attachment: newAttachment(), profile: true});
await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
const summary = await run({policy: {retentionDays: 1}});
await expectGone(pending);
await expectGone(resolved);
await expectKept(pending);
expect(summary).toMatchObject({scanned: 1, pending: 1, expired: 0, deleted: 0, objectsDeleted: 0});
});
it('drops the copied context of a no violation report after the complaint window and keeps the record', async () => {
const outsideWindow = await seed({
resolvedDaysAgo: REPORT_COMPLAINT_WINDOW_DAYS,
outcome: 'no_violation',
attachment: newAttachment(),
profile: true,
additionalInfo: 'They keep messaging my friends',
});
const insideWindow = await seed({
resolvedDaysAgo: REPORT_COMPLAINT_WINDOW_DAYS - 1,
outcome: 'no_violation',
attachment: newAttachment(),
profile: true,
additionalInfo: 'Still open to a complaint',
});
const actioned = await seed({
resolvedDaysAgo: REPORT_COMPLAINT_WINDOW_DAYS + 10,
attachment: newAttachment(),
profile: true,
});
const summary = await run();
await expectContextCleared(outsideWindow);
const cleared = await repository.getReport(outsideWindow.reportId);
expect(cleared).toMatchObject({status: 1, category: 'other', reporterOutcome: 'no_violation'});
expect((await search.searchReports('friends', {}, {limit: 10})).hits).toHaveLength(0);
await expectKept(insideWindow);
expect((await repository.getReport(insideWindow.reportId))?.additionalInfo).toBe('Still open to a complaint');
await expectKept(actioned);
expect(summary).toMatchObject({
scanned: 3,
expired: 0,
contextExpired: 1,
contextCleared: 1,
deleted: 0,
objectsDeleted: 2,
failed: 0,
});
const again = await run();
expect(again).toMatchObject({contextExpired: 0, contextCleared: 0, objectsDeleted: 0});
});
it('deletes a no violation report one year after it was resolved', async () => {
const expired = await seed({resolvedDaysAgo: 365, outcome: 'no_violation', profile: true});
const summary = await run();
await expectGone(expired);
expect(summary).toMatchObject({expired: 1, deleted: 1, contextExpired: 0});
});
it('keeps an evidence copy that an actioned report shares with a no violation report', async () => {
const shared = newAttachment();
const noViolation = await seed({resolvedDaysAgo: 200, outcome: 'no_violation', attachment: shared});
const actioned = await seed({resolvedDaysAgo: 200, attachment: shared});
const summary = await run();
expect((await repository.getReport(noViolation.reportId))?.messageContext).toBeNull();
await expectKept(actioned);
expect(summary).toMatchObject({contextCleared: 1, sharedObjectsKept: 1, objectsDeleted: 0});
});
it('keeps a child safety report and its context for a year even with a shorter period or no violation', async () => {
const childSafety = await seed({
resolvedDaysAgo: 364,
category: 'child_safety',
outcome: 'no_violation',
attachment: newAttachment(),
profile: true,
additionalInfo: 'Context for a CyberTip',
});
const childSafetyAfterYear = await seed({resolvedDaysAgo: 365, category: 'child_safety', profile: true});
const ordinary = await seed({resolvedDaysAgo: 40, profile: true});
const summary = await run({policy: {retentionDays: 30}});
await expectKept(childSafety);
expect((await repository.getReport(childSafety.reportId))?.additionalInfo).toBe('Context for a CyberTip');
await expectGone(childSafetyAfterYear);
await expectGone(ordinary);
expect(summary).toMatchObject({expired: 2, contextExpired: 0, deleted: 2});
});
it('keeps a child safety report under a legal hold past the year', async () => {
const held = await seed({
resolvedDaysAgo: 500,
category: 'child_safety',
profile: true,
legalHoldUntil: new Date(NOW.getTime() + 30 * DAY_MS),
});
const summary = await run();
await expectKept(held);
expect(summary).toMatchObject({expired: 0, held: 1, deleted: 0});
});
it('keeps a report linked to a decision until the decision evidence expires', async () => {
const linked = await seed({
resolvedDaysAgo: 400,
outcome: 'no_violation',
profile: true,
attachment: newAttachment(),
});
const linkedLongAgo = await seed({resolvedDaysAgo: 800, profile: true});
const unlinked = await seed({resolvedDaysAgo: 400, profile: true});
await linkDecision([linked.reportId], daysAgo(500));
await linkDecision([linked.reportId], daysAgo(100));
await linkDecision([linkedLongAgo.reportId], daysAgo(700));
const summary = await run({pageSize: 1});
await expectKept(linked);
expect((await repository.getReport(linked.reportId))?.messageContext).toHaveLength(1);
await expectGone(linkedLongAgo);
await expectGone(unlinked);
expect(summary).toMatchObject({scanned: 3, expired: 2, contextExpired: 0, deleted: 2});
});
it('deletes nothing when the decision scan cannot finish', async () => {
const expired = await seed({resolvedDaysAgo: 400, profile: true});
await expect(
processReportRetention(
{
...deps,
decisionRepository: {
listReportLinks: async () => {
throw new Error('read timeout');
},
},
},
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
),
).rejects.toThrow('read timeout');
await expectKept(expired);
});
it('keeps an object that a retained report still uses', async () => {
const shared = newAttachment();
const expired = await seed({ageDays: 500, attachment: shared});
const retained = await seed({ageDays: 10, attachment: shared});
const expired = await seed({resolvedDaysAgo: 500, attachment: shared});
const retained = await seed({resolvedDaysAgo: 10, attachment: shared});
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
const summary = await run();
await expectGone(expired);
await expectKept(retained);
@@ -246,10 +521,10 @@ describe('report retention', () => {
it('deletes an object shared by two expired reports once', async () => {
const shared = newAttachment();
const first = await seed({ageDays: 500, attachment: shared});
const second = await seed({ageDays: 450, attachment: shared});
const first = await seed({resolvedDaysAgo: 500, attachment: shared});
const second = await seed({resolvedDaysAgo: 450, attachment: shared});
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
const summary = await run();
await expectGone(first);
await expectGone(second);
@@ -261,15 +536,15 @@ describe('report retention', () => {
it('keeps a report under a legal hold that ends later and deletes one whose hold has ended', async () => {
const shared = newAttachment();
const held = await seed({
ageDays: 500,
resolvedDaysAgo: 500,
attachment: shared,
profile: true,
legalHoldUntil: new Date(NOW.getTime() + DAY_MS),
});
const holdEnded = await seed({ageDays: 500, profile: true, legalHoldUntil: daysAgo(1)});
const sharingWithHeld = await seed({ageDays: 500, attachment: shared});
const holdEnded = await seed({resolvedDaysAgo: 500, profile: true, legalHoldUntil: daysAgo(1)});
const sharingWithHeld = await seed({resolvedDaysAgo: 500, attachment: shared});
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
const summary = await run();
await expectKept(held);
expect((await repository.getReport(held.reportId))?.legalHoldReason).toBe('Preservation request');
@@ -279,31 +554,61 @@ describe('report retention', () => {
expect(summary).toMatchObject({scanned: 3, expired: 2, held: 1, deleted: 2, sharedObjectsKept: 1});
});
it('changes nothing in a dry run and reports what it would delete', async () => {
const expired = await seed({ageDays: 366, attachment: newAttachment(), profile: true});
const kept = await seed({ageDays: 1, profile: true});
it('keeps the context of a no violation report under a legal hold', async () => {
const held = await seed({
resolvedDaysAgo: 250,
outcome: 'no_violation',
attachment: newAttachment(),
legalHoldUntil: new Date(NOW.getTime() + DAY_MS),
});
const summary = await processReportRetention(deps, {now: NOW, dryRun: true, policy: DEFAULT_POLICY});
const summary = await run();
await expectKept(expired);
await expectKept(kept);
await expectKept(held);
expect((await repository.getReport(held.reportId))?.messageContext).toHaveLength(1);
expect(summary).toMatchObject({held: 1, contextExpired: 0, contextCleared: 0});
});
it('changes nothing in a dry run and counts what it would do to existing reports', async () => {
const expired = await seed({resolvedDaysAgo: 366, attachment: newAttachment(), profile: true});
const contextDue = await seed({
resolvedDaysAgo: 200,
outcome: 'no_violation',
attachment: newAttachment(),
additionalInfo: 'details',
});
const childSafety = await seed({resolvedDaysAgo: 200, category: 'child_safety', outcome: 'no_violation'});
const held = await seed({resolvedDaysAgo: 400, legalHoldUntil: new Date(NOW.getTime() + DAY_MS)});
const pending = await seed({profile: true});
const linked = await seed({resolvedDaysAgo: 400});
await linkDecision([linked.reportId], daysAgo(30));
const summary = await run({dryRun: true});
for (const report of [expired, contextDue, childSafety, held, pending, linked]) {
await expectKept(report);
}
expect((await repository.getReport(contextDue.reportId))?.additionalInfo).toBe('details');
expect(storage.deleteObjectSpy).not.toHaveBeenCalled();
expect(summary).toEqual({
dryRun: true,
scanned: 2,
scanned: 6,
pending: 1,
expired: 1,
held: 0,
contextExpired: 1,
held: 1,
deleted: 1,
objectsDeleted: 2,
contextCleared: 1,
objectsDeleted: 3,
sharedObjectsKept: 0,
failed: 0,
});
});
it('deletes nothing when the scan cannot finish', async () => {
const expired = await seed({ageDays: 400, attachment: newAttachment(), profile: true});
await seed({ageDays: 400});
await seed({ageDays: 400});
const expired = await seed({resolvedDaysAgo: 400, attachment: newAttachment(), profile: true});
await seed({resolvedDaysAgo: 400});
await seed({resolvedDaysAgo: 400});
let pages = 0;
const failingRepository = Object.create(repository) as ReportRepository;
failingRepository.listAllReportsPaginated = async (limit: number, lastReportId?: ReportID) => {
@@ -315,7 +620,12 @@ describe('report retention', () => {
await expect(
processReportRetention(
{...deps, reportRepository: failingRepository},
{now: NOW, dryRun: false, policy: DEFAULT_POLICY, pageSize: 2},
{
now: NOW,
dryRun: false,
policy: DEFAULT_POLICY,
pageSize: 2,
},
),
).rejects.toThrow('read timeout');
@@ -328,11 +638,11 @@ describe('report retention', () => {
const expired: Array<SeededReport> = [];
const kept: Array<SeededReport> = [];
for (let index = 0; index < 7; index++) {
expired.push(await seed({ageDays: 400 + index, profile: true}));
kept.push(await seed({ageDays: index, profile: true}));
expired.push(await seed({resolvedDaysAgo: 400 + index, profile: true}));
kept.push(await seed({resolvedDaysAgo: index, profile: true}));
}
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY, pageSize: 3});
const summary = await run({pageSize: 3});
expect(summary).toMatchObject({scanned: 14, expired: 7, deleted: 7, objectsDeleted: 7, failed: 0});
for (const report of expired) await expectGone(report);
@@ -340,25 +650,57 @@ describe('report retention', () => {
});
it('keeps the row and search document when an object cannot be deleted, then deletes them on the next run', async () => {
const expired = await seed({ageDays: 400, attachment: newAttachment(), profile: true});
const expired = await seed({resolvedDaysAgo: 400, attachment: newAttachment(), profile: true});
storage.configure({shouldFailDelete: true});
const failed = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
const failed = await run();
expect(failed).toMatchObject({expired: 1, deleted: 0, failed: 1});
expect(await repository.getReport(expired.reportId)).not.toBeNull();
expect(await indexedIds()).toContain(expired.reportId.toString());
storage.configure({shouldFailDelete: false});
const retried = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
const retried = await run();
expect(retried).toMatchObject({expired: 1, deleted: 1, failed: 0});
await expectGone(expired);
expect(storage.hasObject(REPORTS_BUCKET, expired.attachmentKey!)).toBe(false);
});
it('keeps the copied context when an object cannot be deleted, then clears it on the next run', async () => {
const due = await seed({resolvedDaysAgo: 200, outcome: 'no_violation', attachment: newAttachment()});
storage.configure({shouldFailDelete: true});
expect(await run()).toMatchObject({contextExpired: 1, contextCleared: 0, failed: 1});
expect((await repository.getReport(due.reportId))?.messageContext).toHaveLength(1);
storage.configure({shouldFailDelete: false});
expect(await run()).toMatchObject({contextExpired: 1, contextCleared: 1, failed: 0});
await expectContextCleared(due);
});
it('does not bring back a report deleted while its context was being cleared', async () => {
const due = await seed({resolvedDaysAgo: 200, outcome: 'no_violation', additionalInfo: 'details'});
const deletingRepository = Object.create(repository) as ReportRepository;
deletingRepository.clearCopiedContext = async (reportId, reportType) => {
await repository.deleteReport(reportId);
return repository.clearCopiedContext(reportId, reportType);
};
await processReportRetention(
{...deps, reportRepository: deletingRepository},
{
now: NOW,
dryRun: false,
policy: DEFAULT_POLICY,
},
);
expect(await repository.getReport(due.reportId)).toBeNull();
});
it('keeps the row when the search document cannot be deleted', async () => {
const expired = await seed({ageDays: 400});
const expired = await seed({resolvedDaysAgo: 400});
const failingSearch = Object.create(search) as IReportSearchService;
failingSearch.deleteReport = async () => {
throw new Error('search unavailable');
@@ -366,7 +708,11 @@ describe('report retention', () => {
const summary = await processReportRetention(
{...deps, reportSearchService: failingSearch},
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
{
now: NOW,
dryRun: false,
policy: DEFAULT_POLICY,
},
);
expect(summary).toMatchObject({deleted: 0, failed: 1});
@@ -374,11 +720,15 @@ describe('report retention', () => {
});
it('deletes rows when search is not configured', async () => {
const expired = await seed({ageDays: 400, profile: true});
const expired = await seed({resolvedDaysAgo: 400, profile: true});
const summary = await processReportRetention(
{...deps, reportSearchService: null},
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
{
now: NOW,
dryRun: false,
policy: DEFAULT_POLICY,
},
);
expect(summary).toMatchObject({deleted: 1, failed: 0});
@@ -389,8 +739,8 @@ describe('report retention', () => {
it('skips a report that was put on hold after the scan', async () => {
const shared = newAttachment();
const seeded = [
await seed({ageDays: 400, attachment: shared, profile: true}),
await seed({ageDays: 400, attachment: shared}),
await seed({resolvedDaysAgo: 400, attachment: shared, profile: true}),
await seed({resolvedDaysAgo: 400, attachment: shared}),
];
const scanOrder = (await repository.listAllReportsPaginated(10)).map((report) => report.reportId);
const [first, second] = seeded.sort((a, b) => scanOrder.indexOf(a.reportId) - scanOrder.indexOf(b.reportId)) as [
@@ -408,7 +758,11 @@ describe('report retention', () => {
const summary = await processReportRetention(
{...deps, reportRepository: holdingRepository},
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
{
now: NOW,
dryRun: false,
policy: DEFAULT_POLICY,
},
);
await expectKept(first);
@@ -419,7 +773,7 @@ describe('report retention', () => {
it('keeps an object that a report filed during the run stored again', async () => {
const shared = newAttachment();
const expired = await seed({ageDays: 400, attachment: shared, profile: true});
const expired = await seed({resolvedDaysAgo: 400, attachment: shared, profile: true});
const rewritingStorage = Object.create(storage) as MockStorageService;
rewritingStorage.getObjectMetadata = async (bucket: string, key: string) => {
const metadata = await storage.getObjectMetadata(bucket, key);
@@ -431,7 +785,11 @@ describe('report retention', () => {
const summary = await processReportRetention(
{...deps, storageService: rewritingStorage},
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
{
now: NOW,
dryRun: false,
policy: DEFAULT_POLICY,
},
);
await expectGone(expired);
@@ -440,76 +798,25 @@ describe('report retention', () => {
});
it('deletes profile images stored for a report whose snapshot cannot be read', async () => {
const expired = await seed({ageDays: 400, profile: true, snapshotJson: '{"not":"a snapshot"'});
const expired = await seed({resolvedDaysAgo: 400, profile: true, snapshotJson: '{"not":"a snapshot"'});
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
const summary = await run();
expect(summary).toMatchObject({deleted: 1, objectsDeleted: 1});
await expectGone(expired);
});
function run(options: Partial<ReportRetentionOptions> = {}) {
return processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY, ...options});
}
it('deletes earlier when the retention period is shorter', async () => {
const expired = await seed({ageDays: 30, profile: true});
const kept = await seed({ageDays: 29, profile: true});
const expired = await seed({resolvedDaysAgo: 30, profile: true});
const kept = await seed({resolvedDaysAgo: 29, profile: true});
const summary = await run({policy: {retentionDays: 30, resolvedRetentionDays: null}});
const summary = await run({policy: {retentionDays: 30}});
await expectGone(expired);
await expectKept(kept);
expect(summary).toMatchObject({scanned: 2, expired: 1, deleted: 1});
});
it('deletes a resolved report once the resolved-report period has passed since it was resolved', async () => {
const policy = {retentionDays: 365, resolvedRetentionDays: 30};
const resolvedLongAgo = await seed({ageDays: 100, resolvedDaysAgo: 30, attachment: newAttachment()});
const resolvedRecently = await seed({ageDays: 100, resolvedDaysAgo: 29, profile: true});
const pending = await seed({ageDays: 100, profile: true});
const resolvedWithoutTime = await seed({ageDays: 100, status: 1, profile: true});
const summary = await run({policy});
await expectGone(resolvedLongAgo);
expect(storage.hasObject(REPORTS_BUCKET, resolvedLongAgo.attachmentKey!)).toBe(false);
await expectKept(resolvedRecently);
await expectKept(pending);
await expectKept(resolvedWithoutTime);
expect(summary).toMatchObject({scanned: 4, expired: 1, deleted: 1, held: 0});
});
it('applies whichever rule ends first', async () => {
const policy = {retentionDays: 100, resolvedRetentionDays: 300};
const byAge = await seed({ageDays: 100, resolvedDaysAgo: 10});
const neither = await seed({ageDays: 99, resolvedDaysAgo: 90});
await run({policy});
await expectGone(byAge);
await expectKept(neither);
});
it('keeps a held resolved report past the resolved-report period and keeps its evidence', async () => {
const shared = newAttachment();
const held = await seed({
ageDays: 50,
resolvedDaysAgo: 40,
attachment: shared,
profile: true,
legalHoldUntil: new Date(NOW.getTime() + DAY_MS),
});
const sharingWithHeld = await seed({ageDays: 50, resolvedDaysAgo: 40, attachment: shared});
const summary = await run({policy: {retentionDays: 365, resolvedRetentionDays: 7}});
await expectKept(held);
await expectGone(sharingWithHeld);
expect(storage.hasObject(REPORTS_BUCKET, attachmentKey(shared))).toBe(true);
expect(summary).toMatchObject({expired: 1, held: 1, deleted: 1, sharedObjectsKept: 1});
});
it('frees the reporter to report the same account again once the report is deleted', async () => {
const reporterId = nextId();
const reportedUserId = nextId();
@@ -519,8 +826,9 @@ describe('report retention', () => {
report_id: BigInt(reportId),
reporter_id: reporterId,
reported_user_id: reportedUserId,
reported_at: daysAgo(400),
status: 0,
reported_at: daysAgo(420),
status: 1,
resolved_at: daysAgo(400),
report_type: 1,
category: 'harassment',
} as IARSubmissionRow);
@@ -528,7 +836,7 @@ describe('report retention', () => {
reporter_id: createUserID(reporterId),
reported_user_id: createUserID(reportedUserId),
report_id: reportId,
reported_at: daysAgo(400),
reported_at: daysAgo(420),
};
expect(await repository.reserveUserReportByReporter(reservation)).toBe(true);
@@ -559,9 +867,10 @@ describe('report retention', () => {
}
it('deletes for real with the default settings', async () => {
expect(configured).toEqual({days: 365, resolvedDays: null, dryRun: false});
const expired = await seed({ageDays: 366, attachment: newAttachment(), profile: true});
const kept = await seed({ageDays: 364, profile: true});
expect(configured).toEqual({days: 365, dryRun: false});
const expired = await seed({resolvedDaysAgo: 366, attachment: newAttachment(), profile: true});
const kept = await seed({resolvedDaysAgo: 364, profile: true});
const pending = await seed({profile: true});
await runDailyTask();
@@ -569,11 +878,12 @@ describe('report retention', () => {
expect(storage.hasObject(REPORTS_BUCKET, expired.attachmentKey!)).toBe(false);
expect(storage.hasObject(REPORTS_BUCKET, expired.profileKey!)).toBe(false);
expect(await repository.getReport(kept.reportId)).not.toBeNull();
expect(await repository.getReport(pending.reportId)).not.toBeNull();
});
it('only counts what it would delete when FLUXER_REPORT_RETENTION_DRY_RUN is true', async () => {
Object.assign(Config.reportRetention, {dryRun: true});
const expired = await seed({ageDays: 366, attachment: newAttachment(), profile: true});
const expired = await seed({resolvedDaysAgo: 366, attachment: newAttachment(), profile: true});
await runDailyTask();
@@ -581,17 +891,18 @@ describe('report retention', () => {
expect(storage.deleteObjectSpy).not.toHaveBeenCalled();
});
it('reads both day settings from the config', async () => {
Object.assign(Config.reportRetention, {days: 60, resolvedDays: 5});
const byAge = await seed({ageDays: 60});
const byResolution = await seed({ageDays: 20, resolvedDaysAgo: 5});
const kept = await seed({ageDays: 20, resolvedDaysAgo: 4});
it('reads the day setting from the config and honors decisions', async () => {
Object.assign(Config.reportRetention, {days: 60});
const expired = await seed({resolvedDaysAgo: 60});
const kept = await seed({resolvedDaysAgo: 59});
const linked = await seed({resolvedDaysAgo: 90});
await linkDecision([linked.reportId], daysAgo(90));
await runDailyTask();
expect(await repository.getReport(byAge.reportId)).toBeNull();
expect(await repository.getReport(byResolution.reportId)).toBeNull();
expect(await repository.getReport(expired.reportId)).toBeNull();
expect(await repository.getReport(kept.reportId)).not.toBeNull();
expect(await repository.getReport(linked.reportId)).not.toBeNull();
});
});
});
@@ -521,14 +521,22 @@ A report resolution is the reduced view [Update report](#update-report) returns.
## Report retention
Fluxer deletes a report when its retention period ends, whether it is pending or resolved, together with its stored evidence: the attachment copies, the profile snapshot images, and its search index entry. The reporter's duplicate-report reservation for it goes too, so the reporter can report the same message, account or guild again.
A pending report is kept until it is resolved. Once a report is resolved, Fluxer deletes it when its retention period ends, together with its stored evidence: the captured message context, the profile snapshot and its images, the attachment copies, and its search index entry. The reporter's duplicate-report reservation for it goes too, so the reporter can report the same message, account or guild again.
The retention period is 365 days from when the report was filed, or the number of days in `FLUXER_REPORT_RETENTION_DAYS`. With `FLUXER_RESOLVED_REPORT_RETENTION_DAYS` set, a resolved report is deleted that many days after `resolved_at` when that comes first. A report with a [legal hold](#set-report-legal-hold) that ends later is kept until the next run after the hold ends. An evidence copy that a retained report also uses is kept for that report.
The retention period ends 365 days after `resolved_at`, or the number of days in `FLUXER_REPORT_RETENTION_DAYS`. Some reports are kept longer, whatever that setting says:
The deletion runs once a day in `worker`. Each run logs one `Processed report retention` line with the counts it found and deleted and the two periods it used. A report that fails to delete keeps its record and is retried on the next run. [Delete report](#delete-report) deletes one report straight away by the same rules.
- A report in the `child_safety` category is kept for at least 365 days after it was resolved.
- A report listed in the `report_ids` of an [enforcement decision](/admin-api/enforcement/#enforcement-decision-object) is kept for at least 365 days after the newest such decision was recorded, as long as the decision's evidence.
- A report with a [legal hold](#set-report-legal-hold) that ends later is kept until the next run after the hold ends.
A report resolved with the outcome `no_violation` loses its copied context 184 days after it was resolved, which covers the six months the Digital Services Act gives a reporter to contest the outcome. Fluxer then removes the captured message context, the profile snapshot, the reporter's additional information, and the stored evidence copies, and keeps the rest of the record and its search index entry until the end of the retention period. This does not apply to a `child_safety` report, to a report linked to an enforcement decision, or while a legal hold has not ended.
An evidence copy that a retained report also uses is kept for that report.
The job runs once a day in `worker`. Each run logs one `Processed report retention` line with the counts it found, deleted and cleared and the periods it used. A report that fails to delete or clear keeps its record and is retried on the next run. [Delete report](#delete-report) deletes one report straight away.
:::caution[Deleted evidence cannot be restored]
Retention deletes for real unless `FLUXER_REPORT_RETENTION_DRY_RUN` is `true` on `worker`. In a dry run each run only counts what it would delete and deletes nothing. The [operator configuration](/operator/configuration/#moderation-and-abuse) describes the three settings.
Retention deletes for real unless `FLUXER_REPORT_RETENTION_DRY_RUN` is `true` on `worker`. In a dry run each run only counts what it would delete or clear and changes nothing. The [operator configuration](/operator/configuration/#moderation-and-abuse) describes the two settings.
:::
## List reports
@@ -768,7 +776,7 @@ Fluxer records one [Admin audit entry](/admin-api/#admin-audit-entry-object) wit
Deletes one report straight away. Requires `report:delete`, which `report:resolve` does not include.
This is the deletion [report retention](#report-retention) runs, without waiting for the retention period. It removes the report record with its captured message context and profile snapshot, its stored evidence copies, its search index entry, and the reporter's duplicate-report reservation for it. An evidence copy that another report also uses is kept for that report. A pending or a resolved report can be deleted, and the reporter is not told.
This is the deletion [report retention](#report-retention) runs at the end of the retention period, without waiting for it. It removes the report record with its captured message context and profile snapshot, its stored evidence copies, its search index entry, and the reporter's duplicate-report reservation for it. An evidence copy that another report also uses is kept for that report. A pending or a resolved report can be deleted, and the reporter is not told.
:::caution[Deletion is permanent]
Deleted evidence cannot be restored. A report with a legal hold that has not ended is refused. Clear the hold first when the report should go.
@@ -20,7 +20,7 @@ A safety report tells moderators that a message or a user profile broke the rule
Fluxer discards a body member that no schema here defines, so a misspelt member is dropped without an error. A body that is not parseable JSON returns 400 `INVALID_FORM_BODY` with the validation code `INVALID_FORMAT` at the `body` path.
:::caution[A submitted report is durable evidence]
Fluxer copies the reported content when the report is submitted. A message report also copies the surrounding conversation and the bytes of every attachment in it, and every report copies the reported profile with its images. Deleting the message, editing the profile, or leaving the guild afterwards does not remove that snapshot. The snapshot is kept until the report reaches the end of its [retention period](/admin-api/reports/#report-retention).
Fluxer copies the reported content when the report is submitted. A message report also copies the surrounding conversation and the bytes of every attachment in it, and every report copies the reported profile with its images. Deleting the message, editing the profile, or leaving the guild afterwards does not remove that snapshot. The snapshot is kept until [report retention](/admin-api/reports/#report-retention) removes it, which happens no sooner than 184 days after the report is resolved.
:::
## Access rules
@@ -1063,15 +1063,11 @@ With feeds off, Fluxer checks none of this data. The first worker start with fee
#### `FLUXER_REPORT_RETENTION_DAYS`
Default `365`. Read by `worker`. The daily [report retention](/admin-api/reports/#report-retention) job deletes each report this many days after it was filed, pending or resolved, with its evidence copies and search entry, unless a legal hold is in place. Integer 1 to 36500, checked at startup.
#### `FLUXER_RESOLVED_REPORT_RETENTION_DAYS`
Default unset, which leaves resolved reports to `FLUXER_REPORT_RETENTION_DAYS`. Read by `worker`. When set, a resolved report is deleted this many days after it was resolved, if that comes before the end of its retention period. Pending reports are not affected, and a legal hold still keeps a report. Integer 1 to 36500, checked at startup.
Default `365`. Read by `worker`. The daily [report retention](/admin-api/reports/#report-retention) job deletes each resolved report this many days after it was resolved, with its evidence copies and search entry. Pending reports are kept until they are resolved. A legal hold, a `child_safety` category or a linked enforcement decision can keep a report longer. Integer 1 to 36500, checked at startup.
#### `FLUXER_REPORT_RETENTION_DRY_RUN`
Default `false`. Read by `worker`. With `true`, each run only counts what it would delete and deletes nothing. `true` or `false`, checked at startup. Deleted evidence cannot be restored. Check the counts in the `Processed report retention` log line before the first real run.
Default `false`. Read by `worker`. With `true`, each run only counts what it would delete or clear and changes nothing. `true` or `false`, checked at startup. Deleted evidence cannot be restored. Check the counts in the `Processed report retention` log line before the first real run.
#### `FLUXER_BREACHED_PASSWORD_CHECK_ENABLED`
-9
View File
@@ -280,7 +280,6 @@ function defaultConfig(): MasterConfig {
inactivity_deletion_threshold_days: 365,
report_retention: {
days: 365,
resolved_days: null,
dry_run: false,
},
};
@@ -451,14 +450,6 @@ function validateApiWorkerConfig(config: MasterConfig): void {
function validateReportRetentionConfig(config: MasterConfig): void {
const retention = config.report_retention;
assertIntegerInRange(retention.days, 'FLUXER_REPORT_RETENTION_DAYS', 1, MAX_REPORT_RETENTION_DAYS);
if (retention.resolved_days !== null) {
assertIntegerInRange(
retention.resolved_days,
'FLUXER_RESOLVED_REPORT_RETENTION_DAYS',
1,
MAX_REPORT_RETENTION_DAYS,
);
}
assertBoolean(retention.dry_run, 'FLUXER_REPORT_RETENTION_DRY_RUN');
}
-1
View File
@@ -334,7 +334,6 @@ export interface MasterConfig {
inactivity_deletion_threshold_days: number;
report_retention: {
days: number;
resolved_days: number | null;
dry_run: boolean;
};
}
@@ -781,42 +781,22 @@ describe('report retention settings', () => {
vi.unstubAllEnvs();
});
test('deletes reports for real after 365 days with no resolved-report rule by default', async () => {
test('deletes resolved reports for real after 365 days by default', async () => {
stubMinimalEnv();
const config = await loadConfig();
expect(config.report_retention).toEqual({days: 365, resolved_days: null, dry_run: false});
expect(config.report_retention).toEqual({days: 365, dry_run: false});
});
test('reads both day settings and the dry run switch from the environment', async () => {
stubMinimalEnv({
FLUXER_REPORT_RETENTION_DAYS: '730',
FLUXER_RESOLVED_REPORT_RETENTION_DAYS: ' 90 ',
FLUXER_REPORT_RETENTION_DRY_RUN: 'TRUE',
});
test('reads the day setting and the dry run switch from the environment', async () => {
stubMinimalEnv({FLUXER_REPORT_RETENTION_DAYS: ' 730 ', FLUXER_REPORT_RETENTION_DRY_RUN: 'TRUE'});
const config = await loadConfig();
expect(config.report_retention).toEqual({days: 730, resolved_days: 90, dry_run: true});
});
test('an empty resolved-report setting leaves the rule off', async () => {
stubMinimalEnv({FLUXER_RESOLVED_REPORT_RETENTION_DAYS: '', FLUXER_REPORT_RETENTION_DRY_RUN: 'false'});
const config = await loadConfig();
expect(config.report_retention).toEqual({days: 365, resolved_days: null, dry_run: false});
expect(config.report_retention).toEqual({days: 730, dry_run: true});
});
test.each([
['FLUXER_REPORT_RETENTION_DAYS', '0', 'FLUXER_REPORT_RETENTION_DAYS must be an integer between 1 and 36500'],
['FLUXER_REPORT_RETENTION_DAYS', '36501', 'FLUXER_REPORT_RETENTION_DAYS must be an integer between 1 and 36500'],
['FLUXER_REPORT_RETENTION_DAYS', '1.5', 'FLUXER_REPORT_RETENTION_DAYS must be an integer, got "1.5"'],
[
'FLUXER_RESOLVED_REPORT_RETENTION_DAYS',
'0',
'FLUXER_RESOLVED_REPORT_RETENTION_DAYS must be an integer between 1 and 36500',
],
[
'FLUXER_RESOLVED_REPORT_RETENTION_DAYS',
'-30',
'FLUXER_RESOLVED_REPORT_RETENTION_DAYS must be an integer between 1 and 36500',
],
['FLUXER_REPORT_RETENTION_DRY_RUN', '1', 'FLUXER_REPORT_RETENTION_DRY_RUN must be true or false'],
])('refuses %s=%j', async (name, value, message) => {
stubMinimalEnv({[name]: value});
@@ -97,11 +97,10 @@ describe('the shipped compose stack forwards settings from .env', () => {
expect(renamed).toEqual([]);
});
test.each([
'FLUXER_REPORT_RETENTION_DAYS',
'FLUXER_RESOLVED_REPORT_RETENTION_DAYS',
'FLUXER_REPORT_RETENTION_DRY_RUN',
])('the worker is handed %s and gets no value when .env leaves it out', (name) => {
expect(serviceEnvironment('worker')[name]).toBe(`\${${name}:-}`);
});
test.each(['FLUXER_REPORT_RETENTION_DAYS', 'FLUXER_REPORT_RETENTION_DRY_RUN'])(
'the worker is handed %s and gets no value when .env leaves it out',
(name) => {
expect(serviceEnvironment('worker')[name]).toBe(`\${${name}:-}`);
},
);
});
@@ -301,7 +301,6 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
FLUXER_DISCOVERY_MIN_MEMBER_COUNT: {path: ['discovery', 'min_member_count'], parse: parseInteger},
FLUXER_DELETION_GRACE_PERIOD_HOURS: {path: ['deletion_grace_period_hours'], parse: parseInteger},
FLUXER_REPORT_RETENTION_DAYS: {path: ['report_retention', 'days'], parse: parseInteger},
FLUXER_RESOLVED_REPORT_RETENTION_DAYS: {path: ['report_retention', 'resolved_days'], parse: parseInteger},
FLUXER_REPORT_RETENTION_DRY_RUN: {path: ['report_retention', 'dry_run'], parse: parseBoolean},
FLUXER_RELAX_REGISTRATION_RATE_LIMITS: {path: ['dev', 'relax_registration_rate_limits'], parse: parseBoolean},
FLUXER_DISABLE_RATE_LIMITS: {path: ['dev', 'disable_rate_limits'], parse: parseBoolean},