mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-17 21:15:32 +09:00
feat(reports): keep reports until resolved, then by outcome (#3493)
This commit is contained in:
1 parent
f5c30fdf8f
commit
4f88872a3c
18 files changed
+673
-238
No files matched your search
@@ -449,7 +449,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
|
||||
inactivityDeletionThresholdDays: master.inactivity_deletion_threshold_days,
|
||||
reportRetention: {
|
||||
days: master.report_retention.days,
|
||||
resolvedDays: master.report_retention.resolved_days,
|
||||
dryRun: master.report_retention.dry_run,
|
||||
},
|
||||
push: {
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';
|
||||
import {createReportID, type ReportID} from '@app/api/BrandedTypes';
|
||||
import {IAR_SUBMISSION_COLUMNS, type IARSubmissionRow} from '@app/api/database/types/ReportTypes';
|
||||
import {EnforcementDecisionRepository} from '@app/api/enforcement/EnforcementDecisionRepository';
|
||||
import {ReportRepository} from '@app/api/report/ReportRepository';
|
||||
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {MockStorageService} from '@app/api/test/mocks/MockStorageService';
|
||||
@@ -47,6 +48,7 @@ describe('report legal hold', () => {
|
||||
...Object.fromEntries(IAR_SUBMISSION_COLUMNS.map((column) => [column, null])),
|
||||
report_id: sequence,
|
||||
reported_at: new Date(Date.now() - ageDays * ms('1 day')),
|
||||
resolved_at: new Date(Date.now() - ageDays * ms('1 day')),
|
||||
status: 1,
|
||||
report_type: 1,
|
||||
category: 'harassment',
|
||||
@@ -69,8 +71,13 @@ describe('report legal hold', () => {
|
||||
|
||||
async function runRetention() {
|
||||
return processReportRetention(
|
||||
{reportRepository: repository, storageService: new MockStorageService(), reportSearchService: null},
|
||||
{now: new Date(), dryRun: false, policy: {retentionDays: 365, resolvedRetentionDays: null}},
|
||||
{
|
||||
reportRepository: repository,
|
||||
storageService: new MockStorageService(),
|
||||
reportSearchService: null,
|
||||
decisionRepository: new EnforcementDecisionRepository(),
|
||||
},
|
||||
{now: new Date(), dryRun: false, policy: {retentionDays: 365}},
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -308,7 +308,6 @@ export interface APIConfig {
|
||||
deletionGracePeriodHours: number;
|
||||
reportRetention: {
|
||||
days: number;
|
||||
resolvedDays: number | null;
|
||||
dryRun: boolean;
|
||||
};
|
||||
inactivityDeletionThresholdDays?: number;
|
||||
|
||||
@@ -40,6 +40,16 @@ const LIST_NOTES_QUERY = EnforcementDecisionNotes.select({
|
||||
orderBy: {col: 'note_id', direction: 'ASC'},
|
||||
});
|
||||
|
||||
type DecisionReportLink = Pick<EnforcementDecisionRow, 'decision_id' | 'report_ids' | 'created_at'>;
|
||||
|
||||
function listReportLinksQuery(limit: number, paged: boolean) {
|
||||
return EnforcementDecisions.select({
|
||||
columns: ['decision_id', 'report_ids', 'created_at'],
|
||||
...(paged ? {where: EnforcementDecisions.where.tokenGt('decision_id', 'after')} : {}),
|
||||
limit,
|
||||
});
|
||||
}
|
||||
|
||||
const REVIEW_QUEUE_BUCKET = 0;
|
||||
const LIST_REVIEW_QUEUE_QUERY = EnforcementReviewQueue.select({
|
||||
columns: ['decision_id'],
|
||||
@@ -162,6 +172,12 @@ export class EnforcementDecisionRepository {
|
||||
return this.getMany(keys.map((key) => key.decision_id));
|
||||
}
|
||||
|
||||
async listReportLinks(limit: number, after?: bigint): Promise<Array<DecisionReportLink>> {
|
||||
return fetchMany<DecisionReportLink>(
|
||||
listReportLinksQuery(limit, after !== undefined).bind(after !== undefined ? {after} : {}),
|
||||
);
|
||||
}
|
||||
|
||||
async end(decisionId: bigint, end: EnforcementDecisionEnd): Promise<void> {
|
||||
await upsertOne(EnforcementDecisions.patchByPk({decision_id: decisionId}, endPatch(end)));
|
||||
}
|
||||
|
||||
@@ -79,13 +79,12 @@ async function storedObjectKeys(storageService: IStorageService, report: IARSubm
|
||||
return keys;
|
||||
}
|
||||
|
||||
export async function deleteReportWithEvidence(
|
||||
deps: ReportDeletionDeps,
|
||||
async function deleteReportEvidence(
|
||||
storageService: IStorageService,
|
||||
report: IARSubmission,
|
||||
ledger: ReportEvidenceLedger,
|
||||
options: {now: Date; dryRun: boolean},
|
||||
): Promise<ReportDeletionResult> {
|
||||
const {reportRepository, storageService, reportSearchService} = deps;
|
||||
const result: ReportDeletionResult = {objectsDeleted: 0, sharedObjectsKept: 0};
|
||||
for (const key of await storedObjectKeys(storageService, report)) {
|
||||
if (ledger.retainedKeys.has(key)) {
|
||||
@@ -105,6 +104,17 @@ export async function deleteReportWithEvidence(
|
||||
ledger.deletedKeys.add(key);
|
||||
result.objectsDeleted++;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function deleteReportWithEvidence(
|
||||
deps: ReportDeletionDeps,
|
||||
report: IARSubmission,
|
||||
ledger: ReportEvidenceLedger,
|
||||
options: {now: Date; dryRun: boolean},
|
||||
): Promise<ReportDeletionResult> {
|
||||
const {reportRepository, storageService, reportSearchService} = deps;
|
||||
const result = await deleteReportEvidence(storageService, report, ledger, options);
|
||||
if (!options.dryRun) {
|
||||
await reportSearchService?.deleteReport(report.reportId);
|
||||
await reportRepository.deleteReport(report.reportId);
|
||||
@@ -112,6 +122,25 @@ export async function deleteReportWithEvidence(
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function clearCopiedContextWithEvidence(
|
||||
deps: ReportDeletionDeps,
|
||||
report: IARSubmission,
|
||||
ledger: ReportEvidenceLedger,
|
||||
options: {now: Date; dryRun: boolean},
|
||||
): Promise<ReportDeletionResult> {
|
||||
const {reportRepository, storageService, reportSearchService} = deps;
|
||||
const result = await deleteReportEvidence(storageService, report, ledger, options);
|
||||
if (!options.dryRun && (await reportRepository.clearCopiedContext(report.reportId, report.reportType))) {
|
||||
await reportSearchService?.updateReport({
|
||||
...report,
|
||||
messageContext: null,
|
||||
reportedProfileSnapshot: null,
|
||||
additionalInfo: null,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function loadDeletableReport(
|
||||
reportRepository: ReportRepository,
|
||||
reportId: ReportID,
|
||||
|
||||
@@ -186,6 +186,20 @@ export class ReportRepository {
|
||||
return {...report, legalHoldUntil, legalHoldReason};
|
||||
}
|
||||
|
||||
async clearCopiedContext(reportId: ReportID, reportType: number): Promise<boolean> {
|
||||
return executeConditional(
|
||||
IARSubmissions.conditionalPatchByPk(
|
||||
{report_id: reportId},
|
||||
{
|
||||
message_context: Db.set(null),
|
||||
reported_profile_snapshot: Db.set(null),
|
||||
additional_info: Db.set(null),
|
||||
},
|
||||
{report_type: reportType},
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async clearReporterEmail(reportId: ReportID, reporterId: UserID): Promise<boolean> {
|
||||
return executeConditional(
|
||||
IARSubmissions.conditionalPatchByPk(
|
||||
|
||||
@@ -2,23 +2,36 @@
|
||||
|
||||
import type {ReportID} from '@app/api/BrandedTypes';
|
||||
import {Config} from '@app/api/Config';
|
||||
import {EnforcementDecisionRepository} from '@app/api/enforcement/EnforcementDecisionRepository';
|
||||
import {Logger} from '@app/api/Logger';
|
||||
import {
|
||||
clearCopiedContextWithEvidence,
|
||||
deleteReportWithEvidence,
|
||||
forEachStoredReport,
|
||||
isReportUnderLegalHold,
|
||||
type ReportDeletionDeps,
|
||||
type ReportEvidenceLedger,
|
||||
referencedObjectKeys,
|
||||
} from '@app/api/report/ReportDeletion';
|
||||
import {type IARSubmission, ReportStatus} from '@app/api/report/ReportModels';
|
||||
import {getReportSearchService} from '@app/api/SearchFactory';
|
||||
import {EnforcementDecisionEvidence} from '@app/api/Tables';
|
||||
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
|
||||
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
|
||||
import {ms} from 'itty-time';
|
||||
|
||||
export const REPORT_COMPLAINT_WINDOW_DAYS = 184;
|
||||
export const CHILD_SAFETY_REPORT_RETENTION_DAYS = 365;
|
||||
const CHILD_SAFETY_CATEGORY = 'child_safety';
|
||||
const DECISION_EVIDENCE_RETENTION_MS = EnforcementDecisionEvidence.defaultTtlSeconds! * 1000;
|
||||
const DEFAULT_SCAN_PAGE_SIZE = 500;
|
||||
|
||||
export interface ReportRetentionPolicy {
|
||||
retentionDays: number;
|
||||
resolvedRetentionDays: number | null;
|
||||
}
|
||||
|
||||
export interface ReportRetentionDeps extends ReportDeletionDeps {
|
||||
decisionRepository: Pick<EnforcementDecisionRepository, 'listReportLinks'>;
|
||||
}
|
||||
|
||||
export interface ReportRetentionOptions {
|
||||
@@ -31,65 +44,136 @@ export interface ReportRetentionOptions {
|
||||
export interface ReportRetentionSummary {
|
||||
dryRun: boolean;
|
||||
scanned: number;
|
||||
pending: number;
|
||||
expired: number;
|
||||
contextExpired: number;
|
||||
held: number;
|
||||
deleted: number;
|
||||
contextCleared: number;
|
||||
objectsDeleted: number;
|
||||
sharedObjectsKept: number;
|
||||
failed: number;
|
||||
}
|
||||
|
||||
type ReportRetentionState = 'expired' | 'held' | 'retained';
|
||||
|
||||
function reportExpiresAt(report: IARSubmission, policy: ReportRetentionPolicy): number {
|
||||
const reportedAt = report.reportedAt instanceof Date ? report.reportedAt.getTime() : Number.NaN;
|
||||
const byAge = reportedAt + policy.retentionDays * ms('1 day');
|
||||
if (policy.resolvedRetentionDays === null || report.status !== ReportStatus.RESOLVED) {
|
||||
return byAge;
|
||||
}
|
||||
const resolvedAt = report.resolvedAt instanceof Date ? report.resolvedAt.getTime() : Number.NaN;
|
||||
if (Number.isNaN(resolvedAt)) {
|
||||
return byAge;
|
||||
}
|
||||
return Math.min(byAge, resolvedAt + policy.resolvedRetentionDays * ms('1 day'));
|
||||
export interface ReportRetentionDates {
|
||||
deleteAt: number | null;
|
||||
contextExpiresAt: number | null;
|
||||
}
|
||||
|
||||
function reportRetentionState(report: IARSubmission, now: Date, policy: ReportRetentionPolicy): ReportRetentionState {
|
||||
const expiresAt = reportExpiresAt(report, policy);
|
||||
if (Number.isNaN(expiresAt) || expiresAt > now.getTime()) {
|
||||
type ReportRetentionState = 'expired' | 'context_expired' | 'held' | 'retained';
|
||||
|
||||
const KEEP_UNTIL_RESOLVED: ReportRetentionDates = {deleteAt: null, contextExpiresAt: null};
|
||||
|
||||
export function reportRetentionDates(
|
||||
report: IARSubmission,
|
||||
policy: ReportRetentionPolicy,
|
||||
decidedAt: Date | null,
|
||||
): ReportRetentionDates {
|
||||
if (report.status !== ReportStatus.RESOLVED) {
|
||||
return KEEP_UNTIL_RESOLVED;
|
||||
}
|
||||
const resolvedAt = (report.resolvedAt ?? report.reportedAt).getTime();
|
||||
if (Number.isNaN(resolvedAt)) {
|
||||
return KEEP_UNTIL_RESOLVED;
|
||||
}
|
||||
const childSafety = report.category === CHILD_SAFETY_CATEGORY;
|
||||
const ends = [resolvedAt + policy.retentionDays * ms('1 day')];
|
||||
if (childSafety) {
|
||||
ends.push(resolvedAt + CHILD_SAFETY_REPORT_RETENTION_DAYS * ms('1 day'));
|
||||
}
|
||||
if (decidedAt) {
|
||||
ends.push(decidedAt.getTime() + DECISION_EVIDENCE_RETENTION_MS);
|
||||
}
|
||||
const dropsContextEarly = report.reporterOutcome === 'no_violation' && !childSafety && !decidedAt;
|
||||
return {
|
||||
deleteAt: Math.max(...ends),
|
||||
contextExpiresAt: dropsContextEarly ? resolvedAt + REPORT_COMPLAINT_WINDOW_DAYS * ms('1 day') : null,
|
||||
};
|
||||
}
|
||||
|
||||
function hasCopiedContext(report: IARSubmission): boolean {
|
||||
return (
|
||||
(report.messageContext?.length ?? 0) > 0 ||
|
||||
report.reportedProfileSnapshot !== null ||
|
||||
report.additionalInfo !== null
|
||||
);
|
||||
}
|
||||
|
||||
function reportRetentionState(
|
||||
report: IARSubmission,
|
||||
now: Date,
|
||||
policy: ReportRetentionPolicy,
|
||||
decidedAt: Date | null,
|
||||
): ReportRetentionState {
|
||||
const {deleteAt, contextExpiresAt} = reportRetentionDates(report, policy, decidedAt);
|
||||
const due =
|
||||
deleteAt !== null && deleteAt <= now.getTime()
|
||||
? 'expired'
|
||||
: contextExpiresAt !== null && contextExpiresAt <= now.getTime() && hasCopiedContext(report)
|
||||
? 'context_expired'
|
||||
: null;
|
||||
if (due === null) {
|
||||
return 'retained';
|
||||
}
|
||||
if (isReportUnderLegalHold(report, now)) {
|
||||
return 'held';
|
||||
return isReportUnderLegalHold(report, now) ? 'held' : due;
|
||||
}
|
||||
|
||||
async function latestDecisionByReport(
|
||||
decisionRepository: ReportRetentionDeps['decisionRepository'],
|
||||
pageSize: number,
|
||||
): Promise<Map<bigint, Date>> {
|
||||
const decidedAt = new Map<bigint, Date>();
|
||||
let cursor: bigint | undefined;
|
||||
while (true) {
|
||||
const page = await decisionRepository.listReportLinks(pageSize, cursor);
|
||||
for (const decision of page) {
|
||||
for (const reportId of decision.report_ids ?? []) {
|
||||
const known = decidedAt.get(reportId);
|
||||
if (!known || known.getTime() < decision.created_at.getTime()) {
|
||||
decidedAt.set(reportId, decision.created_at);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (page.length < pageSize) return decidedAt;
|
||||
cursor = page[page.length - 1]!.decision_id;
|
||||
}
|
||||
return 'expired';
|
||||
}
|
||||
|
||||
export async function processReportRetention(
|
||||
deps: ReportDeletionDeps,
|
||||
deps: ReportRetentionDeps,
|
||||
options: ReportRetentionOptions,
|
||||
): Promise<ReportRetentionSummary> {
|
||||
const {reportRepository} = deps;
|
||||
const {now, dryRun, policy} = options;
|
||||
const pageSize = options.pageSize ?? DEFAULT_SCAN_PAGE_SIZE;
|
||||
const summary: ReportRetentionSummary = {
|
||||
dryRun,
|
||||
scanned: 0,
|
||||
pending: 0,
|
||||
expired: 0,
|
||||
contextExpired: 0,
|
||||
held: 0,
|
||||
deleted: 0,
|
||||
contextCleared: 0,
|
||||
objectsDeleted: 0,
|
||||
sharedObjectsKept: 0,
|
||||
failed: 0,
|
||||
};
|
||||
const decidedAt = await latestDecisionByReport(deps.decisionRepository, pageSize);
|
||||
const stateOf = (report: IARSubmission) =>
|
||||
reportRetentionState(report, now, policy, decidedAt.get(report.reportId) ?? null);
|
||||
const retainedKeys = new Set<string>();
|
||||
const expiredIds: Array<ReportID> = [];
|
||||
const due: Array<{reportId: ReportID; state: 'expired' | 'context_expired'}> = [];
|
||||
await forEachStoredReport(
|
||||
reportRepository,
|
||||
(report) => {
|
||||
summary.scanned++;
|
||||
const state = reportRetentionState(report, now, policy);
|
||||
if (state === 'expired') {
|
||||
expiredIds.push(report.reportId);
|
||||
if (report.status !== ReportStatus.RESOLVED) {
|
||||
summary.pending++;
|
||||
}
|
||||
const state = stateOf(report);
|
||||
if (state === 'expired' || state === 'context_expired') {
|
||||
due.push({reportId: report.reportId, state});
|
||||
return;
|
||||
}
|
||||
if (state === 'held') {
|
||||
@@ -99,35 +183,37 @@ export async function processReportRetention(
|
||||
retainedKeys.add(key);
|
||||
}
|
||||
},
|
||||
options.pageSize,
|
||||
pageSize,
|
||||
);
|
||||
summary.expired = expiredIds.length;
|
||||
const ledger = {retainedKeys, deletedKeys: new Set<string>()};
|
||||
for (const reportId of expiredIds) {
|
||||
summary.expired = due.filter((entry) => entry.state === 'expired').length;
|
||||
summary.contextExpired = due.length - summary.expired;
|
||||
const ledger: ReportEvidenceLedger = {retainedKeys, deletedKeys: new Set<string>()};
|
||||
for (const {reportId, state} of due) {
|
||||
try {
|
||||
const report = await reportRepository.getReport(reportId);
|
||||
if (!report) continue;
|
||||
if (reportRetentionState(report, now, policy) !== 'expired') {
|
||||
if (stateOf(report) !== state) {
|
||||
for (const key of referencedObjectKeys(report)) {
|
||||
retainedKeys.add(key);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const result = await deleteReportWithEvidence(deps, report, ledger, {now, dryRun});
|
||||
const apply = state === 'expired' ? deleteReportWithEvidence : clearCopiedContextWithEvidence;
|
||||
const result = await apply(deps, report, ledger, {now, dryRun});
|
||||
summary.objectsDeleted += result.objectsDeleted;
|
||||
summary.sharedObjectsKept += result.sharedObjectsKept;
|
||||
summary.deleted++;
|
||||
if (state === 'expired') {
|
||||
summary.deleted++;
|
||||
} else {
|
||||
summary.contextCleared++;
|
||||
}
|
||||
} catch (error) {
|
||||
summary.failed++;
|
||||
Logger.error({error, reportId: reportId.toString()}, 'Failed to delete an expired report');
|
||||
Logger.error({error, reportId: reportId.toString(), state}, 'Failed to apply report retention');
|
||||
}
|
||||
}
|
||||
Logger.info(
|
||||
{
|
||||
...summary,
|
||||
retentionDays: policy.retentionDays,
|
||||
resolvedRetentionDays: policy.resolvedRetentionDays,
|
||||
},
|
||||
{...summary, retentionDays: policy.retentionDays, complaintWindowDays: REPORT_COMPLAINT_WINDOW_DAYS},
|
||||
'Processed report retention',
|
||||
);
|
||||
return summary;
|
||||
@@ -135,10 +221,15 @@ export async function processReportRetention(
|
||||
|
||||
const expireReportSnapshots: WorkerTaskHandler = async () => {
|
||||
const {reportRepository, storageService} = getWorkerDependencies();
|
||||
const {days, resolvedDays, dryRun} = Config.reportRetention;
|
||||
const {days, dryRun} = Config.reportRetention;
|
||||
await processReportRetention(
|
||||
{reportRepository, storageService, reportSearchService: getReportSearchService()},
|
||||
{now: new Date(), dryRun, policy: {retentionDays: days, resolvedRetentionDays: resolvedDays}},
|
||||
{
|
||||
reportRepository,
|
||||
storageService,
|
||||
reportSearchService: getReportSearchService(),
|
||||
decisionRepository: new EnforcementDecisionRepository(),
|
||||
},
|
||||
{now: new Date(), dryRun, policy: {retentionDays: days}},
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
@@ -3,13 +3,14 @@
|
||||
import {createAttachmentID, createChannelID, createReportID, createUserID, type ReportID} from '@app/api/BrandedTypes';
|
||||
import {Config} from '@app/api/Config';
|
||||
import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';
|
||||
import {ENFORCEMENT_DECISION_COLUMNS, type EnforcementDecisionRow} from '@app/api/database/types/EnforcementTypes';
|
||||
import type {MessageAttachment} from '@app/api/database/types/MessageTypes';
|
||||
import {
|
||||
IAR_SUBMISSION_COLUMNS,
|
||||
type IARMessageContextRow,
|
||||
type IARSubmissionRow,
|
||||
} from '@app/api/database/types/ReportTypes';
|
||||
import type {ReportDeletionDeps} from '@app/api/report/ReportDeletion';
|
||||
import {EnforcementDecisionRepository} from '@app/api/enforcement/EnforcementDecisionRepository';
|
||||
import type {IARSubmission} from '@app/api/report/ReportModels';
|
||||
import {ReportRepository} from '@app/api/report/ReportRepository';
|
||||
import type {IReportSearchService} from '@app/api/search/IReportSearchService';
|
||||
@@ -17,11 +18,17 @@ import {MockStorageService} from '@app/api/test/mocks/MockStorageService';
|
||||
import {NoopLogger} from '@app/api/test/mocks/NoopLogger';
|
||||
import {InMemorySearchProvider} from '@app/api/test/search/InMemorySearchProvider';
|
||||
import expireReportSnapshots, {
|
||||
CHILD_SAFETY_REPORT_RETENTION_DAYS,
|
||||
processReportRetention,
|
||||
REPORT_COMPLAINT_WINDOW_DAYS,
|
||||
type ReportRetentionDeps,
|
||||
type ReportRetentionOptions,
|
||||
type ReportRetentionPolicy,
|
||||
reportRetentionDates,
|
||||
} from '@app/api/worker/tasks/ExpireReportSnapshots';
|
||||
import {clearWorkerDependencies, setWorkerDependenciesForTest} from '@app/api/worker/WorkerContext';
|
||||
import {DeletionReasons} from '@fluxer/constants/src/Core';
|
||||
import type {ReportItemOutcome} from '@fluxer/schema/src/domains/admin/AdminSchemas';
|
||||
import {serializeReportProfileSnapshot} from '@fluxer/schema/src/domains/report/ReportProfileSnapshotSchemas';
|
||||
import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
|
||||
import {ms} from 'itty-time';
|
||||
@@ -31,7 +38,7 @@ const NOW = new Date('2027-06-01T12:00:00.000Z');
|
||||
const DAY_MS = ms('1 day');
|
||||
const REPORTS_BUCKET = Config.s3.buckets.reports;
|
||||
const WORKER_HELPERS = {logger: new NoopLogger()} as unknown as WorkerTaskHelpers;
|
||||
const DEFAULT_POLICY: ReportRetentionPolicy = {retentionDays: 365, resolvedRetentionDays: null};
|
||||
const DEFAULT_POLICY: ReportRetentionPolicy = {retentionDays: 365};
|
||||
let sequence = 1_480_000_000_000_000_000n;
|
||||
|
||||
function nextId(): bigint {
|
||||
@@ -93,12 +100,16 @@ interface SharedAttachment {
|
||||
}
|
||||
|
||||
interface SeedOptions {
|
||||
ageDays: number;
|
||||
ageDays?: number;
|
||||
resolvedDaysAgo?: number;
|
||||
attachment?: SharedAttachment;
|
||||
profile?: boolean;
|
||||
legalHoldUntil?: Date | null;
|
||||
status?: number;
|
||||
resolvedDaysAgo?: number;
|
||||
category?: string;
|
||||
outcome?: ReportItemOutcome | null;
|
||||
additionalInfo?: string | null;
|
||||
reporterId?: bigint | null;
|
||||
snapshotJson?: string;
|
||||
}
|
||||
|
||||
@@ -116,20 +127,96 @@ function attachmentKey(attachment: SharedAttachment): string {
|
||||
return makeAttachmentCdnKey(createChannelID(attachment.channelId), attachment.attachmentId, attachment.filename);
|
||||
}
|
||||
|
||||
function storedReport(overrides: Partial<IARSubmission>): IARSubmission {
|
||||
return {
|
||||
reportId: createReportID(nextId()),
|
||||
reportedAt: daysAgo(500),
|
||||
status: 1,
|
||||
resolvedAt: daysAgo(10),
|
||||
category: 'harassment',
|
||||
reporterOutcome: 'removed',
|
||||
...overrides,
|
||||
} as IARSubmission;
|
||||
}
|
||||
|
||||
describe('report retention dates', () => {
|
||||
const day = DAY_MS;
|
||||
|
||||
it('keeps a pending report until it is resolved', () => {
|
||||
expect(reportRetentionDates(storedReport({status: 0, resolvedAt: null}), DEFAULT_POLICY, null)).toEqual({
|
||||
deleteAt: null,
|
||||
contextExpiresAt: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('deletes an actioned report one year after it was resolved and keeps its context until then', () => {
|
||||
const report = storedReport({});
|
||||
expect(reportRetentionDates(report, DEFAULT_POLICY, null)).toEqual({
|
||||
deleteAt: report.resolvedAt!.getTime() + 365 * day,
|
||||
contextExpiresAt: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('drops the context of a no violation report once the complaint window has passed', () => {
|
||||
const report = storedReport({reporterOutcome: 'no_violation'});
|
||||
const dates = reportRetentionDates(report, DEFAULT_POLICY, null);
|
||||
expect(dates.contextExpiresAt).toBe(report.resolvedAt!.getTime() + REPORT_COMPLAINT_WINDOW_DAYS * day);
|
||||
expect(REPORT_COMPLAINT_WINDOW_DAYS).toBeGreaterThanOrEqual(183);
|
||||
expect(dates.deleteAt).toBe(report.resolvedAt!.getTime() + 365 * day);
|
||||
});
|
||||
|
||||
it('keeps a child safety report and its context for at least a year whatever the configured period', () => {
|
||||
const report = storedReport({category: 'child_safety', reporterOutcome: 'no_violation'});
|
||||
expect(reportRetentionDates(report, {retentionDays: 30}, null)).toEqual({
|
||||
deleteAt: report.resolvedAt!.getTime() + CHILD_SAFETY_REPORT_RETENTION_DAYS * day,
|
||||
contextExpiresAt: null,
|
||||
});
|
||||
expect(CHILD_SAFETY_REPORT_RETENTION_DAYS).toBeGreaterThanOrEqual(365);
|
||||
});
|
||||
|
||||
it('keeps a report linked to a decision as long as the decision evidence and keeps its context', () => {
|
||||
const report = storedReport({resolvedAt: daysAgo(300), reporterOutcome: 'no_violation'});
|
||||
const decidedAt = daysAgo(100);
|
||||
expect(reportRetentionDates(report, DEFAULT_POLICY, decidedAt)).toEqual({
|
||||
deleteAt: decidedAt.getTime() + 365 * day,
|
||||
contextExpiresAt: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('uses the longer period when the configured one is longer than a floor', () => {
|
||||
const report = storedReport({category: 'child_safety'});
|
||||
expect(reportRetentionDates(report, {retentionDays: 730}, null).deleteAt).toBe(
|
||||
report.resolvedAt!.getTime() + 730 * day,
|
||||
);
|
||||
});
|
||||
|
||||
it('counts from the filing time for a resolved report without a resolution time', () => {
|
||||
const report = storedReport({resolvedAt: null});
|
||||
expect(reportRetentionDates(report, DEFAULT_POLICY, null).deleteAt).toBe(report.reportedAt.getTime() + 365 * day);
|
||||
});
|
||||
});
|
||||
|
||||
describe('report retention', () => {
|
||||
let repository: ReportRepository;
|
||||
let decisions: EnforcementDecisionRepository;
|
||||
let storage: MockStorageService;
|
||||
let search: IReportSearchService;
|
||||
let deps: ReportDeletionDeps;
|
||||
let deps: ReportRetentionDeps;
|
||||
|
||||
beforeEach(() => {
|
||||
repository = new ReportRepository();
|
||||
decisions = new EnforcementDecisionRepository();
|
||||
storage = new MockStorageService();
|
||||
search = new InMemorySearchProvider().getReportSearchService();
|
||||
deps = {reportRepository: repository, storageService: storage, reportSearchService: search};
|
||||
deps = {
|
||||
reportRepository: repository,
|
||||
storageService: storage,
|
||||
reportSearchService: search,
|
||||
decisionRepository: decisions,
|
||||
};
|
||||
});
|
||||
|
||||
async function seed(options: SeedOptions): Promise<SeededReport> {
|
||||
async function seed(options: SeedOptions = {}): Promise<SeededReport> {
|
||||
const id = nextId();
|
||||
const reportId = createReportID(id);
|
||||
let storedAttachmentKey: string | null = null;
|
||||
@@ -161,14 +248,18 @@ describe('report retention', () => {
|
||||
guild: null,
|
||||
});
|
||||
}
|
||||
const resolved = options.resolvedDaysAgo !== undefined;
|
||||
const row = {
|
||||
...Object.fromEntries(IAR_SUBMISSION_COLUMNS.map((column) => [column, null])),
|
||||
report_id: id,
|
||||
reported_at: daysAgo(options.ageDays),
|
||||
status: options.status ?? (options.resolvedDaysAgo === undefined ? 0 : 1),
|
||||
resolved_at: options.resolvedDaysAgo === undefined ? null : daysAgo(options.resolvedDaysAgo),
|
||||
reporter_id: options.reporterId ?? null,
|
||||
reported_at: daysAgo(options.ageDays ?? (options.resolvedDaysAgo ?? 0) + 5),
|
||||
status: options.status ?? (resolved ? 1 : 0),
|
||||
resolved_at: resolved ? daysAgo(options.resolvedDaysAgo!) : null,
|
||||
reporter_outcome: options.outcome === undefined ? (resolved ? 'removed' : null) : options.outcome,
|
||||
report_type: 0,
|
||||
category: 'other',
|
||||
category: options.category ?? 'other',
|
||||
additional_info: options.additionalInfo ?? null,
|
||||
reported_channel_id: options.attachment?.channelId ?? null,
|
||||
message_context: messageContext,
|
||||
reported_profile_snapshot: snapshot,
|
||||
@@ -180,6 +271,30 @@ describe('report retention', () => {
|
||||
return {reportId, attachmentKey: storedAttachmentKey, profileKey};
|
||||
}
|
||||
|
||||
async function linkDecision(reportIds: Array<ReportID>, createdAt: Date): Promise<void> {
|
||||
const decisionId = nextId();
|
||||
const decision = {
|
||||
...Object.fromEntries(ENFORCEMENT_DECISION_COLUMNS.map((column) => [column, null])),
|
||||
decision_id: decisionId,
|
||||
subject_type: 'user',
|
||||
subject_id: nextId(),
|
||||
kind: 'warning',
|
||||
reason_code: DeletionReasons.HARASSMENT_OR_BULLYING,
|
||||
source: 'report',
|
||||
automation: 'none',
|
||||
report_ids: new Set<bigint>(reportIds),
|
||||
decided_by: nextId(),
|
||||
created_at: createdAt,
|
||||
status: 'active',
|
||||
notice_status: 'sent',
|
||||
} as EnforcementDecisionRow;
|
||||
await decisions.create({
|
||||
decision,
|
||||
evidence: null,
|
||||
firstNote: {decision_id: decisionId, note_id: nextId(), author_id: nextId(), body: 'note', created_at: createdAt},
|
||||
});
|
||||
}
|
||||
|
||||
async function indexedIds(): Promise<Array<string>> {
|
||||
const result = await search.searchReports('', {}, {limit: 1000});
|
||||
return result.hits.map((hit) => hit.id).sort();
|
||||
@@ -199,11 +314,27 @@ describe('report retention', () => {
|
||||
if (seeded.profileKey) expect(storage.hasObject(REPORTS_BUCKET, seeded.profileKey)).toBe(false);
|
||||
}
|
||||
|
||||
it('deletes a report 365 days after it was filed with its objects, search document and row', async () => {
|
||||
const expired = await seed({ageDays: 365, attachment: newAttachment(), profile: true});
|
||||
const kept = await seed({ageDays: 364, attachment: newAttachment(), profile: true});
|
||||
async function expectContextCleared(seeded: SeededReport): Promise<void> {
|
||||
const report = await repository.getReport(seeded.reportId);
|
||||
expect(report).not.toBeNull();
|
||||
expect(report?.messageContext).toBeNull();
|
||||
expect(report?.reportedProfileSnapshot).toBeNull();
|
||||
expect(report?.additionalInfo).toBeNull();
|
||||
expect(await indexedIds()).toContain(seeded.reportId.toString());
|
||||
for (const key of [seeded.attachmentKey, seeded.profileKey]) {
|
||||
if (key) expect(storage.hasObject(REPORTS_BUCKET, key)).toBe(false);
|
||||
}
|
||||
}
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
function run(options: Partial<ReportRetentionOptions> = {}) {
|
||||
return processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY, ...options});
|
||||
}
|
||||
|
||||
it('deletes a report 365 days after it was resolved with its objects, search document and row', async () => {
|
||||
const expired = await seed({resolvedDaysAgo: 365, attachment: newAttachment(), profile: true});
|
||||
const kept = await seed({resolvedDaysAgo: 364, attachment: newAttachment(), profile: true});
|
||||
|
||||
const summary = await run();
|
||||
|
||||
await expectGone(expired);
|
||||
expect(storage.hasObject(REPORTS_BUCKET, expired.attachmentKey!)).toBe(false);
|
||||
@@ -211,31 +342,175 @@ describe('report retention', () => {
|
||||
expect(summary).toEqual({
|
||||
dryRun: false,
|
||||
scanned: 2,
|
||||
pending: 0,
|
||||
expired: 1,
|
||||
contextExpired: 0,
|
||||
held: 0,
|
||||
deleted: 1,
|
||||
contextCleared: 0,
|
||||
objectsDeleted: 2,
|
||||
sharedObjectsKept: 0,
|
||||
failed: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it('deletes expired reports whatever their status', async () => {
|
||||
const pending = await seed({ageDays: 400, status: 0});
|
||||
const resolved = await seed({ageDays: 400, status: 1});
|
||||
it('keeps a pending report however old it is', async () => {
|
||||
const pending = await seed({ageDays: 4000, attachment: newAttachment(), profile: true});
|
||||
|
||||
await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
const summary = await run({policy: {retentionDays: 1}});
|
||||
|
||||
await expectGone(pending);
|
||||
await expectGone(resolved);
|
||||
await expectKept(pending);
|
||||
expect(summary).toMatchObject({scanned: 1, pending: 1, expired: 0, deleted: 0, objectsDeleted: 0});
|
||||
});
|
||||
|
||||
it('drops the copied context of a no violation report after the complaint window and keeps the record', async () => {
|
||||
const outsideWindow = await seed({
|
||||
resolvedDaysAgo: REPORT_COMPLAINT_WINDOW_DAYS,
|
||||
outcome: 'no_violation',
|
||||
attachment: newAttachment(),
|
||||
profile: true,
|
||||
additionalInfo: 'They keep messaging my friends',
|
||||
});
|
||||
const insideWindow = await seed({
|
||||
resolvedDaysAgo: REPORT_COMPLAINT_WINDOW_DAYS - 1,
|
||||
outcome: 'no_violation',
|
||||
attachment: newAttachment(),
|
||||
profile: true,
|
||||
additionalInfo: 'Still open to a complaint',
|
||||
});
|
||||
const actioned = await seed({
|
||||
resolvedDaysAgo: REPORT_COMPLAINT_WINDOW_DAYS + 10,
|
||||
attachment: newAttachment(),
|
||||
profile: true,
|
||||
});
|
||||
|
||||
const summary = await run();
|
||||
|
||||
await expectContextCleared(outsideWindow);
|
||||
const cleared = await repository.getReport(outsideWindow.reportId);
|
||||
expect(cleared).toMatchObject({status: 1, category: 'other', reporterOutcome: 'no_violation'});
|
||||
expect((await search.searchReports('friends', {}, {limit: 10})).hits).toHaveLength(0);
|
||||
await expectKept(insideWindow);
|
||||
expect((await repository.getReport(insideWindow.reportId))?.additionalInfo).toBe('Still open to a complaint');
|
||||
await expectKept(actioned);
|
||||
expect(summary).toMatchObject({
|
||||
scanned: 3,
|
||||
expired: 0,
|
||||
contextExpired: 1,
|
||||
contextCleared: 1,
|
||||
deleted: 0,
|
||||
objectsDeleted: 2,
|
||||
failed: 0,
|
||||
});
|
||||
|
||||
const again = await run();
|
||||
expect(again).toMatchObject({contextExpired: 0, contextCleared: 0, objectsDeleted: 0});
|
||||
});
|
||||
|
||||
it('deletes a no violation report one year after it was resolved', async () => {
|
||||
const expired = await seed({resolvedDaysAgo: 365, outcome: 'no_violation', profile: true});
|
||||
|
||||
const summary = await run();
|
||||
|
||||
await expectGone(expired);
|
||||
expect(summary).toMatchObject({expired: 1, deleted: 1, contextExpired: 0});
|
||||
});
|
||||
|
||||
it('keeps an evidence copy that an actioned report shares with a no violation report', async () => {
|
||||
const shared = newAttachment();
|
||||
const noViolation = await seed({resolvedDaysAgo: 200, outcome: 'no_violation', attachment: shared});
|
||||
const actioned = await seed({resolvedDaysAgo: 200, attachment: shared});
|
||||
|
||||
const summary = await run();
|
||||
|
||||
expect((await repository.getReport(noViolation.reportId))?.messageContext).toBeNull();
|
||||
await expectKept(actioned);
|
||||
expect(summary).toMatchObject({contextCleared: 1, sharedObjectsKept: 1, objectsDeleted: 0});
|
||||
});
|
||||
|
||||
it('keeps a child safety report and its context for a year even with a shorter period or no violation', async () => {
|
||||
const childSafety = await seed({
|
||||
resolvedDaysAgo: 364,
|
||||
category: 'child_safety',
|
||||
outcome: 'no_violation',
|
||||
attachment: newAttachment(),
|
||||
profile: true,
|
||||
additionalInfo: 'Context for a CyberTip',
|
||||
});
|
||||
const childSafetyAfterYear = await seed({resolvedDaysAgo: 365, category: 'child_safety', profile: true});
|
||||
const ordinary = await seed({resolvedDaysAgo: 40, profile: true});
|
||||
|
||||
const summary = await run({policy: {retentionDays: 30}});
|
||||
|
||||
await expectKept(childSafety);
|
||||
expect((await repository.getReport(childSafety.reportId))?.additionalInfo).toBe('Context for a CyberTip');
|
||||
await expectGone(childSafetyAfterYear);
|
||||
await expectGone(ordinary);
|
||||
expect(summary).toMatchObject({expired: 2, contextExpired: 0, deleted: 2});
|
||||
});
|
||||
|
||||
it('keeps a child safety report under a legal hold past the year', async () => {
|
||||
const held = await seed({
|
||||
resolvedDaysAgo: 500,
|
||||
category: 'child_safety',
|
||||
profile: true,
|
||||
legalHoldUntil: new Date(NOW.getTime() + 30 * DAY_MS),
|
||||
});
|
||||
|
||||
const summary = await run();
|
||||
|
||||
await expectKept(held);
|
||||
expect(summary).toMatchObject({expired: 0, held: 1, deleted: 0});
|
||||
});
|
||||
|
||||
it('keeps a report linked to a decision until the decision evidence expires', async () => {
|
||||
const linked = await seed({
|
||||
resolvedDaysAgo: 400,
|
||||
outcome: 'no_violation',
|
||||
profile: true,
|
||||
attachment: newAttachment(),
|
||||
});
|
||||
const linkedLongAgo = await seed({resolvedDaysAgo: 800, profile: true});
|
||||
const unlinked = await seed({resolvedDaysAgo: 400, profile: true});
|
||||
await linkDecision([linked.reportId], daysAgo(500));
|
||||
await linkDecision([linked.reportId], daysAgo(100));
|
||||
await linkDecision([linkedLongAgo.reportId], daysAgo(700));
|
||||
|
||||
const summary = await run({pageSize: 1});
|
||||
|
||||
await expectKept(linked);
|
||||
expect((await repository.getReport(linked.reportId))?.messageContext).toHaveLength(1);
|
||||
await expectGone(linkedLongAgo);
|
||||
await expectGone(unlinked);
|
||||
expect(summary).toMatchObject({scanned: 3, expired: 2, contextExpired: 0, deleted: 2});
|
||||
});
|
||||
|
||||
it('deletes nothing when the decision scan cannot finish', async () => {
|
||||
const expired = await seed({resolvedDaysAgo: 400, profile: true});
|
||||
|
||||
await expect(
|
||||
processReportRetention(
|
||||
{
|
||||
...deps,
|
||||
decisionRepository: {
|
||||
listReportLinks: async () => {
|
||||
throw new Error('read timeout');
|
||||
},
|
||||
},
|
||||
},
|
||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
|
||||
),
|
||||
).rejects.toThrow('read timeout');
|
||||
|
||||
await expectKept(expired);
|
||||
});
|
||||
|
||||
it('keeps an object that a retained report still uses', async () => {
|
||||
const shared = newAttachment();
|
||||
const expired = await seed({ageDays: 500, attachment: shared});
|
||||
const retained = await seed({ageDays: 10, attachment: shared});
|
||||
const expired = await seed({resolvedDaysAgo: 500, attachment: shared});
|
||||
const retained = await seed({resolvedDaysAgo: 10, attachment: shared});
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
const summary = await run();
|
||||
|
||||
await expectGone(expired);
|
||||
await expectKept(retained);
|
||||
@@ -246,10 +521,10 @@ describe('report retention', () => {
|
||||
|
||||
it('deletes an object shared by two expired reports once', async () => {
|
||||
const shared = newAttachment();
|
||||
const first = await seed({ageDays: 500, attachment: shared});
|
||||
const second = await seed({ageDays: 450, attachment: shared});
|
||||
const first = await seed({resolvedDaysAgo: 500, attachment: shared});
|
||||
const second = await seed({resolvedDaysAgo: 450, attachment: shared});
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
const summary = await run();
|
||||
|
||||
await expectGone(first);
|
||||
await expectGone(second);
|
||||
@@ -261,15 +536,15 @@ describe('report retention', () => {
|
||||
it('keeps a report under a legal hold that ends later and deletes one whose hold has ended', async () => {
|
||||
const shared = newAttachment();
|
||||
const held = await seed({
|
||||
ageDays: 500,
|
||||
resolvedDaysAgo: 500,
|
||||
attachment: shared,
|
||||
profile: true,
|
||||
legalHoldUntil: new Date(NOW.getTime() + DAY_MS),
|
||||
});
|
||||
const holdEnded = await seed({ageDays: 500, profile: true, legalHoldUntil: daysAgo(1)});
|
||||
const sharingWithHeld = await seed({ageDays: 500, attachment: shared});
|
||||
const holdEnded = await seed({resolvedDaysAgo: 500, profile: true, legalHoldUntil: daysAgo(1)});
|
||||
const sharingWithHeld = await seed({resolvedDaysAgo: 500, attachment: shared});
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
const summary = await run();
|
||||
|
||||
await expectKept(held);
|
||||
expect((await repository.getReport(held.reportId))?.legalHoldReason).toBe('Preservation request');
|
||||
@@ -279,31 +554,61 @@ describe('report retention', () => {
|
||||
expect(summary).toMatchObject({scanned: 3, expired: 2, held: 1, deleted: 2, sharedObjectsKept: 1});
|
||||
});
|
||||
|
||||
it('changes nothing in a dry run and reports what it would delete', async () => {
|
||||
const expired = await seed({ageDays: 366, attachment: newAttachment(), profile: true});
|
||||
const kept = await seed({ageDays: 1, profile: true});
|
||||
it('keeps the context of a no violation report under a legal hold', async () => {
|
||||
const held = await seed({
|
||||
resolvedDaysAgo: 250,
|
||||
outcome: 'no_violation',
|
||||
attachment: newAttachment(),
|
||||
legalHoldUntil: new Date(NOW.getTime() + DAY_MS),
|
||||
});
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: true, policy: DEFAULT_POLICY});
|
||||
const summary = await run();
|
||||
|
||||
await expectKept(expired);
|
||||
await expectKept(kept);
|
||||
await expectKept(held);
|
||||
expect((await repository.getReport(held.reportId))?.messageContext).toHaveLength(1);
|
||||
expect(summary).toMatchObject({held: 1, contextExpired: 0, contextCleared: 0});
|
||||
});
|
||||
|
||||
it('changes nothing in a dry run and counts what it would do to existing reports', async () => {
|
||||
const expired = await seed({resolvedDaysAgo: 366, attachment: newAttachment(), profile: true});
|
||||
const contextDue = await seed({
|
||||
resolvedDaysAgo: 200,
|
||||
outcome: 'no_violation',
|
||||
attachment: newAttachment(),
|
||||
additionalInfo: 'details',
|
||||
});
|
||||
const childSafety = await seed({resolvedDaysAgo: 200, category: 'child_safety', outcome: 'no_violation'});
|
||||
const held = await seed({resolvedDaysAgo: 400, legalHoldUntil: new Date(NOW.getTime() + DAY_MS)});
|
||||
const pending = await seed({profile: true});
|
||||
const linked = await seed({resolvedDaysAgo: 400});
|
||||
await linkDecision([linked.reportId], daysAgo(30));
|
||||
|
||||
const summary = await run({dryRun: true});
|
||||
|
||||
for (const report of [expired, contextDue, childSafety, held, pending, linked]) {
|
||||
await expectKept(report);
|
||||
}
|
||||
expect((await repository.getReport(contextDue.reportId))?.additionalInfo).toBe('details');
|
||||
expect(storage.deleteObjectSpy).not.toHaveBeenCalled();
|
||||
expect(summary).toEqual({
|
||||
dryRun: true,
|
||||
scanned: 2,
|
||||
scanned: 6,
|
||||
pending: 1,
|
||||
expired: 1,
|
||||
held: 0,
|
||||
contextExpired: 1,
|
||||
held: 1,
|
||||
deleted: 1,
|
||||
objectsDeleted: 2,
|
||||
contextCleared: 1,
|
||||
objectsDeleted: 3,
|
||||
sharedObjectsKept: 0,
|
||||
failed: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it('deletes nothing when the scan cannot finish', async () => {
|
||||
const expired = await seed({ageDays: 400, attachment: newAttachment(), profile: true});
|
||||
await seed({ageDays: 400});
|
||||
await seed({ageDays: 400});
|
||||
const expired = await seed({resolvedDaysAgo: 400, attachment: newAttachment(), profile: true});
|
||||
await seed({resolvedDaysAgo: 400});
|
||||
await seed({resolvedDaysAgo: 400});
|
||||
let pages = 0;
|
||||
const failingRepository = Object.create(repository) as ReportRepository;
|
||||
failingRepository.listAllReportsPaginated = async (limit: number, lastReportId?: ReportID) => {
|
||||
@@ -315,7 +620,12 @@ describe('report retention', () => {
|
||||
await expect(
|
||||
processReportRetention(
|
||||
{...deps, reportRepository: failingRepository},
|
||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY, pageSize: 2},
|
||||
{
|
||||
now: NOW,
|
||||
dryRun: false,
|
||||
policy: DEFAULT_POLICY,
|
||||
pageSize: 2,
|
||||
},
|
||||
),
|
||||
).rejects.toThrow('read timeout');
|
||||
|
||||
@@ -328,11 +638,11 @@ describe('report retention', () => {
|
||||
const expired: Array<SeededReport> = [];
|
||||
const kept: Array<SeededReport> = [];
|
||||
for (let index = 0; index < 7; index++) {
|
||||
expired.push(await seed({ageDays: 400 + index, profile: true}));
|
||||
kept.push(await seed({ageDays: index, profile: true}));
|
||||
expired.push(await seed({resolvedDaysAgo: 400 + index, profile: true}));
|
||||
kept.push(await seed({resolvedDaysAgo: index, profile: true}));
|
||||
}
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY, pageSize: 3});
|
||||
const summary = await run({pageSize: 3});
|
||||
|
||||
expect(summary).toMatchObject({scanned: 14, expired: 7, deleted: 7, objectsDeleted: 7, failed: 0});
|
||||
for (const report of expired) await expectGone(report);
|
||||
@@ -340,25 +650,57 @@ describe('report retention', () => {
|
||||
});
|
||||
|
||||
it('keeps the row and search document when an object cannot be deleted, then deletes them on the next run', async () => {
|
||||
const expired = await seed({ageDays: 400, attachment: newAttachment(), profile: true});
|
||||
const expired = await seed({resolvedDaysAgo: 400, attachment: newAttachment(), profile: true});
|
||||
storage.configure({shouldFailDelete: true});
|
||||
|
||||
const failed = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
const failed = await run();
|
||||
|
||||
expect(failed).toMatchObject({expired: 1, deleted: 0, failed: 1});
|
||||
expect(await repository.getReport(expired.reportId)).not.toBeNull();
|
||||
expect(await indexedIds()).toContain(expired.reportId.toString());
|
||||
|
||||
storage.configure({shouldFailDelete: false});
|
||||
const retried = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
const retried = await run();
|
||||
|
||||
expect(retried).toMatchObject({expired: 1, deleted: 1, failed: 0});
|
||||
await expectGone(expired);
|
||||
expect(storage.hasObject(REPORTS_BUCKET, expired.attachmentKey!)).toBe(false);
|
||||
});
|
||||
|
||||
it('keeps the copied context when an object cannot be deleted, then clears it on the next run', async () => {
|
||||
const due = await seed({resolvedDaysAgo: 200, outcome: 'no_violation', attachment: newAttachment()});
|
||||
storage.configure({shouldFailDelete: true});
|
||||
|
||||
expect(await run()).toMatchObject({contextExpired: 1, contextCleared: 0, failed: 1});
|
||||
expect((await repository.getReport(due.reportId))?.messageContext).toHaveLength(1);
|
||||
|
||||
storage.configure({shouldFailDelete: false});
|
||||
expect(await run()).toMatchObject({contextExpired: 1, contextCleared: 1, failed: 0});
|
||||
await expectContextCleared(due);
|
||||
});
|
||||
|
||||
it('does not bring back a report deleted while its context was being cleared', async () => {
|
||||
const due = await seed({resolvedDaysAgo: 200, outcome: 'no_violation', additionalInfo: 'details'});
|
||||
const deletingRepository = Object.create(repository) as ReportRepository;
|
||||
deletingRepository.clearCopiedContext = async (reportId, reportType) => {
|
||||
await repository.deleteReport(reportId);
|
||||
return repository.clearCopiedContext(reportId, reportType);
|
||||
};
|
||||
|
||||
await processReportRetention(
|
||||
{...deps, reportRepository: deletingRepository},
|
||||
{
|
||||
now: NOW,
|
||||
dryRun: false,
|
||||
policy: DEFAULT_POLICY,
|
||||
},
|
||||
);
|
||||
|
||||
expect(await repository.getReport(due.reportId)).toBeNull();
|
||||
});
|
||||
|
||||
it('keeps the row when the search document cannot be deleted', async () => {
|
||||
const expired = await seed({ageDays: 400});
|
||||
const expired = await seed({resolvedDaysAgo: 400});
|
||||
const failingSearch = Object.create(search) as IReportSearchService;
|
||||
failingSearch.deleteReport = async () => {
|
||||
throw new Error('search unavailable');
|
||||
@@ -366,7 +708,11 @@ describe('report retention', () => {
|
||||
|
||||
const summary = await processReportRetention(
|
||||
{...deps, reportSearchService: failingSearch},
|
||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
|
||||
{
|
||||
now: NOW,
|
||||
dryRun: false,
|
||||
policy: DEFAULT_POLICY,
|
||||
},
|
||||
);
|
||||
|
||||
expect(summary).toMatchObject({deleted: 0, failed: 1});
|
||||
@@ -374,11 +720,15 @@ describe('report retention', () => {
|
||||
});
|
||||
|
||||
it('deletes rows when search is not configured', async () => {
|
||||
const expired = await seed({ageDays: 400, profile: true});
|
||||
const expired = await seed({resolvedDaysAgo: 400, profile: true});
|
||||
|
||||
const summary = await processReportRetention(
|
||||
{...deps, reportSearchService: null},
|
||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
|
||||
{
|
||||
now: NOW,
|
||||
dryRun: false,
|
||||
policy: DEFAULT_POLICY,
|
||||
},
|
||||
);
|
||||
|
||||
expect(summary).toMatchObject({deleted: 1, failed: 0});
|
||||
@@ -389,8 +739,8 @@ describe('report retention', () => {
|
||||
it('skips a report that was put on hold after the scan', async () => {
|
||||
const shared = newAttachment();
|
||||
const seeded = [
|
||||
await seed({ageDays: 400, attachment: shared, profile: true}),
|
||||
await seed({ageDays: 400, attachment: shared}),
|
||||
await seed({resolvedDaysAgo: 400, attachment: shared, profile: true}),
|
||||
await seed({resolvedDaysAgo: 400, attachment: shared}),
|
||||
];
|
||||
const scanOrder = (await repository.listAllReportsPaginated(10)).map((report) => report.reportId);
|
||||
const [first, second] = seeded.sort((a, b) => scanOrder.indexOf(a.reportId) - scanOrder.indexOf(b.reportId)) as [
|
||||
@@ -408,7 +758,11 @@ describe('report retention', () => {
|
||||
|
||||
const summary = await processReportRetention(
|
||||
{...deps, reportRepository: holdingRepository},
|
||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
|
||||
{
|
||||
now: NOW,
|
||||
dryRun: false,
|
||||
policy: DEFAULT_POLICY,
|
||||
},
|
||||
);
|
||||
|
||||
await expectKept(first);
|
||||
@@ -419,7 +773,7 @@ describe('report retention', () => {
|
||||
|
||||
it('keeps an object that a report filed during the run stored again', async () => {
|
||||
const shared = newAttachment();
|
||||
const expired = await seed({ageDays: 400, attachment: shared, profile: true});
|
||||
const expired = await seed({resolvedDaysAgo: 400, attachment: shared, profile: true});
|
||||
const rewritingStorage = Object.create(storage) as MockStorageService;
|
||||
rewritingStorage.getObjectMetadata = async (bucket: string, key: string) => {
|
||||
const metadata = await storage.getObjectMetadata(bucket, key);
|
||||
@@ -431,7 +785,11 @@ describe('report retention', () => {
|
||||
|
||||
const summary = await processReportRetention(
|
||||
{...deps, storageService: rewritingStorage},
|
||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
|
||||
{
|
||||
now: NOW,
|
||||
dryRun: false,
|
||||
policy: DEFAULT_POLICY,
|
||||
},
|
||||
);
|
||||
|
||||
await expectGone(expired);
|
||||
@@ -440,76 +798,25 @@ describe('report retention', () => {
|
||||
});
|
||||
|
||||
it('deletes profile images stored for a report whose snapshot cannot be read', async () => {
|
||||
const expired = await seed({ageDays: 400, profile: true, snapshotJson: '{"not":"a snapshot"'});
|
||||
const expired = await seed({resolvedDaysAgo: 400, profile: true, snapshotJson: '{"not":"a snapshot"'});
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
const summary = await run();
|
||||
|
||||
expect(summary).toMatchObject({deleted: 1, objectsDeleted: 1});
|
||||
await expectGone(expired);
|
||||
});
|
||||
|
||||
function run(options: Partial<ReportRetentionOptions> = {}) {
|
||||
return processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY, ...options});
|
||||
}
|
||||
|
||||
it('deletes earlier when the retention period is shorter', async () => {
|
||||
const expired = await seed({ageDays: 30, profile: true});
|
||||
const kept = await seed({ageDays: 29, profile: true});
|
||||
const expired = await seed({resolvedDaysAgo: 30, profile: true});
|
||||
const kept = await seed({resolvedDaysAgo: 29, profile: true});
|
||||
|
||||
const summary = await run({policy: {retentionDays: 30, resolvedRetentionDays: null}});
|
||||
const summary = await run({policy: {retentionDays: 30}});
|
||||
|
||||
await expectGone(expired);
|
||||
await expectKept(kept);
|
||||
expect(summary).toMatchObject({scanned: 2, expired: 1, deleted: 1});
|
||||
});
|
||||
|
||||
it('deletes a resolved report once the resolved-report period has passed since it was resolved', async () => {
|
||||
const policy = {retentionDays: 365, resolvedRetentionDays: 30};
|
||||
const resolvedLongAgo = await seed({ageDays: 100, resolvedDaysAgo: 30, attachment: newAttachment()});
|
||||
const resolvedRecently = await seed({ageDays: 100, resolvedDaysAgo: 29, profile: true});
|
||||
const pending = await seed({ageDays: 100, profile: true});
|
||||
const resolvedWithoutTime = await seed({ageDays: 100, status: 1, profile: true});
|
||||
|
||||
const summary = await run({policy});
|
||||
|
||||
await expectGone(resolvedLongAgo);
|
||||
expect(storage.hasObject(REPORTS_BUCKET, resolvedLongAgo.attachmentKey!)).toBe(false);
|
||||
await expectKept(resolvedRecently);
|
||||
await expectKept(pending);
|
||||
await expectKept(resolvedWithoutTime);
|
||||
expect(summary).toMatchObject({scanned: 4, expired: 1, deleted: 1, held: 0});
|
||||
});
|
||||
|
||||
it('applies whichever rule ends first', async () => {
|
||||
const policy = {retentionDays: 100, resolvedRetentionDays: 300};
|
||||
const byAge = await seed({ageDays: 100, resolvedDaysAgo: 10});
|
||||
const neither = await seed({ageDays: 99, resolvedDaysAgo: 90});
|
||||
|
||||
await run({policy});
|
||||
|
||||
await expectGone(byAge);
|
||||
await expectKept(neither);
|
||||
});
|
||||
|
||||
it('keeps a held resolved report past the resolved-report period and keeps its evidence', async () => {
|
||||
const shared = newAttachment();
|
||||
const held = await seed({
|
||||
ageDays: 50,
|
||||
resolvedDaysAgo: 40,
|
||||
attachment: shared,
|
||||
profile: true,
|
||||
legalHoldUntil: new Date(NOW.getTime() + DAY_MS),
|
||||
});
|
||||
const sharingWithHeld = await seed({ageDays: 50, resolvedDaysAgo: 40, attachment: shared});
|
||||
|
||||
const summary = await run({policy: {retentionDays: 365, resolvedRetentionDays: 7}});
|
||||
|
||||
await expectKept(held);
|
||||
await expectGone(sharingWithHeld);
|
||||
expect(storage.hasObject(REPORTS_BUCKET, attachmentKey(shared))).toBe(true);
|
||||
expect(summary).toMatchObject({expired: 1, held: 1, deleted: 1, sharedObjectsKept: 1});
|
||||
});
|
||||
|
||||
it('frees the reporter to report the same account again once the report is deleted', async () => {
|
||||
const reporterId = nextId();
|
||||
const reportedUserId = nextId();
|
||||
@@ -519,8 +826,9 @@ describe('report retention', () => {
|
||||
report_id: BigInt(reportId),
|
||||
reporter_id: reporterId,
|
||||
reported_user_id: reportedUserId,
|
||||
reported_at: daysAgo(400),
|
||||
status: 0,
|
||||
reported_at: daysAgo(420),
|
||||
status: 1,
|
||||
resolved_at: daysAgo(400),
|
||||
report_type: 1,
|
||||
category: 'harassment',
|
||||
} as IARSubmissionRow);
|
||||
@@ -528,7 +836,7 @@ describe('report retention', () => {
|
||||
reporter_id: createUserID(reporterId),
|
||||
reported_user_id: createUserID(reportedUserId),
|
||||
report_id: reportId,
|
||||
reported_at: daysAgo(400),
|
||||
reported_at: daysAgo(420),
|
||||
};
|
||||
expect(await repository.reserveUserReportByReporter(reservation)).toBe(true);
|
||||
|
||||
@@ -559,9 +867,10 @@ describe('report retention', () => {
|
||||
}
|
||||
|
||||
it('deletes for real with the default settings', async () => {
|
||||
expect(configured).toEqual({days: 365, resolvedDays: null, dryRun: false});
|
||||
const expired = await seed({ageDays: 366, attachment: newAttachment(), profile: true});
|
||||
const kept = await seed({ageDays: 364, profile: true});
|
||||
expect(configured).toEqual({days: 365, dryRun: false});
|
||||
const expired = await seed({resolvedDaysAgo: 366, attachment: newAttachment(), profile: true});
|
||||
const kept = await seed({resolvedDaysAgo: 364, profile: true});
|
||||
const pending = await seed({profile: true});
|
||||
|
||||
await runDailyTask();
|
||||
|
||||
@@ -569,11 +878,12 @@ describe('report retention', () => {
|
||||
expect(storage.hasObject(REPORTS_BUCKET, expired.attachmentKey!)).toBe(false);
|
||||
expect(storage.hasObject(REPORTS_BUCKET, expired.profileKey!)).toBe(false);
|
||||
expect(await repository.getReport(kept.reportId)).not.toBeNull();
|
||||
expect(await repository.getReport(pending.reportId)).not.toBeNull();
|
||||
});
|
||||
|
||||
it('only counts what it would delete when FLUXER_REPORT_RETENTION_DRY_RUN is true', async () => {
|
||||
Object.assign(Config.reportRetention, {dryRun: true});
|
||||
const expired = await seed({ageDays: 366, attachment: newAttachment(), profile: true});
|
||||
const expired = await seed({resolvedDaysAgo: 366, attachment: newAttachment(), profile: true});
|
||||
|
||||
await runDailyTask();
|
||||
|
||||
@@ -581,17 +891,18 @@ describe('report retention', () => {
|
||||
expect(storage.deleteObjectSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('reads both day settings from the config', async () => {
|
||||
Object.assign(Config.reportRetention, {days: 60, resolvedDays: 5});
|
||||
const byAge = await seed({ageDays: 60});
|
||||
const byResolution = await seed({ageDays: 20, resolvedDaysAgo: 5});
|
||||
const kept = await seed({ageDays: 20, resolvedDaysAgo: 4});
|
||||
it('reads the day setting from the config and honors decisions', async () => {
|
||||
Object.assign(Config.reportRetention, {days: 60});
|
||||
const expired = await seed({resolvedDaysAgo: 60});
|
||||
const kept = await seed({resolvedDaysAgo: 59});
|
||||
const linked = await seed({resolvedDaysAgo: 90});
|
||||
await linkDecision([linked.reportId], daysAgo(90));
|
||||
|
||||
await runDailyTask();
|
||||
|
||||
expect(await repository.getReport(byAge.reportId)).toBeNull();
|
||||
expect(await repository.getReport(byResolution.reportId)).toBeNull();
|
||||
expect(await repository.getReport(expired.reportId)).toBeNull();
|
||||
expect(await repository.getReport(kept.reportId)).not.toBeNull();
|
||||
expect(await repository.getReport(linked.reportId)).not.toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user