mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-17 04:55:42 +09:00
feat(reports): keep reports until resolved, then by outcome (#3493)
This commit is contained in:
1 parent
f5c30fdf8f
commit
4f88872a3c
18 files changed
+673
-238
No files matched your search
@@ -353,12 +353,10 @@ FLUXER_DISCOVERY_ENABLED=true
|
|||||||
#FLUXER_AUTO_JOIN_INVITE_CODE=
|
#FLUXER_AUTO_JOIN_INVITE_CODE=
|
||||||
#FLUXER_DELETION_GRACE_PERIOD_HOURS=336
|
#FLUXER_DELETION_GRACE_PERIOD_HOURS=336
|
||||||
|
|
||||||
# Report retention. A daily job deletes each report this many days after it was
|
# Report retention. A daily job deletes each resolved report this many days
|
||||||
# filed, with its evidence copies and search entry, unless a legal hold is set.
|
# after it was resolved, with its evidence copies and search entry, unless a
|
||||||
|
# legal hold is set. Pending reports are kept until they are resolved.
|
||||||
#FLUXER_REPORT_RETENTION_DAYS=365
|
#FLUXER_REPORT_RETENTION_DAYS=365
|
||||||
# Delete resolved reports this many days after they were resolved, when that
|
|
||||||
# comes first. Unset leaves them to FLUXER_REPORT_RETENTION_DAYS.
|
|
||||||
#FLUXER_RESOLVED_REPORT_RETENTION_DAYS=
|
|
||||||
# true only logs what the job would delete. Deleted evidence cannot be
|
# true only logs what the job would delete. Deleted evidence cannot be
|
||||||
# restored, so check the "Processed report retention" log line of the worker
|
# restored, so check the "Processed report retention" log line of the worker
|
||||||
# in a dry run first if you are unsure.
|
# in a dry run first if you are unsure.
|
||||||
|
|||||||
@@ -159,7 +159,6 @@ x-fluxer-env: &fluxer-env
|
|||||||
FLUXER_DISCOVERY_MIN_MEMBER_COUNT: ${FLUXER_DISCOVERY_MIN_MEMBER_COUNT:-}
|
FLUXER_DISCOVERY_MIN_MEMBER_COUNT: ${FLUXER_DISCOVERY_MIN_MEMBER_COUNT:-}
|
||||||
FLUXER_DELETION_GRACE_PERIOD_HOURS: ${FLUXER_DELETION_GRACE_PERIOD_HOURS:-}
|
FLUXER_DELETION_GRACE_PERIOD_HOURS: ${FLUXER_DELETION_GRACE_PERIOD_HOURS:-}
|
||||||
FLUXER_REPORT_RETENTION_DAYS: ${FLUXER_REPORT_RETENTION_DAYS:-}
|
FLUXER_REPORT_RETENTION_DAYS: ${FLUXER_REPORT_RETENTION_DAYS:-}
|
||||||
FLUXER_RESOLVED_REPORT_RETENTION_DAYS: ${FLUXER_RESOLVED_REPORT_RETENTION_DAYS:-}
|
|
||||||
FLUXER_REPORT_RETENTION_DRY_RUN: ${FLUXER_REPORT_RETENTION_DRY_RUN:-}
|
FLUXER_REPORT_RETENTION_DRY_RUN: ${FLUXER_REPORT_RETENTION_DRY_RUN:-}
|
||||||
FLUXER_API_WORKER_LANE_CONCURRENCY_OVERRIDES: ${FLUXER_API_WORKER_LANE_CONCURRENCY_OVERRIDES:-}
|
FLUXER_API_WORKER_LANE_CONCURRENCY_OVERRIDES: ${FLUXER_API_WORKER_LANE_CONCURRENCY_OVERRIDES:-}
|
||||||
|
|
||||||
|
|||||||
@@ -449,7 +449,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
|
|||||||
inactivityDeletionThresholdDays: master.inactivity_deletion_threshold_days,
|
inactivityDeletionThresholdDays: master.inactivity_deletion_threshold_days,
|
||||||
reportRetention: {
|
reportRetention: {
|
||||||
days: master.report_retention.days,
|
days: master.report_retention.days,
|
||||||
resolvedDays: master.report_retention.resolved_days,
|
|
||||||
dryRun: master.report_retention.dry_run,
|
dryRun: master.report_retention.dry_run,
|
||||||
},
|
},
|
||||||
push: {
|
push: {
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';
|
import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';
|
||||||
import {createReportID, type ReportID} from '@app/api/BrandedTypes';
|
import {createReportID, type ReportID} from '@app/api/BrandedTypes';
|
||||||
import {IAR_SUBMISSION_COLUMNS, type IARSubmissionRow} from '@app/api/database/types/ReportTypes';
|
import {IAR_SUBMISSION_COLUMNS, type IARSubmissionRow} from '@app/api/database/types/ReportTypes';
|
||||||
|
import {EnforcementDecisionRepository} from '@app/api/enforcement/EnforcementDecisionRepository';
|
||||||
import {ReportRepository} from '@app/api/report/ReportRepository';
|
import {ReportRepository} from '@app/api/report/ReportRepository';
|
||||||
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
|
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||||
import {MockStorageService} from '@app/api/test/mocks/MockStorageService';
|
import {MockStorageService} from '@app/api/test/mocks/MockStorageService';
|
||||||
@@ -47,6 +48,7 @@ describe('report legal hold', () => {
|
|||||||
...Object.fromEntries(IAR_SUBMISSION_COLUMNS.map((column) => [column, null])),
|
...Object.fromEntries(IAR_SUBMISSION_COLUMNS.map((column) => [column, null])),
|
||||||
report_id: sequence,
|
report_id: sequence,
|
||||||
reported_at: new Date(Date.now() - ageDays * ms('1 day')),
|
reported_at: new Date(Date.now() - ageDays * ms('1 day')),
|
||||||
|
resolved_at: new Date(Date.now() - ageDays * ms('1 day')),
|
||||||
status: 1,
|
status: 1,
|
||||||
report_type: 1,
|
report_type: 1,
|
||||||
category: 'harassment',
|
category: 'harassment',
|
||||||
@@ -69,8 +71,13 @@ describe('report legal hold', () => {
|
|||||||
|
|
||||||
async function runRetention() {
|
async function runRetention() {
|
||||||
return processReportRetention(
|
return processReportRetention(
|
||||||
{reportRepository: repository, storageService: new MockStorageService(), reportSearchService: null},
|
{
|
||||||
{now: new Date(), dryRun: false, policy: {retentionDays: 365, resolvedRetentionDays: null}},
|
reportRepository: repository,
|
||||||
|
storageService: new MockStorageService(),
|
||||||
|
reportSearchService: null,
|
||||||
|
decisionRepository: new EnforcementDecisionRepository(),
|
||||||
|
},
|
||||||
|
{now: new Date(), dryRun: false, policy: {retentionDays: 365}},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -308,7 +308,6 @@ export interface APIConfig {
|
|||||||
deletionGracePeriodHours: number;
|
deletionGracePeriodHours: number;
|
||||||
reportRetention: {
|
reportRetention: {
|
||||||
days: number;
|
days: number;
|
||||||
resolvedDays: number | null;
|
|
||||||
dryRun: boolean;
|
dryRun: boolean;
|
||||||
};
|
};
|
||||||
inactivityDeletionThresholdDays?: number;
|
inactivityDeletionThresholdDays?: number;
|
||||||
|
|||||||
@@ -40,6 +40,16 @@ const LIST_NOTES_QUERY = EnforcementDecisionNotes.select({
|
|||||||
orderBy: {col: 'note_id', direction: 'ASC'},
|
orderBy: {col: 'note_id', direction: 'ASC'},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
type DecisionReportLink = Pick<EnforcementDecisionRow, 'decision_id' | 'report_ids' | 'created_at'>;
|
||||||
|
|
||||||
|
function listReportLinksQuery(limit: number, paged: boolean) {
|
||||||
|
return EnforcementDecisions.select({
|
||||||
|
columns: ['decision_id', 'report_ids', 'created_at'],
|
||||||
|
...(paged ? {where: EnforcementDecisions.where.tokenGt('decision_id', 'after')} : {}),
|
||||||
|
limit,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
const REVIEW_QUEUE_BUCKET = 0;
|
const REVIEW_QUEUE_BUCKET = 0;
|
||||||
const LIST_REVIEW_QUEUE_QUERY = EnforcementReviewQueue.select({
|
const LIST_REVIEW_QUEUE_QUERY = EnforcementReviewQueue.select({
|
||||||
columns: ['decision_id'],
|
columns: ['decision_id'],
|
||||||
@@ -162,6 +172,12 @@ export class EnforcementDecisionRepository {
|
|||||||
return this.getMany(keys.map((key) => key.decision_id));
|
return this.getMany(keys.map((key) => key.decision_id));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async listReportLinks(limit: number, after?: bigint): Promise<Array<DecisionReportLink>> {
|
||||||
|
return fetchMany<DecisionReportLink>(
|
||||||
|
listReportLinksQuery(limit, after !== undefined).bind(after !== undefined ? {after} : {}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
async end(decisionId: bigint, end: EnforcementDecisionEnd): Promise<void> {
|
async end(decisionId: bigint, end: EnforcementDecisionEnd): Promise<void> {
|
||||||
await upsertOne(EnforcementDecisions.patchByPk({decision_id: decisionId}, endPatch(end)));
|
await upsertOne(EnforcementDecisions.patchByPk({decision_id: decisionId}, endPatch(end)));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -79,13 +79,12 @@ async function storedObjectKeys(storageService: IStorageService, report: IARSubm
|
|||||||
return keys;
|
return keys;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function deleteReportWithEvidence(
|
async function deleteReportEvidence(
|
||||||
deps: ReportDeletionDeps,
|
storageService: IStorageService,
|
||||||
report: IARSubmission,
|
report: IARSubmission,
|
||||||
ledger: ReportEvidenceLedger,
|
ledger: ReportEvidenceLedger,
|
||||||
options: {now: Date; dryRun: boolean},
|
options: {now: Date; dryRun: boolean},
|
||||||
): Promise<ReportDeletionResult> {
|
): Promise<ReportDeletionResult> {
|
||||||
const {reportRepository, storageService, reportSearchService} = deps;
|
|
||||||
const result: ReportDeletionResult = {objectsDeleted: 0, sharedObjectsKept: 0};
|
const result: ReportDeletionResult = {objectsDeleted: 0, sharedObjectsKept: 0};
|
||||||
for (const key of await storedObjectKeys(storageService, report)) {
|
for (const key of await storedObjectKeys(storageService, report)) {
|
||||||
if (ledger.retainedKeys.has(key)) {
|
if (ledger.retainedKeys.has(key)) {
|
||||||
@@ -105,6 +104,17 @@ export async function deleteReportWithEvidence(
|
|||||||
ledger.deletedKeys.add(key);
|
ledger.deletedKeys.add(key);
|
||||||
result.objectsDeleted++;
|
result.objectsDeleted++;
|
||||||
}
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function deleteReportWithEvidence(
|
||||||
|
deps: ReportDeletionDeps,
|
||||||
|
report: IARSubmission,
|
||||||
|
ledger: ReportEvidenceLedger,
|
||||||
|
options: {now: Date; dryRun: boolean},
|
||||||
|
): Promise<ReportDeletionResult> {
|
||||||
|
const {reportRepository, storageService, reportSearchService} = deps;
|
||||||
|
const result = await deleteReportEvidence(storageService, report, ledger, options);
|
||||||
if (!options.dryRun) {
|
if (!options.dryRun) {
|
||||||
await reportSearchService?.deleteReport(report.reportId);
|
await reportSearchService?.deleteReport(report.reportId);
|
||||||
await reportRepository.deleteReport(report.reportId);
|
await reportRepository.deleteReport(report.reportId);
|
||||||
@@ -112,6 +122,25 @@ export async function deleteReportWithEvidence(
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function clearCopiedContextWithEvidence(
|
||||||
|
deps: ReportDeletionDeps,
|
||||||
|
report: IARSubmission,
|
||||||
|
ledger: ReportEvidenceLedger,
|
||||||
|
options: {now: Date; dryRun: boolean},
|
||||||
|
): Promise<ReportDeletionResult> {
|
||||||
|
const {reportRepository, storageService, reportSearchService} = deps;
|
||||||
|
const result = await deleteReportEvidence(storageService, report, ledger, options);
|
||||||
|
if (!options.dryRun && (await reportRepository.clearCopiedContext(report.reportId, report.reportType))) {
|
||||||
|
await reportSearchService?.updateReport({
|
||||||
|
...report,
|
||||||
|
messageContext: null,
|
||||||
|
reportedProfileSnapshot: null,
|
||||||
|
additionalInfo: null,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
async function loadDeletableReport(
|
async function loadDeletableReport(
|
||||||
reportRepository: ReportRepository,
|
reportRepository: ReportRepository,
|
||||||
reportId: ReportID,
|
reportId: ReportID,
|
||||||
|
|||||||
@@ -186,6 +186,20 @@ export class ReportRepository {
|
|||||||
return {...report, legalHoldUntil, legalHoldReason};
|
return {...report, legalHoldUntil, legalHoldReason};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async clearCopiedContext(reportId: ReportID, reportType: number): Promise<boolean> {
|
||||||
|
return executeConditional(
|
||||||
|
IARSubmissions.conditionalPatchByPk(
|
||||||
|
{report_id: reportId},
|
||||||
|
{
|
||||||
|
message_context: Db.set(null),
|
||||||
|
reported_profile_snapshot: Db.set(null),
|
||||||
|
additional_info: Db.set(null),
|
||||||
|
},
|
||||||
|
{report_type: reportType},
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
async clearReporterEmail(reportId: ReportID, reporterId: UserID): Promise<boolean> {
|
async clearReporterEmail(reportId: ReportID, reporterId: UserID): Promise<boolean> {
|
||||||
return executeConditional(
|
return executeConditional(
|
||||||
IARSubmissions.conditionalPatchByPk(
|
IARSubmissions.conditionalPatchByPk(
|
||||||
|
|||||||
@@ -2,23 +2,36 @@
|
|||||||
|
|
||||||
import type {ReportID} from '@app/api/BrandedTypes';
|
import type {ReportID} from '@app/api/BrandedTypes';
|
||||||
import {Config} from '@app/api/Config';
|
import {Config} from '@app/api/Config';
|
||||||
|
import {EnforcementDecisionRepository} from '@app/api/enforcement/EnforcementDecisionRepository';
|
||||||
import {Logger} from '@app/api/Logger';
|
import {Logger} from '@app/api/Logger';
|
||||||
import {
|
import {
|
||||||
|
clearCopiedContextWithEvidence,
|
||||||
deleteReportWithEvidence,
|
deleteReportWithEvidence,
|
||||||
forEachStoredReport,
|
forEachStoredReport,
|
||||||
isReportUnderLegalHold,
|
isReportUnderLegalHold,
|
||||||
type ReportDeletionDeps,
|
type ReportDeletionDeps,
|
||||||
|
type ReportEvidenceLedger,
|
||||||
referencedObjectKeys,
|
referencedObjectKeys,
|
||||||
} from '@app/api/report/ReportDeletion';
|
} from '@app/api/report/ReportDeletion';
|
||||||
import {type IARSubmission, ReportStatus} from '@app/api/report/ReportModels';
|
import {type IARSubmission, ReportStatus} from '@app/api/report/ReportModels';
|
||||||
import {getReportSearchService} from '@app/api/SearchFactory';
|
import {getReportSearchService} from '@app/api/SearchFactory';
|
||||||
|
import {EnforcementDecisionEvidence} from '@app/api/Tables';
|
||||||
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
|
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
|
||||||
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
|
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
|
||||||
import {ms} from 'itty-time';
|
import {ms} from 'itty-time';
|
||||||
|
|
||||||
|
export const REPORT_COMPLAINT_WINDOW_DAYS = 184;
|
||||||
|
export const CHILD_SAFETY_REPORT_RETENTION_DAYS = 365;
|
||||||
|
const CHILD_SAFETY_CATEGORY = 'child_safety';
|
||||||
|
const DECISION_EVIDENCE_RETENTION_MS = EnforcementDecisionEvidence.defaultTtlSeconds! * 1000;
|
||||||
|
const DEFAULT_SCAN_PAGE_SIZE = 500;
|
||||||
|
|
||||||
export interface ReportRetentionPolicy {
|
export interface ReportRetentionPolicy {
|
||||||
retentionDays: number;
|
retentionDays: number;
|
||||||
resolvedRetentionDays: number | null;
|
}
|
||||||
|
|
||||||
|
export interface ReportRetentionDeps extends ReportDeletionDeps {
|
||||||
|
decisionRepository: Pick<EnforcementDecisionRepository, 'listReportLinks'>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ReportRetentionOptions {
|
export interface ReportRetentionOptions {
|
||||||
@@ -31,65 +44,136 @@ export interface ReportRetentionOptions {
|
|||||||
export interface ReportRetentionSummary {
|
export interface ReportRetentionSummary {
|
||||||
dryRun: boolean;
|
dryRun: boolean;
|
||||||
scanned: number;
|
scanned: number;
|
||||||
|
pending: number;
|
||||||
expired: number;
|
expired: number;
|
||||||
|
contextExpired: number;
|
||||||
held: number;
|
held: number;
|
||||||
deleted: number;
|
deleted: number;
|
||||||
|
contextCleared: number;
|
||||||
objectsDeleted: number;
|
objectsDeleted: number;
|
||||||
sharedObjectsKept: number;
|
sharedObjectsKept: number;
|
||||||
failed: number;
|
failed: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
type ReportRetentionState = 'expired' | 'held' | 'retained';
|
export interface ReportRetentionDates {
|
||||||
|
deleteAt: number | null;
|
||||||
function reportExpiresAt(report: IARSubmission, policy: ReportRetentionPolicy): number {
|
contextExpiresAt: number | null;
|
||||||
const reportedAt = report.reportedAt instanceof Date ? report.reportedAt.getTime() : Number.NaN;
|
|
||||||
const byAge = reportedAt + policy.retentionDays * ms('1 day');
|
|
||||||
if (policy.resolvedRetentionDays === null || report.status !== ReportStatus.RESOLVED) {
|
|
||||||
return byAge;
|
|
||||||
}
|
|
||||||
const resolvedAt = report.resolvedAt instanceof Date ? report.resolvedAt.getTime() : Number.NaN;
|
|
||||||
if (Number.isNaN(resolvedAt)) {
|
|
||||||
return byAge;
|
|
||||||
}
|
|
||||||
return Math.min(byAge, resolvedAt + policy.resolvedRetentionDays * ms('1 day'));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function reportRetentionState(report: IARSubmission, now: Date, policy: ReportRetentionPolicy): ReportRetentionState {
|
type ReportRetentionState = 'expired' | 'context_expired' | 'held' | 'retained';
|
||||||
const expiresAt = reportExpiresAt(report, policy);
|
|
||||||
if (Number.isNaN(expiresAt) || expiresAt > now.getTime()) {
|
const KEEP_UNTIL_RESOLVED: ReportRetentionDates = {deleteAt: null, contextExpiresAt: null};
|
||||||
|
|
||||||
|
export function reportRetentionDates(
|
||||||
|
report: IARSubmission,
|
||||||
|
policy: ReportRetentionPolicy,
|
||||||
|
decidedAt: Date | null,
|
||||||
|
): ReportRetentionDates {
|
||||||
|
if (report.status !== ReportStatus.RESOLVED) {
|
||||||
|
return KEEP_UNTIL_RESOLVED;
|
||||||
|
}
|
||||||
|
const resolvedAt = (report.resolvedAt ?? report.reportedAt).getTime();
|
||||||
|
if (Number.isNaN(resolvedAt)) {
|
||||||
|
return KEEP_UNTIL_RESOLVED;
|
||||||
|
}
|
||||||
|
const childSafety = report.category === CHILD_SAFETY_CATEGORY;
|
||||||
|
const ends = [resolvedAt + policy.retentionDays * ms('1 day')];
|
||||||
|
if (childSafety) {
|
||||||
|
ends.push(resolvedAt + CHILD_SAFETY_REPORT_RETENTION_DAYS * ms('1 day'));
|
||||||
|
}
|
||||||
|
if (decidedAt) {
|
||||||
|
ends.push(decidedAt.getTime() + DECISION_EVIDENCE_RETENTION_MS);
|
||||||
|
}
|
||||||
|
const dropsContextEarly = report.reporterOutcome === 'no_violation' && !childSafety && !decidedAt;
|
||||||
|
return {
|
||||||
|
deleteAt: Math.max(...ends),
|
||||||
|
contextExpiresAt: dropsContextEarly ? resolvedAt + REPORT_COMPLAINT_WINDOW_DAYS * ms('1 day') : null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasCopiedContext(report: IARSubmission): boolean {
|
||||||
|
return (
|
||||||
|
(report.messageContext?.length ?? 0) > 0 ||
|
||||||
|
report.reportedProfileSnapshot !== null ||
|
||||||
|
report.additionalInfo !== null
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function reportRetentionState(
|
||||||
|
report: IARSubmission,
|
||||||
|
now: Date,
|
||||||
|
policy: ReportRetentionPolicy,
|
||||||
|
decidedAt: Date | null,
|
||||||
|
): ReportRetentionState {
|
||||||
|
const {deleteAt, contextExpiresAt} = reportRetentionDates(report, policy, decidedAt);
|
||||||
|
const due =
|
||||||
|
deleteAt !== null && deleteAt <= now.getTime()
|
||||||
|
? 'expired'
|
||||||
|
: contextExpiresAt !== null && contextExpiresAt <= now.getTime() && hasCopiedContext(report)
|
||||||
|
? 'context_expired'
|
||||||
|
: null;
|
||||||
|
if (due === null) {
|
||||||
return 'retained';
|
return 'retained';
|
||||||
}
|
}
|
||||||
if (isReportUnderLegalHold(report, now)) {
|
return isReportUnderLegalHold(report, now) ? 'held' : due;
|
||||||
return 'held';
|
}
|
||||||
|
|
||||||
|
async function latestDecisionByReport(
|
||||||
|
decisionRepository: ReportRetentionDeps['decisionRepository'],
|
||||||
|
pageSize: number,
|
||||||
|
): Promise<Map<bigint, Date>> {
|
||||||
|
const decidedAt = new Map<bigint, Date>();
|
||||||
|
let cursor: bigint | undefined;
|
||||||
|
while (true) {
|
||||||
|
const page = await decisionRepository.listReportLinks(pageSize, cursor);
|
||||||
|
for (const decision of page) {
|
||||||
|
for (const reportId of decision.report_ids ?? []) {
|
||||||
|
const known = decidedAt.get(reportId);
|
||||||
|
if (!known || known.getTime() < decision.created_at.getTime()) {
|
||||||
|
decidedAt.set(reportId, decision.created_at);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (page.length < pageSize) return decidedAt;
|
||||||
|
cursor = page[page.length - 1]!.decision_id;
|
||||||
}
|
}
|
||||||
return 'expired';
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function processReportRetention(
|
export async function processReportRetention(
|
||||||
deps: ReportDeletionDeps,
|
deps: ReportRetentionDeps,
|
||||||
options: ReportRetentionOptions,
|
options: ReportRetentionOptions,
|
||||||
): Promise<ReportRetentionSummary> {
|
): Promise<ReportRetentionSummary> {
|
||||||
const {reportRepository} = deps;
|
const {reportRepository} = deps;
|
||||||
const {now, dryRun, policy} = options;
|
const {now, dryRun, policy} = options;
|
||||||
|
const pageSize = options.pageSize ?? DEFAULT_SCAN_PAGE_SIZE;
|
||||||
const summary: ReportRetentionSummary = {
|
const summary: ReportRetentionSummary = {
|
||||||
dryRun,
|
dryRun,
|
||||||
scanned: 0,
|
scanned: 0,
|
||||||
|
pending: 0,
|
||||||
expired: 0,
|
expired: 0,
|
||||||
|
contextExpired: 0,
|
||||||
held: 0,
|
held: 0,
|
||||||
deleted: 0,
|
deleted: 0,
|
||||||
|
contextCleared: 0,
|
||||||
objectsDeleted: 0,
|
objectsDeleted: 0,
|
||||||
sharedObjectsKept: 0,
|
sharedObjectsKept: 0,
|
||||||
failed: 0,
|
failed: 0,
|
||||||
};
|
};
|
||||||
|
const decidedAt = await latestDecisionByReport(deps.decisionRepository, pageSize);
|
||||||
|
const stateOf = (report: IARSubmission) =>
|
||||||
|
reportRetentionState(report, now, policy, decidedAt.get(report.reportId) ?? null);
|
||||||
const retainedKeys = new Set<string>();
|
const retainedKeys = new Set<string>();
|
||||||
const expiredIds: Array<ReportID> = [];
|
const due: Array<{reportId: ReportID; state: 'expired' | 'context_expired'}> = [];
|
||||||
await forEachStoredReport(
|
await forEachStoredReport(
|
||||||
reportRepository,
|
reportRepository,
|
||||||
(report) => {
|
(report) => {
|
||||||
summary.scanned++;
|
summary.scanned++;
|
||||||
const state = reportRetentionState(report, now, policy);
|
if (report.status !== ReportStatus.RESOLVED) {
|
||||||
if (state === 'expired') {
|
summary.pending++;
|
||||||
expiredIds.push(report.reportId);
|
}
|
||||||
|
const state = stateOf(report);
|
||||||
|
if (state === 'expired' || state === 'context_expired') {
|
||||||
|
due.push({reportId: report.reportId, state});
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (state === 'held') {
|
if (state === 'held') {
|
||||||
@@ -99,35 +183,37 @@ export async function processReportRetention(
|
|||||||
retainedKeys.add(key);
|
retainedKeys.add(key);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
options.pageSize,
|
pageSize,
|
||||||
);
|
);
|
||||||
summary.expired = expiredIds.length;
|
summary.expired = due.filter((entry) => entry.state === 'expired').length;
|
||||||
const ledger = {retainedKeys, deletedKeys: new Set<string>()};
|
summary.contextExpired = due.length - summary.expired;
|
||||||
for (const reportId of expiredIds) {
|
const ledger: ReportEvidenceLedger = {retainedKeys, deletedKeys: new Set<string>()};
|
||||||
|
for (const {reportId, state} of due) {
|
||||||
try {
|
try {
|
||||||
const report = await reportRepository.getReport(reportId);
|
const report = await reportRepository.getReport(reportId);
|
||||||
if (!report) continue;
|
if (!report) continue;
|
||||||
if (reportRetentionState(report, now, policy) !== 'expired') {
|
if (stateOf(report) !== state) {
|
||||||
for (const key of referencedObjectKeys(report)) {
|
for (const key of referencedObjectKeys(report)) {
|
||||||
retainedKeys.add(key);
|
retainedKeys.add(key);
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
const result = await deleteReportWithEvidence(deps, report, ledger, {now, dryRun});
|
const apply = state === 'expired' ? deleteReportWithEvidence : clearCopiedContextWithEvidence;
|
||||||
|
const result = await apply(deps, report, ledger, {now, dryRun});
|
||||||
summary.objectsDeleted += result.objectsDeleted;
|
summary.objectsDeleted += result.objectsDeleted;
|
||||||
summary.sharedObjectsKept += result.sharedObjectsKept;
|
summary.sharedObjectsKept += result.sharedObjectsKept;
|
||||||
summary.deleted++;
|
if (state === 'expired') {
|
||||||
|
summary.deleted++;
|
||||||
|
} else {
|
||||||
|
summary.contextCleared++;
|
||||||
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
summary.failed++;
|
summary.failed++;
|
||||||
Logger.error({error, reportId: reportId.toString()}, 'Failed to delete an expired report');
|
Logger.error({error, reportId: reportId.toString(), state}, 'Failed to apply report retention');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Logger.info(
|
Logger.info(
|
||||||
{
|
{...summary, retentionDays: policy.retentionDays, complaintWindowDays: REPORT_COMPLAINT_WINDOW_DAYS},
|
||||||
...summary,
|
|
||||||
retentionDays: policy.retentionDays,
|
|
||||||
resolvedRetentionDays: policy.resolvedRetentionDays,
|
|
||||||
},
|
|
||||||
'Processed report retention',
|
'Processed report retention',
|
||||||
);
|
);
|
||||||
return summary;
|
return summary;
|
||||||
@@ -135,10 +221,15 @@ export async function processReportRetention(
|
|||||||
|
|
||||||
const expireReportSnapshots: WorkerTaskHandler = async () => {
|
const expireReportSnapshots: WorkerTaskHandler = async () => {
|
||||||
const {reportRepository, storageService} = getWorkerDependencies();
|
const {reportRepository, storageService} = getWorkerDependencies();
|
||||||
const {days, resolvedDays, dryRun} = Config.reportRetention;
|
const {days, dryRun} = Config.reportRetention;
|
||||||
await processReportRetention(
|
await processReportRetention(
|
||||||
{reportRepository, storageService, reportSearchService: getReportSearchService()},
|
{
|
||||||
{now: new Date(), dryRun, policy: {retentionDays: days, resolvedRetentionDays: resolvedDays}},
|
reportRepository,
|
||||||
|
storageService,
|
||||||
|
reportSearchService: getReportSearchService(),
|
||||||
|
decisionRepository: new EnforcementDecisionRepository(),
|
||||||
|
},
|
||||||
|
{now: new Date(), dryRun, policy: {retentionDays: days}},
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -3,13 +3,14 @@
|
|||||||
import {createAttachmentID, createChannelID, createReportID, createUserID, type ReportID} from '@app/api/BrandedTypes';
|
import {createAttachmentID, createChannelID, createReportID, createUserID, type ReportID} from '@app/api/BrandedTypes';
|
||||||
import {Config} from '@app/api/Config';
|
import {Config} from '@app/api/Config';
|
||||||
import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';
|
import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';
|
||||||
|
import {ENFORCEMENT_DECISION_COLUMNS, type EnforcementDecisionRow} from '@app/api/database/types/EnforcementTypes';
|
||||||
import type {MessageAttachment} from '@app/api/database/types/MessageTypes';
|
import type {MessageAttachment} from '@app/api/database/types/MessageTypes';
|
||||||
import {
|
import {
|
||||||
IAR_SUBMISSION_COLUMNS,
|
IAR_SUBMISSION_COLUMNS,
|
||||||
type IARMessageContextRow,
|
type IARMessageContextRow,
|
||||||
type IARSubmissionRow,
|
type IARSubmissionRow,
|
||||||
} from '@app/api/database/types/ReportTypes';
|
} from '@app/api/database/types/ReportTypes';
|
||||||
import type {ReportDeletionDeps} from '@app/api/report/ReportDeletion';
|
import {EnforcementDecisionRepository} from '@app/api/enforcement/EnforcementDecisionRepository';
|
||||||
import type {IARSubmission} from '@app/api/report/ReportModels';
|
import type {IARSubmission} from '@app/api/report/ReportModels';
|
||||||
import {ReportRepository} from '@app/api/report/ReportRepository';
|
import {ReportRepository} from '@app/api/report/ReportRepository';
|
||||||
import type {IReportSearchService} from '@app/api/search/IReportSearchService';
|
import type {IReportSearchService} from '@app/api/search/IReportSearchService';
|
||||||
@@ -17,11 +18,17 @@ import {MockStorageService} from '@app/api/test/mocks/MockStorageService';
|
|||||||
import {NoopLogger} from '@app/api/test/mocks/NoopLogger';
|
import {NoopLogger} from '@app/api/test/mocks/NoopLogger';
|
||||||
import {InMemorySearchProvider} from '@app/api/test/search/InMemorySearchProvider';
|
import {InMemorySearchProvider} from '@app/api/test/search/InMemorySearchProvider';
|
||||||
import expireReportSnapshots, {
|
import expireReportSnapshots, {
|
||||||
|
CHILD_SAFETY_REPORT_RETENTION_DAYS,
|
||||||
processReportRetention,
|
processReportRetention,
|
||||||
|
REPORT_COMPLAINT_WINDOW_DAYS,
|
||||||
|
type ReportRetentionDeps,
|
||||||
type ReportRetentionOptions,
|
type ReportRetentionOptions,
|
||||||
type ReportRetentionPolicy,
|
type ReportRetentionPolicy,
|
||||||
|
reportRetentionDates,
|
||||||
} from '@app/api/worker/tasks/ExpireReportSnapshots';
|
} from '@app/api/worker/tasks/ExpireReportSnapshots';
|
||||||
import {clearWorkerDependencies, setWorkerDependenciesForTest} from '@app/api/worker/WorkerContext';
|
import {clearWorkerDependencies, setWorkerDependenciesForTest} from '@app/api/worker/WorkerContext';
|
||||||
|
import {DeletionReasons} from '@fluxer/constants/src/Core';
|
||||||
|
import type {ReportItemOutcome} from '@fluxer/schema/src/domains/admin/AdminSchemas';
|
||||||
import {serializeReportProfileSnapshot} from '@fluxer/schema/src/domains/report/ReportProfileSnapshotSchemas';
|
import {serializeReportProfileSnapshot} from '@fluxer/schema/src/domains/report/ReportProfileSnapshotSchemas';
|
||||||
import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
|
import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
|
||||||
import {ms} from 'itty-time';
|
import {ms} from 'itty-time';
|
||||||
@@ -31,7 +38,7 @@ const NOW = new Date('2027-06-01T12:00:00.000Z');
|
|||||||
const DAY_MS = ms('1 day');
|
const DAY_MS = ms('1 day');
|
||||||
const REPORTS_BUCKET = Config.s3.buckets.reports;
|
const REPORTS_BUCKET = Config.s3.buckets.reports;
|
||||||
const WORKER_HELPERS = {logger: new NoopLogger()} as unknown as WorkerTaskHelpers;
|
const WORKER_HELPERS = {logger: new NoopLogger()} as unknown as WorkerTaskHelpers;
|
||||||
const DEFAULT_POLICY: ReportRetentionPolicy = {retentionDays: 365, resolvedRetentionDays: null};
|
const DEFAULT_POLICY: ReportRetentionPolicy = {retentionDays: 365};
|
||||||
let sequence = 1_480_000_000_000_000_000n;
|
let sequence = 1_480_000_000_000_000_000n;
|
||||||
|
|
||||||
function nextId(): bigint {
|
function nextId(): bigint {
|
||||||
@@ -93,12 +100,16 @@ interface SharedAttachment {
|
|||||||
}
|
}
|
||||||
|
|
||||||
interface SeedOptions {
|
interface SeedOptions {
|
||||||
ageDays: number;
|
ageDays?: number;
|
||||||
|
resolvedDaysAgo?: number;
|
||||||
attachment?: SharedAttachment;
|
attachment?: SharedAttachment;
|
||||||
profile?: boolean;
|
profile?: boolean;
|
||||||
legalHoldUntil?: Date | null;
|
legalHoldUntil?: Date | null;
|
||||||
status?: number;
|
status?: number;
|
||||||
resolvedDaysAgo?: number;
|
category?: string;
|
||||||
|
outcome?: ReportItemOutcome | null;
|
||||||
|
additionalInfo?: string | null;
|
||||||
|
reporterId?: bigint | null;
|
||||||
snapshotJson?: string;
|
snapshotJson?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -116,20 +127,96 @@ function attachmentKey(attachment: SharedAttachment): string {
|
|||||||
return makeAttachmentCdnKey(createChannelID(attachment.channelId), attachment.attachmentId, attachment.filename);
|
return makeAttachmentCdnKey(createChannelID(attachment.channelId), attachment.attachmentId, attachment.filename);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function storedReport(overrides: Partial<IARSubmission>): IARSubmission {
|
||||||
|
return {
|
||||||
|
reportId: createReportID(nextId()),
|
||||||
|
reportedAt: daysAgo(500),
|
||||||
|
status: 1,
|
||||||
|
resolvedAt: daysAgo(10),
|
||||||
|
category: 'harassment',
|
||||||
|
reporterOutcome: 'removed',
|
||||||
|
...overrides,
|
||||||
|
} as IARSubmission;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('report retention dates', () => {
|
||||||
|
const day = DAY_MS;
|
||||||
|
|
||||||
|
it('keeps a pending report until it is resolved', () => {
|
||||||
|
expect(reportRetentionDates(storedReport({status: 0, resolvedAt: null}), DEFAULT_POLICY, null)).toEqual({
|
||||||
|
deleteAt: null,
|
||||||
|
contextExpiresAt: null,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('deletes an actioned report one year after it was resolved and keeps its context until then', () => {
|
||||||
|
const report = storedReport({});
|
||||||
|
expect(reportRetentionDates(report, DEFAULT_POLICY, null)).toEqual({
|
||||||
|
deleteAt: report.resolvedAt!.getTime() + 365 * day,
|
||||||
|
contextExpiresAt: null,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('drops the context of a no violation report once the complaint window has passed', () => {
|
||||||
|
const report = storedReport({reporterOutcome: 'no_violation'});
|
||||||
|
const dates = reportRetentionDates(report, DEFAULT_POLICY, null);
|
||||||
|
expect(dates.contextExpiresAt).toBe(report.resolvedAt!.getTime() + REPORT_COMPLAINT_WINDOW_DAYS * day);
|
||||||
|
expect(REPORT_COMPLAINT_WINDOW_DAYS).toBeGreaterThanOrEqual(183);
|
||||||
|
expect(dates.deleteAt).toBe(report.resolvedAt!.getTime() + 365 * day);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps a child safety report and its context for at least a year whatever the configured period', () => {
|
||||||
|
const report = storedReport({category: 'child_safety', reporterOutcome: 'no_violation'});
|
||||||
|
expect(reportRetentionDates(report, {retentionDays: 30}, null)).toEqual({
|
||||||
|
deleteAt: report.resolvedAt!.getTime() + CHILD_SAFETY_REPORT_RETENTION_DAYS * day,
|
||||||
|
contextExpiresAt: null,
|
||||||
|
});
|
||||||
|
expect(CHILD_SAFETY_REPORT_RETENTION_DAYS).toBeGreaterThanOrEqual(365);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps a report linked to a decision as long as the decision evidence and keeps its context', () => {
|
||||||
|
const report = storedReport({resolvedAt: daysAgo(300), reporterOutcome: 'no_violation'});
|
||||||
|
const decidedAt = daysAgo(100);
|
||||||
|
expect(reportRetentionDates(report, DEFAULT_POLICY, decidedAt)).toEqual({
|
||||||
|
deleteAt: decidedAt.getTime() + 365 * day,
|
||||||
|
contextExpiresAt: null,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses the longer period when the configured one is longer than a floor', () => {
|
||||||
|
const report = storedReport({category: 'child_safety'});
|
||||||
|
expect(reportRetentionDates(report, {retentionDays: 730}, null).deleteAt).toBe(
|
||||||
|
report.resolvedAt!.getTime() + 730 * day,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('counts from the filing time for a resolved report without a resolution time', () => {
|
||||||
|
const report = storedReport({resolvedAt: null});
|
||||||
|
expect(reportRetentionDates(report, DEFAULT_POLICY, null).deleteAt).toBe(report.reportedAt.getTime() + 365 * day);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('report retention', () => {
|
describe('report retention', () => {
|
||||||
let repository: ReportRepository;
|
let repository: ReportRepository;
|
||||||
|
let decisions: EnforcementDecisionRepository;
|
||||||
let storage: MockStorageService;
|
let storage: MockStorageService;
|
||||||
let search: IReportSearchService;
|
let search: IReportSearchService;
|
||||||
let deps: ReportDeletionDeps;
|
let deps: ReportRetentionDeps;
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
repository = new ReportRepository();
|
repository = new ReportRepository();
|
||||||
|
decisions = new EnforcementDecisionRepository();
|
||||||
storage = new MockStorageService();
|
storage = new MockStorageService();
|
||||||
search = new InMemorySearchProvider().getReportSearchService();
|
search = new InMemorySearchProvider().getReportSearchService();
|
||||||
deps = {reportRepository: repository, storageService: storage, reportSearchService: search};
|
deps = {
|
||||||
|
reportRepository: repository,
|
||||||
|
storageService: storage,
|
||||||
|
reportSearchService: search,
|
||||||
|
decisionRepository: decisions,
|
||||||
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
async function seed(options: SeedOptions): Promise<SeededReport> {
|
async function seed(options: SeedOptions = {}): Promise<SeededReport> {
|
||||||
const id = nextId();
|
const id = nextId();
|
||||||
const reportId = createReportID(id);
|
const reportId = createReportID(id);
|
||||||
let storedAttachmentKey: string | null = null;
|
let storedAttachmentKey: string | null = null;
|
||||||
@@ -161,14 +248,18 @@ describe('report retention', () => {
|
|||||||
guild: null,
|
guild: null,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
const resolved = options.resolvedDaysAgo !== undefined;
|
||||||
const row = {
|
const row = {
|
||||||
...Object.fromEntries(IAR_SUBMISSION_COLUMNS.map((column) => [column, null])),
|
...Object.fromEntries(IAR_SUBMISSION_COLUMNS.map((column) => [column, null])),
|
||||||
report_id: id,
|
report_id: id,
|
||||||
reported_at: daysAgo(options.ageDays),
|
reporter_id: options.reporterId ?? null,
|
||||||
status: options.status ?? (options.resolvedDaysAgo === undefined ? 0 : 1),
|
reported_at: daysAgo(options.ageDays ?? (options.resolvedDaysAgo ?? 0) + 5),
|
||||||
resolved_at: options.resolvedDaysAgo === undefined ? null : daysAgo(options.resolvedDaysAgo),
|
status: options.status ?? (resolved ? 1 : 0),
|
||||||
|
resolved_at: resolved ? daysAgo(options.resolvedDaysAgo!) : null,
|
||||||
|
reporter_outcome: options.outcome === undefined ? (resolved ? 'removed' : null) : options.outcome,
|
||||||
report_type: 0,
|
report_type: 0,
|
||||||
category: 'other',
|
category: options.category ?? 'other',
|
||||||
|
additional_info: options.additionalInfo ?? null,
|
||||||
reported_channel_id: options.attachment?.channelId ?? null,
|
reported_channel_id: options.attachment?.channelId ?? null,
|
||||||
message_context: messageContext,
|
message_context: messageContext,
|
||||||
reported_profile_snapshot: snapshot,
|
reported_profile_snapshot: snapshot,
|
||||||
@@ -180,6 +271,30 @@ describe('report retention', () => {
|
|||||||
return {reportId, attachmentKey: storedAttachmentKey, profileKey};
|
return {reportId, attachmentKey: storedAttachmentKey, profileKey};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function linkDecision(reportIds: Array<ReportID>, createdAt: Date): Promise<void> {
|
||||||
|
const decisionId = nextId();
|
||||||
|
const decision = {
|
||||||
|
...Object.fromEntries(ENFORCEMENT_DECISION_COLUMNS.map((column) => [column, null])),
|
||||||
|
decision_id: decisionId,
|
||||||
|
subject_type: 'user',
|
||||||
|
subject_id: nextId(),
|
||||||
|
kind: 'warning',
|
||||||
|
reason_code: DeletionReasons.HARASSMENT_OR_BULLYING,
|
||||||
|
source: 'report',
|
||||||
|
automation: 'none',
|
||||||
|
report_ids: new Set<bigint>(reportIds),
|
||||||
|
decided_by: nextId(),
|
||||||
|
created_at: createdAt,
|
||||||
|
status: 'active',
|
||||||
|
notice_status: 'sent',
|
||||||
|
} as EnforcementDecisionRow;
|
||||||
|
await decisions.create({
|
||||||
|
decision,
|
||||||
|
evidence: null,
|
||||||
|
firstNote: {decision_id: decisionId, note_id: nextId(), author_id: nextId(), body: 'note', created_at: createdAt},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
async function indexedIds(): Promise<Array<string>> {
|
async function indexedIds(): Promise<Array<string>> {
|
||||||
const result = await search.searchReports('', {}, {limit: 1000});
|
const result = await search.searchReports('', {}, {limit: 1000});
|
||||||
return result.hits.map((hit) => hit.id).sort();
|
return result.hits.map((hit) => hit.id).sort();
|
||||||
@@ -199,11 +314,27 @@ describe('report retention', () => {
|
|||||||
if (seeded.profileKey) expect(storage.hasObject(REPORTS_BUCKET, seeded.profileKey)).toBe(false);
|
if (seeded.profileKey) expect(storage.hasObject(REPORTS_BUCKET, seeded.profileKey)).toBe(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
it('deletes a report 365 days after it was filed with its objects, search document and row', async () => {
|
async function expectContextCleared(seeded: SeededReport): Promise<void> {
|
||||||
const expired = await seed({ageDays: 365, attachment: newAttachment(), profile: true});
|
const report = await repository.getReport(seeded.reportId);
|
||||||
const kept = await seed({ageDays: 364, attachment: newAttachment(), profile: true});
|
expect(report).not.toBeNull();
|
||||||
|
expect(report?.messageContext).toBeNull();
|
||||||
|
expect(report?.reportedProfileSnapshot).toBeNull();
|
||||||
|
expect(report?.additionalInfo).toBeNull();
|
||||||
|
expect(await indexedIds()).toContain(seeded.reportId.toString());
|
||||||
|
for (const key of [seeded.attachmentKey, seeded.profileKey]) {
|
||||||
|
if (key) expect(storage.hasObject(REPORTS_BUCKET, key)).toBe(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
function run(options: Partial<ReportRetentionOptions> = {}) {
|
||||||
|
return processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY, ...options});
|
||||||
|
}
|
||||||
|
|
||||||
|
it('deletes a report 365 days after it was resolved with its objects, search document and row', async () => {
|
||||||
|
const expired = await seed({resolvedDaysAgo: 365, attachment: newAttachment(), profile: true});
|
||||||
|
const kept = await seed({resolvedDaysAgo: 364, attachment: newAttachment(), profile: true});
|
||||||
|
|
||||||
|
const summary = await run();
|
||||||
|
|
||||||
await expectGone(expired);
|
await expectGone(expired);
|
||||||
expect(storage.hasObject(REPORTS_BUCKET, expired.attachmentKey!)).toBe(false);
|
expect(storage.hasObject(REPORTS_BUCKET, expired.attachmentKey!)).toBe(false);
|
||||||
@@ -211,31 +342,175 @@ describe('report retention', () => {
|
|||||||
expect(summary).toEqual({
|
expect(summary).toEqual({
|
||||||
dryRun: false,
|
dryRun: false,
|
||||||
scanned: 2,
|
scanned: 2,
|
||||||
|
pending: 0,
|
||||||
expired: 1,
|
expired: 1,
|
||||||
|
contextExpired: 0,
|
||||||
held: 0,
|
held: 0,
|
||||||
deleted: 1,
|
deleted: 1,
|
||||||
|
contextCleared: 0,
|
||||||
objectsDeleted: 2,
|
objectsDeleted: 2,
|
||||||
sharedObjectsKept: 0,
|
sharedObjectsKept: 0,
|
||||||
failed: 0,
|
failed: 0,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('deletes expired reports whatever their status', async () => {
|
it('keeps a pending report however old it is', async () => {
|
||||||
const pending = await seed({ageDays: 400, status: 0});
|
const pending = await seed({ageDays: 4000, attachment: newAttachment(), profile: true});
|
||||||
const resolved = await seed({ageDays: 400, status: 1});
|
|
||||||
|
|
||||||
await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
const summary = await run({policy: {retentionDays: 1}});
|
||||||
|
|
||||||
await expectGone(pending);
|
await expectKept(pending);
|
||||||
await expectGone(resolved);
|
expect(summary).toMatchObject({scanned: 1, pending: 1, expired: 0, deleted: 0, objectsDeleted: 0});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('drops the copied context of a no violation report after the complaint window and keeps the record', async () => {
|
||||||
|
const outsideWindow = await seed({
|
||||||
|
resolvedDaysAgo: REPORT_COMPLAINT_WINDOW_DAYS,
|
||||||
|
outcome: 'no_violation',
|
||||||
|
attachment: newAttachment(),
|
||||||
|
profile: true,
|
||||||
|
additionalInfo: 'They keep messaging my friends',
|
||||||
|
});
|
||||||
|
const insideWindow = await seed({
|
||||||
|
resolvedDaysAgo: REPORT_COMPLAINT_WINDOW_DAYS - 1,
|
||||||
|
outcome: 'no_violation',
|
||||||
|
attachment: newAttachment(),
|
||||||
|
profile: true,
|
||||||
|
additionalInfo: 'Still open to a complaint',
|
||||||
|
});
|
||||||
|
const actioned = await seed({
|
||||||
|
resolvedDaysAgo: REPORT_COMPLAINT_WINDOW_DAYS + 10,
|
||||||
|
attachment: newAttachment(),
|
||||||
|
profile: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const summary = await run();
|
||||||
|
|
||||||
|
await expectContextCleared(outsideWindow);
|
||||||
|
const cleared = await repository.getReport(outsideWindow.reportId);
|
||||||
|
expect(cleared).toMatchObject({status: 1, category: 'other', reporterOutcome: 'no_violation'});
|
||||||
|
expect((await search.searchReports('friends', {}, {limit: 10})).hits).toHaveLength(0);
|
||||||
|
await expectKept(insideWindow);
|
||||||
|
expect((await repository.getReport(insideWindow.reportId))?.additionalInfo).toBe('Still open to a complaint');
|
||||||
|
await expectKept(actioned);
|
||||||
|
expect(summary).toMatchObject({
|
||||||
|
scanned: 3,
|
||||||
|
expired: 0,
|
||||||
|
contextExpired: 1,
|
||||||
|
contextCleared: 1,
|
||||||
|
deleted: 0,
|
||||||
|
objectsDeleted: 2,
|
||||||
|
failed: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
const again = await run();
|
||||||
|
expect(again).toMatchObject({contextExpired: 0, contextCleared: 0, objectsDeleted: 0});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('deletes a no violation report one year after it was resolved', async () => {
|
||||||
|
const expired = await seed({resolvedDaysAgo: 365, outcome: 'no_violation', profile: true});
|
||||||
|
|
||||||
|
const summary = await run();
|
||||||
|
|
||||||
|
await expectGone(expired);
|
||||||
|
expect(summary).toMatchObject({expired: 1, deleted: 1, contextExpired: 0});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps an evidence copy that an actioned report shares with a no violation report', async () => {
|
||||||
|
const shared = newAttachment();
|
||||||
|
const noViolation = await seed({resolvedDaysAgo: 200, outcome: 'no_violation', attachment: shared});
|
||||||
|
const actioned = await seed({resolvedDaysAgo: 200, attachment: shared});
|
||||||
|
|
||||||
|
const summary = await run();
|
||||||
|
|
||||||
|
expect((await repository.getReport(noViolation.reportId))?.messageContext).toBeNull();
|
||||||
|
await expectKept(actioned);
|
||||||
|
expect(summary).toMatchObject({contextCleared: 1, sharedObjectsKept: 1, objectsDeleted: 0});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps a child safety report and its context for a year even with a shorter period or no violation', async () => {
|
||||||
|
const childSafety = await seed({
|
||||||
|
resolvedDaysAgo: 364,
|
||||||
|
category: 'child_safety',
|
||||||
|
outcome: 'no_violation',
|
||||||
|
attachment: newAttachment(),
|
||||||
|
profile: true,
|
||||||
|
additionalInfo: 'Context for a CyberTip',
|
||||||
|
});
|
||||||
|
const childSafetyAfterYear = await seed({resolvedDaysAgo: 365, category: 'child_safety', profile: true});
|
||||||
|
const ordinary = await seed({resolvedDaysAgo: 40, profile: true});
|
||||||
|
|
||||||
|
const summary = await run({policy: {retentionDays: 30}});
|
||||||
|
|
||||||
|
await expectKept(childSafety);
|
||||||
|
expect((await repository.getReport(childSafety.reportId))?.additionalInfo).toBe('Context for a CyberTip');
|
||||||
|
await expectGone(childSafetyAfterYear);
|
||||||
|
await expectGone(ordinary);
|
||||||
|
expect(summary).toMatchObject({expired: 2, contextExpired: 0, deleted: 2});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps a child safety report under a legal hold past the year', async () => {
|
||||||
|
const held = await seed({
|
||||||
|
resolvedDaysAgo: 500,
|
||||||
|
category: 'child_safety',
|
||||||
|
profile: true,
|
||||||
|
legalHoldUntil: new Date(NOW.getTime() + 30 * DAY_MS),
|
||||||
|
});
|
||||||
|
|
||||||
|
const summary = await run();
|
||||||
|
|
||||||
|
await expectKept(held);
|
||||||
|
expect(summary).toMatchObject({expired: 0, held: 1, deleted: 0});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps a report linked to a decision until the decision evidence expires', async () => {
|
||||||
|
const linked = await seed({
|
||||||
|
resolvedDaysAgo: 400,
|
||||||
|
outcome: 'no_violation',
|
||||||
|
profile: true,
|
||||||
|
attachment: newAttachment(),
|
||||||
|
});
|
||||||
|
const linkedLongAgo = await seed({resolvedDaysAgo: 800, profile: true});
|
||||||
|
const unlinked = await seed({resolvedDaysAgo: 400, profile: true});
|
||||||
|
await linkDecision([linked.reportId], daysAgo(500));
|
||||||
|
await linkDecision([linked.reportId], daysAgo(100));
|
||||||
|
await linkDecision([linkedLongAgo.reportId], daysAgo(700));
|
||||||
|
|
||||||
|
const summary = await run({pageSize: 1});
|
||||||
|
|
||||||
|
await expectKept(linked);
|
||||||
|
expect((await repository.getReport(linked.reportId))?.messageContext).toHaveLength(1);
|
||||||
|
await expectGone(linkedLongAgo);
|
||||||
|
await expectGone(unlinked);
|
||||||
|
expect(summary).toMatchObject({scanned: 3, expired: 2, contextExpired: 0, deleted: 2});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('deletes nothing when the decision scan cannot finish', async () => {
|
||||||
|
const expired = await seed({resolvedDaysAgo: 400, profile: true});
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
processReportRetention(
|
||||||
|
{
|
||||||
|
...deps,
|
||||||
|
decisionRepository: {
|
||||||
|
listReportLinks: async () => {
|
||||||
|
throw new Error('read timeout');
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
|
||||||
|
),
|
||||||
|
).rejects.toThrow('read timeout');
|
||||||
|
|
||||||
|
await expectKept(expired);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('keeps an object that a retained report still uses', async () => {
|
it('keeps an object that a retained report still uses', async () => {
|
||||||
const shared = newAttachment();
|
const shared = newAttachment();
|
||||||
const expired = await seed({ageDays: 500, attachment: shared});
|
const expired = await seed({resolvedDaysAgo: 500, attachment: shared});
|
||||||
const retained = await seed({ageDays: 10, attachment: shared});
|
const retained = await seed({resolvedDaysAgo: 10, attachment: shared});
|
||||||
|
|
||||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
const summary = await run();
|
||||||
|
|
||||||
await expectGone(expired);
|
await expectGone(expired);
|
||||||
await expectKept(retained);
|
await expectKept(retained);
|
||||||
@@ -246,10 +521,10 @@ describe('report retention', () => {
|
|||||||
|
|
||||||
it('deletes an object shared by two expired reports once', async () => {
|
it('deletes an object shared by two expired reports once', async () => {
|
||||||
const shared = newAttachment();
|
const shared = newAttachment();
|
||||||
const first = await seed({ageDays: 500, attachment: shared});
|
const first = await seed({resolvedDaysAgo: 500, attachment: shared});
|
||||||
const second = await seed({ageDays: 450, attachment: shared});
|
const second = await seed({resolvedDaysAgo: 450, attachment: shared});
|
||||||
|
|
||||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
const summary = await run();
|
||||||
|
|
||||||
await expectGone(first);
|
await expectGone(first);
|
||||||
await expectGone(second);
|
await expectGone(second);
|
||||||
@@ -261,15 +536,15 @@ describe('report retention', () => {
|
|||||||
it('keeps a report under a legal hold that ends later and deletes one whose hold has ended', async () => {
|
it('keeps a report under a legal hold that ends later and deletes one whose hold has ended', async () => {
|
||||||
const shared = newAttachment();
|
const shared = newAttachment();
|
||||||
const held = await seed({
|
const held = await seed({
|
||||||
ageDays: 500,
|
resolvedDaysAgo: 500,
|
||||||
attachment: shared,
|
attachment: shared,
|
||||||
profile: true,
|
profile: true,
|
||||||
legalHoldUntil: new Date(NOW.getTime() + DAY_MS),
|
legalHoldUntil: new Date(NOW.getTime() + DAY_MS),
|
||||||
});
|
});
|
||||||
const holdEnded = await seed({ageDays: 500, profile: true, legalHoldUntil: daysAgo(1)});
|
const holdEnded = await seed({resolvedDaysAgo: 500, profile: true, legalHoldUntil: daysAgo(1)});
|
||||||
const sharingWithHeld = await seed({ageDays: 500, attachment: shared});
|
const sharingWithHeld = await seed({resolvedDaysAgo: 500, attachment: shared});
|
||||||
|
|
||||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
const summary = await run();
|
||||||
|
|
||||||
await expectKept(held);
|
await expectKept(held);
|
||||||
expect((await repository.getReport(held.reportId))?.legalHoldReason).toBe('Preservation request');
|
expect((await repository.getReport(held.reportId))?.legalHoldReason).toBe('Preservation request');
|
||||||
@@ -279,31 +554,61 @@ describe('report retention', () => {
|
|||||||
expect(summary).toMatchObject({scanned: 3, expired: 2, held: 1, deleted: 2, sharedObjectsKept: 1});
|
expect(summary).toMatchObject({scanned: 3, expired: 2, held: 1, deleted: 2, sharedObjectsKept: 1});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('changes nothing in a dry run and reports what it would delete', async () => {
|
it('keeps the context of a no violation report under a legal hold', async () => {
|
||||||
const expired = await seed({ageDays: 366, attachment: newAttachment(), profile: true});
|
const held = await seed({
|
||||||
const kept = await seed({ageDays: 1, profile: true});
|
resolvedDaysAgo: 250,
|
||||||
|
outcome: 'no_violation',
|
||||||
|
attachment: newAttachment(),
|
||||||
|
legalHoldUntil: new Date(NOW.getTime() + DAY_MS),
|
||||||
|
});
|
||||||
|
|
||||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: true, policy: DEFAULT_POLICY});
|
const summary = await run();
|
||||||
|
|
||||||
await expectKept(expired);
|
await expectKept(held);
|
||||||
await expectKept(kept);
|
expect((await repository.getReport(held.reportId))?.messageContext).toHaveLength(1);
|
||||||
|
expect(summary).toMatchObject({held: 1, contextExpired: 0, contextCleared: 0});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('changes nothing in a dry run and counts what it would do to existing reports', async () => {
|
||||||
|
const expired = await seed({resolvedDaysAgo: 366, attachment: newAttachment(), profile: true});
|
||||||
|
const contextDue = await seed({
|
||||||
|
resolvedDaysAgo: 200,
|
||||||
|
outcome: 'no_violation',
|
||||||
|
attachment: newAttachment(),
|
||||||
|
additionalInfo: 'details',
|
||||||
|
});
|
||||||
|
const childSafety = await seed({resolvedDaysAgo: 200, category: 'child_safety', outcome: 'no_violation'});
|
||||||
|
const held = await seed({resolvedDaysAgo: 400, legalHoldUntil: new Date(NOW.getTime() + DAY_MS)});
|
||||||
|
const pending = await seed({profile: true});
|
||||||
|
const linked = await seed({resolvedDaysAgo: 400});
|
||||||
|
await linkDecision([linked.reportId], daysAgo(30));
|
||||||
|
|
||||||
|
const summary = await run({dryRun: true});
|
||||||
|
|
||||||
|
for (const report of [expired, contextDue, childSafety, held, pending, linked]) {
|
||||||
|
await expectKept(report);
|
||||||
|
}
|
||||||
|
expect((await repository.getReport(contextDue.reportId))?.additionalInfo).toBe('details');
|
||||||
expect(storage.deleteObjectSpy).not.toHaveBeenCalled();
|
expect(storage.deleteObjectSpy).not.toHaveBeenCalled();
|
||||||
expect(summary).toEqual({
|
expect(summary).toEqual({
|
||||||
dryRun: true,
|
dryRun: true,
|
||||||
scanned: 2,
|
scanned: 6,
|
||||||
|
pending: 1,
|
||||||
expired: 1,
|
expired: 1,
|
||||||
held: 0,
|
contextExpired: 1,
|
||||||
|
held: 1,
|
||||||
deleted: 1,
|
deleted: 1,
|
||||||
objectsDeleted: 2,
|
contextCleared: 1,
|
||||||
|
objectsDeleted: 3,
|
||||||
sharedObjectsKept: 0,
|
sharedObjectsKept: 0,
|
||||||
failed: 0,
|
failed: 0,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('deletes nothing when the scan cannot finish', async () => {
|
it('deletes nothing when the scan cannot finish', async () => {
|
||||||
const expired = await seed({ageDays: 400, attachment: newAttachment(), profile: true});
|
const expired = await seed({resolvedDaysAgo: 400, attachment: newAttachment(), profile: true});
|
||||||
await seed({ageDays: 400});
|
await seed({resolvedDaysAgo: 400});
|
||||||
await seed({ageDays: 400});
|
await seed({resolvedDaysAgo: 400});
|
||||||
let pages = 0;
|
let pages = 0;
|
||||||
const failingRepository = Object.create(repository) as ReportRepository;
|
const failingRepository = Object.create(repository) as ReportRepository;
|
||||||
failingRepository.listAllReportsPaginated = async (limit: number, lastReportId?: ReportID) => {
|
failingRepository.listAllReportsPaginated = async (limit: number, lastReportId?: ReportID) => {
|
||||||
@@ -315,7 +620,12 @@ describe('report retention', () => {
|
|||||||
await expect(
|
await expect(
|
||||||
processReportRetention(
|
processReportRetention(
|
||||||
{...deps, reportRepository: failingRepository},
|
{...deps, reportRepository: failingRepository},
|
||||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY, pageSize: 2},
|
{
|
||||||
|
now: NOW,
|
||||||
|
dryRun: false,
|
||||||
|
policy: DEFAULT_POLICY,
|
||||||
|
pageSize: 2,
|
||||||
|
},
|
||||||
),
|
),
|
||||||
).rejects.toThrow('read timeout');
|
).rejects.toThrow('read timeout');
|
||||||
|
|
||||||
@@ -328,11 +638,11 @@ describe('report retention', () => {
|
|||||||
const expired: Array<SeededReport> = [];
|
const expired: Array<SeededReport> = [];
|
||||||
const kept: Array<SeededReport> = [];
|
const kept: Array<SeededReport> = [];
|
||||||
for (let index = 0; index < 7; index++) {
|
for (let index = 0; index < 7; index++) {
|
||||||
expired.push(await seed({ageDays: 400 + index, profile: true}));
|
expired.push(await seed({resolvedDaysAgo: 400 + index, profile: true}));
|
||||||
kept.push(await seed({ageDays: index, profile: true}));
|
kept.push(await seed({resolvedDaysAgo: index, profile: true}));
|
||||||
}
|
}
|
||||||
|
|
||||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY, pageSize: 3});
|
const summary = await run({pageSize: 3});
|
||||||
|
|
||||||
expect(summary).toMatchObject({scanned: 14, expired: 7, deleted: 7, objectsDeleted: 7, failed: 0});
|
expect(summary).toMatchObject({scanned: 14, expired: 7, deleted: 7, objectsDeleted: 7, failed: 0});
|
||||||
for (const report of expired) await expectGone(report);
|
for (const report of expired) await expectGone(report);
|
||||||
@@ -340,25 +650,57 @@ describe('report retention', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('keeps the row and search document when an object cannot be deleted, then deletes them on the next run', async () => {
|
it('keeps the row and search document when an object cannot be deleted, then deletes them on the next run', async () => {
|
||||||
const expired = await seed({ageDays: 400, attachment: newAttachment(), profile: true});
|
const expired = await seed({resolvedDaysAgo: 400, attachment: newAttachment(), profile: true});
|
||||||
storage.configure({shouldFailDelete: true});
|
storage.configure({shouldFailDelete: true});
|
||||||
|
|
||||||
const failed = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
const failed = await run();
|
||||||
|
|
||||||
expect(failed).toMatchObject({expired: 1, deleted: 0, failed: 1});
|
expect(failed).toMatchObject({expired: 1, deleted: 0, failed: 1});
|
||||||
expect(await repository.getReport(expired.reportId)).not.toBeNull();
|
expect(await repository.getReport(expired.reportId)).not.toBeNull();
|
||||||
expect(await indexedIds()).toContain(expired.reportId.toString());
|
expect(await indexedIds()).toContain(expired.reportId.toString());
|
||||||
|
|
||||||
storage.configure({shouldFailDelete: false});
|
storage.configure({shouldFailDelete: false});
|
||||||
const retried = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
const retried = await run();
|
||||||
|
|
||||||
expect(retried).toMatchObject({expired: 1, deleted: 1, failed: 0});
|
expect(retried).toMatchObject({expired: 1, deleted: 1, failed: 0});
|
||||||
await expectGone(expired);
|
await expectGone(expired);
|
||||||
expect(storage.hasObject(REPORTS_BUCKET, expired.attachmentKey!)).toBe(false);
|
expect(storage.hasObject(REPORTS_BUCKET, expired.attachmentKey!)).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('keeps the copied context when an object cannot be deleted, then clears it on the next run', async () => {
|
||||||
|
const due = await seed({resolvedDaysAgo: 200, outcome: 'no_violation', attachment: newAttachment()});
|
||||||
|
storage.configure({shouldFailDelete: true});
|
||||||
|
|
||||||
|
expect(await run()).toMatchObject({contextExpired: 1, contextCleared: 0, failed: 1});
|
||||||
|
expect((await repository.getReport(due.reportId))?.messageContext).toHaveLength(1);
|
||||||
|
|
||||||
|
storage.configure({shouldFailDelete: false});
|
||||||
|
expect(await run()).toMatchObject({contextExpired: 1, contextCleared: 1, failed: 0});
|
||||||
|
await expectContextCleared(due);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not bring back a report deleted while its context was being cleared', async () => {
|
||||||
|
const due = await seed({resolvedDaysAgo: 200, outcome: 'no_violation', additionalInfo: 'details'});
|
||||||
|
const deletingRepository = Object.create(repository) as ReportRepository;
|
||||||
|
deletingRepository.clearCopiedContext = async (reportId, reportType) => {
|
||||||
|
await repository.deleteReport(reportId);
|
||||||
|
return repository.clearCopiedContext(reportId, reportType);
|
||||||
|
};
|
||||||
|
|
||||||
|
await processReportRetention(
|
||||||
|
{...deps, reportRepository: deletingRepository},
|
||||||
|
{
|
||||||
|
now: NOW,
|
||||||
|
dryRun: false,
|
||||||
|
policy: DEFAULT_POLICY,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(await repository.getReport(due.reportId)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
it('keeps the row when the search document cannot be deleted', async () => {
|
it('keeps the row when the search document cannot be deleted', async () => {
|
||||||
const expired = await seed({ageDays: 400});
|
const expired = await seed({resolvedDaysAgo: 400});
|
||||||
const failingSearch = Object.create(search) as IReportSearchService;
|
const failingSearch = Object.create(search) as IReportSearchService;
|
||||||
failingSearch.deleteReport = async () => {
|
failingSearch.deleteReport = async () => {
|
||||||
throw new Error('search unavailable');
|
throw new Error('search unavailable');
|
||||||
@@ -366,7 +708,11 @@ describe('report retention', () => {
|
|||||||
|
|
||||||
const summary = await processReportRetention(
|
const summary = await processReportRetention(
|
||||||
{...deps, reportSearchService: failingSearch},
|
{...deps, reportSearchService: failingSearch},
|
||||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
|
{
|
||||||
|
now: NOW,
|
||||||
|
dryRun: false,
|
||||||
|
policy: DEFAULT_POLICY,
|
||||||
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(summary).toMatchObject({deleted: 0, failed: 1});
|
expect(summary).toMatchObject({deleted: 0, failed: 1});
|
||||||
@@ -374,11 +720,15 @@ describe('report retention', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('deletes rows when search is not configured', async () => {
|
it('deletes rows when search is not configured', async () => {
|
||||||
const expired = await seed({ageDays: 400, profile: true});
|
const expired = await seed({resolvedDaysAgo: 400, profile: true});
|
||||||
|
|
||||||
const summary = await processReportRetention(
|
const summary = await processReportRetention(
|
||||||
{...deps, reportSearchService: null},
|
{...deps, reportSearchService: null},
|
||||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
|
{
|
||||||
|
now: NOW,
|
||||||
|
dryRun: false,
|
||||||
|
policy: DEFAULT_POLICY,
|
||||||
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(summary).toMatchObject({deleted: 1, failed: 0});
|
expect(summary).toMatchObject({deleted: 1, failed: 0});
|
||||||
@@ -389,8 +739,8 @@ describe('report retention', () => {
|
|||||||
it('skips a report that was put on hold after the scan', async () => {
|
it('skips a report that was put on hold after the scan', async () => {
|
||||||
const shared = newAttachment();
|
const shared = newAttachment();
|
||||||
const seeded = [
|
const seeded = [
|
||||||
await seed({ageDays: 400, attachment: shared, profile: true}),
|
await seed({resolvedDaysAgo: 400, attachment: shared, profile: true}),
|
||||||
await seed({ageDays: 400, attachment: shared}),
|
await seed({resolvedDaysAgo: 400, attachment: shared}),
|
||||||
];
|
];
|
||||||
const scanOrder = (await repository.listAllReportsPaginated(10)).map((report) => report.reportId);
|
const scanOrder = (await repository.listAllReportsPaginated(10)).map((report) => report.reportId);
|
||||||
const [first, second] = seeded.sort((a, b) => scanOrder.indexOf(a.reportId) - scanOrder.indexOf(b.reportId)) as [
|
const [first, second] = seeded.sort((a, b) => scanOrder.indexOf(a.reportId) - scanOrder.indexOf(b.reportId)) as [
|
||||||
@@ -408,7 +758,11 @@ describe('report retention', () => {
|
|||||||
|
|
||||||
const summary = await processReportRetention(
|
const summary = await processReportRetention(
|
||||||
{...deps, reportRepository: holdingRepository},
|
{...deps, reportRepository: holdingRepository},
|
||||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
|
{
|
||||||
|
now: NOW,
|
||||||
|
dryRun: false,
|
||||||
|
policy: DEFAULT_POLICY,
|
||||||
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
await expectKept(first);
|
await expectKept(first);
|
||||||
@@ -419,7 +773,7 @@ describe('report retention', () => {
|
|||||||
|
|
||||||
it('keeps an object that a report filed during the run stored again', async () => {
|
it('keeps an object that a report filed during the run stored again', async () => {
|
||||||
const shared = newAttachment();
|
const shared = newAttachment();
|
||||||
const expired = await seed({ageDays: 400, attachment: shared, profile: true});
|
const expired = await seed({resolvedDaysAgo: 400, attachment: shared, profile: true});
|
||||||
const rewritingStorage = Object.create(storage) as MockStorageService;
|
const rewritingStorage = Object.create(storage) as MockStorageService;
|
||||||
rewritingStorage.getObjectMetadata = async (bucket: string, key: string) => {
|
rewritingStorage.getObjectMetadata = async (bucket: string, key: string) => {
|
||||||
const metadata = await storage.getObjectMetadata(bucket, key);
|
const metadata = await storage.getObjectMetadata(bucket, key);
|
||||||
@@ -431,7 +785,11 @@ describe('report retention', () => {
|
|||||||
|
|
||||||
const summary = await processReportRetention(
|
const summary = await processReportRetention(
|
||||||
{...deps, storageService: rewritingStorage},
|
{...deps, storageService: rewritingStorage},
|
||||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
|
{
|
||||||
|
now: NOW,
|
||||||
|
dryRun: false,
|
||||||
|
policy: DEFAULT_POLICY,
|
||||||
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
await expectGone(expired);
|
await expectGone(expired);
|
||||||
@@ -440,76 +798,25 @@ describe('report retention', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('deletes profile images stored for a report whose snapshot cannot be read', async () => {
|
it('deletes profile images stored for a report whose snapshot cannot be read', async () => {
|
||||||
const expired = await seed({ageDays: 400, profile: true, snapshotJson: '{"not":"a snapshot"'});
|
const expired = await seed({resolvedDaysAgo: 400, profile: true, snapshotJson: '{"not":"a snapshot"'});
|
||||||
|
|
||||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
const summary = await run();
|
||||||
|
|
||||||
expect(summary).toMatchObject({deleted: 1, objectsDeleted: 1});
|
expect(summary).toMatchObject({deleted: 1, objectsDeleted: 1});
|
||||||
await expectGone(expired);
|
await expectGone(expired);
|
||||||
});
|
});
|
||||||
|
|
||||||
function run(options: Partial<ReportRetentionOptions> = {}) {
|
|
||||||
return processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY, ...options});
|
|
||||||
}
|
|
||||||
|
|
||||||
it('deletes earlier when the retention period is shorter', async () => {
|
it('deletes earlier when the retention period is shorter', async () => {
|
||||||
const expired = await seed({ageDays: 30, profile: true});
|
const expired = await seed({resolvedDaysAgo: 30, profile: true});
|
||||||
const kept = await seed({ageDays: 29, profile: true});
|
const kept = await seed({resolvedDaysAgo: 29, profile: true});
|
||||||
|
|
||||||
const summary = await run({policy: {retentionDays: 30, resolvedRetentionDays: null}});
|
const summary = await run({policy: {retentionDays: 30}});
|
||||||
|
|
||||||
await expectGone(expired);
|
await expectGone(expired);
|
||||||
await expectKept(kept);
|
await expectKept(kept);
|
||||||
expect(summary).toMatchObject({scanned: 2, expired: 1, deleted: 1});
|
expect(summary).toMatchObject({scanned: 2, expired: 1, deleted: 1});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('deletes a resolved report once the resolved-report period has passed since it was resolved', async () => {
|
|
||||||
const policy = {retentionDays: 365, resolvedRetentionDays: 30};
|
|
||||||
const resolvedLongAgo = await seed({ageDays: 100, resolvedDaysAgo: 30, attachment: newAttachment()});
|
|
||||||
const resolvedRecently = await seed({ageDays: 100, resolvedDaysAgo: 29, profile: true});
|
|
||||||
const pending = await seed({ageDays: 100, profile: true});
|
|
||||||
const resolvedWithoutTime = await seed({ageDays: 100, status: 1, profile: true});
|
|
||||||
|
|
||||||
const summary = await run({policy});
|
|
||||||
|
|
||||||
await expectGone(resolvedLongAgo);
|
|
||||||
expect(storage.hasObject(REPORTS_BUCKET, resolvedLongAgo.attachmentKey!)).toBe(false);
|
|
||||||
await expectKept(resolvedRecently);
|
|
||||||
await expectKept(pending);
|
|
||||||
await expectKept(resolvedWithoutTime);
|
|
||||||
expect(summary).toMatchObject({scanned: 4, expired: 1, deleted: 1, held: 0});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('applies whichever rule ends first', async () => {
|
|
||||||
const policy = {retentionDays: 100, resolvedRetentionDays: 300};
|
|
||||||
const byAge = await seed({ageDays: 100, resolvedDaysAgo: 10});
|
|
||||||
const neither = await seed({ageDays: 99, resolvedDaysAgo: 90});
|
|
||||||
|
|
||||||
await run({policy});
|
|
||||||
|
|
||||||
await expectGone(byAge);
|
|
||||||
await expectKept(neither);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('keeps a held resolved report past the resolved-report period and keeps its evidence', async () => {
|
|
||||||
const shared = newAttachment();
|
|
||||||
const held = await seed({
|
|
||||||
ageDays: 50,
|
|
||||||
resolvedDaysAgo: 40,
|
|
||||||
attachment: shared,
|
|
||||||
profile: true,
|
|
||||||
legalHoldUntil: new Date(NOW.getTime() + DAY_MS),
|
|
||||||
});
|
|
||||||
const sharingWithHeld = await seed({ageDays: 50, resolvedDaysAgo: 40, attachment: shared});
|
|
||||||
|
|
||||||
const summary = await run({policy: {retentionDays: 365, resolvedRetentionDays: 7}});
|
|
||||||
|
|
||||||
await expectKept(held);
|
|
||||||
await expectGone(sharingWithHeld);
|
|
||||||
expect(storage.hasObject(REPORTS_BUCKET, attachmentKey(shared))).toBe(true);
|
|
||||||
expect(summary).toMatchObject({expired: 1, held: 1, deleted: 1, sharedObjectsKept: 1});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('frees the reporter to report the same account again once the report is deleted', async () => {
|
it('frees the reporter to report the same account again once the report is deleted', async () => {
|
||||||
const reporterId = nextId();
|
const reporterId = nextId();
|
||||||
const reportedUserId = nextId();
|
const reportedUserId = nextId();
|
||||||
@@ -519,8 +826,9 @@ describe('report retention', () => {
|
|||||||
report_id: BigInt(reportId),
|
report_id: BigInt(reportId),
|
||||||
reporter_id: reporterId,
|
reporter_id: reporterId,
|
||||||
reported_user_id: reportedUserId,
|
reported_user_id: reportedUserId,
|
||||||
reported_at: daysAgo(400),
|
reported_at: daysAgo(420),
|
||||||
status: 0,
|
status: 1,
|
||||||
|
resolved_at: daysAgo(400),
|
||||||
report_type: 1,
|
report_type: 1,
|
||||||
category: 'harassment',
|
category: 'harassment',
|
||||||
} as IARSubmissionRow);
|
} as IARSubmissionRow);
|
||||||
@@ -528,7 +836,7 @@ describe('report retention', () => {
|
|||||||
reporter_id: createUserID(reporterId),
|
reporter_id: createUserID(reporterId),
|
||||||
reported_user_id: createUserID(reportedUserId),
|
reported_user_id: createUserID(reportedUserId),
|
||||||
report_id: reportId,
|
report_id: reportId,
|
||||||
reported_at: daysAgo(400),
|
reported_at: daysAgo(420),
|
||||||
};
|
};
|
||||||
expect(await repository.reserveUserReportByReporter(reservation)).toBe(true);
|
expect(await repository.reserveUserReportByReporter(reservation)).toBe(true);
|
||||||
|
|
||||||
@@ -559,9 +867,10 @@ describe('report retention', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
it('deletes for real with the default settings', async () => {
|
it('deletes for real with the default settings', async () => {
|
||||||
expect(configured).toEqual({days: 365, resolvedDays: null, dryRun: false});
|
expect(configured).toEqual({days: 365, dryRun: false});
|
||||||
const expired = await seed({ageDays: 366, attachment: newAttachment(), profile: true});
|
const expired = await seed({resolvedDaysAgo: 366, attachment: newAttachment(), profile: true});
|
||||||
const kept = await seed({ageDays: 364, profile: true});
|
const kept = await seed({resolvedDaysAgo: 364, profile: true});
|
||||||
|
const pending = await seed({profile: true});
|
||||||
|
|
||||||
await runDailyTask();
|
await runDailyTask();
|
||||||
|
|
||||||
@@ -569,11 +878,12 @@ describe('report retention', () => {
|
|||||||
expect(storage.hasObject(REPORTS_BUCKET, expired.attachmentKey!)).toBe(false);
|
expect(storage.hasObject(REPORTS_BUCKET, expired.attachmentKey!)).toBe(false);
|
||||||
expect(storage.hasObject(REPORTS_BUCKET, expired.profileKey!)).toBe(false);
|
expect(storage.hasObject(REPORTS_BUCKET, expired.profileKey!)).toBe(false);
|
||||||
expect(await repository.getReport(kept.reportId)).not.toBeNull();
|
expect(await repository.getReport(kept.reportId)).not.toBeNull();
|
||||||
|
expect(await repository.getReport(pending.reportId)).not.toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('only counts what it would delete when FLUXER_REPORT_RETENTION_DRY_RUN is true', async () => {
|
it('only counts what it would delete when FLUXER_REPORT_RETENTION_DRY_RUN is true', async () => {
|
||||||
Object.assign(Config.reportRetention, {dryRun: true});
|
Object.assign(Config.reportRetention, {dryRun: true});
|
||||||
const expired = await seed({ageDays: 366, attachment: newAttachment(), profile: true});
|
const expired = await seed({resolvedDaysAgo: 366, attachment: newAttachment(), profile: true});
|
||||||
|
|
||||||
await runDailyTask();
|
await runDailyTask();
|
||||||
|
|
||||||
@@ -581,17 +891,18 @@ describe('report retention', () => {
|
|||||||
expect(storage.deleteObjectSpy).not.toHaveBeenCalled();
|
expect(storage.deleteObjectSpy).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('reads both day settings from the config', async () => {
|
it('reads the day setting from the config and honors decisions', async () => {
|
||||||
Object.assign(Config.reportRetention, {days: 60, resolvedDays: 5});
|
Object.assign(Config.reportRetention, {days: 60});
|
||||||
const byAge = await seed({ageDays: 60});
|
const expired = await seed({resolvedDaysAgo: 60});
|
||||||
const byResolution = await seed({ageDays: 20, resolvedDaysAgo: 5});
|
const kept = await seed({resolvedDaysAgo: 59});
|
||||||
const kept = await seed({ageDays: 20, resolvedDaysAgo: 4});
|
const linked = await seed({resolvedDaysAgo: 90});
|
||||||
|
await linkDecision([linked.reportId], daysAgo(90));
|
||||||
|
|
||||||
await runDailyTask();
|
await runDailyTask();
|
||||||
|
|
||||||
expect(await repository.getReport(byAge.reportId)).toBeNull();
|
expect(await repository.getReport(expired.reportId)).toBeNull();
|
||||||
expect(await repository.getReport(byResolution.reportId)).toBeNull();
|
|
||||||
expect(await repository.getReport(kept.reportId)).not.toBeNull();
|
expect(await repository.getReport(kept.reportId)).not.toBeNull();
|
||||||
|
expect(await repository.getReport(linked.reportId)).not.toBeNull();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -521,14 +521,22 @@ A report resolution is the reduced view [Update report](#update-report) returns.
|
|||||||
|
|
||||||
## Report retention
|
## Report retention
|
||||||
|
|
||||||
Fluxer deletes a report when its retention period ends, whether it is pending or resolved, together with its stored evidence: the attachment copies, the profile snapshot images, and its search index entry. The reporter's duplicate-report reservation for it goes too, so the reporter can report the same message, account or guild again.
|
A pending report is kept until it is resolved. Once a report is resolved, Fluxer deletes it when its retention period ends, together with its stored evidence: the captured message context, the profile snapshot and its images, the attachment copies, and its search index entry. The reporter's duplicate-report reservation for it goes too, so the reporter can report the same message, account or guild again.
|
||||||
|
|
||||||
The retention period is 365 days from when the report was filed, or the number of days in `FLUXER_REPORT_RETENTION_DAYS`. With `FLUXER_RESOLVED_REPORT_RETENTION_DAYS` set, a resolved report is deleted that many days after `resolved_at` when that comes first. A report with a [legal hold](#set-report-legal-hold) that ends later is kept until the next run after the hold ends. An evidence copy that a retained report also uses is kept for that report.
|
The retention period ends 365 days after `resolved_at`, or the number of days in `FLUXER_REPORT_RETENTION_DAYS`. Some reports are kept longer, whatever that setting says:
|
||||||
|
|
||||||
The deletion runs once a day in `worker`. Each run logs one `Processed report retention` line with the counts it found and deleted and the two periods it used. A report that fails to delete keeps its record and is retried on the next run. [Delete report](#delete-report) deletes one report straight away by the same rules.
|
- A report in the `child_safety` category is kept for at least 365 days after it was resolved.
|
||||||
|
- A report listed in the `report_ids` of an [enforcement decision](/admin-api/enforcement/#enforcement-decision-object) is kept for at least 365 days after the newest such decision was recorded, as long as the decision's evidence.
|
||||||
|
- A report with a [legal hold](#set-report-legal-hold) that ends later is kept until the next run after the hold ends.
|
||||||
|
|
||||||
|
A report resolved with the outcome `no_violation` loses its copied context 184 days after it was resolved, which covers the six months the Digital Services Act gives a reporter to contest the outcome. Fluxer then removes the captured message context, the profile snapshot, the reporter's additional information, and the stored evidence copies, and keeps the rest of the record and its search index entry until the end of the retention period. This does not apply to a `child_safety` report, to a report linked to an enforcement decision, or while a legal hold has not ended.
|
||||||
|
|
||||||
|
An evidence copy that a retained report also uses is kept for that report.
|
||||||
|
|
||||||
|
The job runs once a day in `worker`. Each run logs one `Processed report retention` line with the counts it found, deleted and cleared and the periods it used. A report that fails to delete or clear keeps its record and is retried on the next run. [Delete report](#delete-report) deletes one report straight away.
|
||||||
|
|
||||||
:::caution[Deleted evidence cannot be restored]
|
:::caution[Deleted evidence cannot be restored]
|
||||||
Retention deletes for real unless `FLUXER_REPORT_RETENTION_DRY_RUN` is `true` on `worker`. In a dry run each run only counts what it would delete and deletes nothing. The [operator configuration](/operator/configuration/#moderation-and-abuse) describes the three settings.
|
Retention deletes for real unless `FLUXER_REPORT_RETENTION_DRY_RUN` is `true` on `worker`. In a dry run each run only counts what it would delete or clear and changes nothing. The [operator configuration](/operator/configuration/#moderation-and-abuse) describes the two settings.
|
||||||
:::
|
:::
|
||||||
|
|
||||||
## List reports
|
## List reports
|
||||||
@@ -768,7 +776,7 @@ Fluxer records one [Admin audit entry](/admin-api/#admin-audit-entry-object) wit
|
|||||||
|
|
||||||
Deletes one report straight away. Requires `report:delete`, which `report:resolve` does not include.
|
Deletes one report straight away. Requires `report:delete`, which `report:resolve` does not include.
|
||||||
|
|
||||||
This is the deletion [report retention](#report-retention) runs, without waiting for the retention period. It removes the report record with its captured message context and profile snapshot, its stored evidence copies, its search index entry, and the reporter's duplicate-report reservation for it. An evidence copy that another report also uses is kept for that report. A pending or a resolved report can be deleted, and the reporter is not told.
|
This is the deletion [report retention](#report-retention) runs at the end of the retention period, without waiting for it. It removes the report record with its captured message context and profile snapshot, its stored evidence copies, its search index entry, and the reporter's duplicate-report reservation for it. An evidence copy that another report also uses is kept for that report. A pending or a resolved report can be deleted, and the reporter is not told.
|
||||||
|
|
||||||
:::caution[Deletion is permanent]
|
:::caution[Deletion is permanent]
|
||||||
Deleted evidence cannot be restored. A report with a legal hold that has not ended is refused. Clear the hold first when the report should go.
|
Deleted evidence cannot be restored. A report with a legal hold that has not ended is refused. Clear the hold first when the report should go.
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ A safety report tells moderators that a message or a user profile broke the rule
|
|||||||
Fluxer discards a body member that no schema here defines, so a misspelt member is dropped without an error. A body that is not parseable JSON returns 400 `INVALID_FORM_BODY` with the validation code `INVALID_FORMAT` at the `body` path.
|
Fluxer discards a body member that no schema here defines, so a misspelt member is dropped without an error. A body that is not parseable JSON returns 400 `INVALID_FORM_BODY` with the validation code `INVALID_FORMAT` at the `body` path.
|
||||||
|
|
||||||
:::caution[A submitted report is durable evidence]
|
:::caution[A submitted report is durable evidence]
|
||||||
Fluxer copies the reported content when the report is submitted. A message report also copies the surrounding conversation and the bytes of every attachment in it, and every report copies the reported profile with its images. Deleting the message, editing the profile, or leaving the guild afterwards does not remove that snapshot. The snapshot is kept until the report reaches the end of its [retention period](/admin-api/reports/#report-retention).
|
Fluxer copies the reported content when the report is submitted. A message report also copies the surrounding conversation and the bytes of every attachment in it, and every report copies the reported profile with its images. Deleting the message, editing the profile, or leaving the guild afterwards does not remove that snapshot. The snapshot is kept until [report retention](/admin-api/reports/#report-retention) removes it, which happens no sooner than 184 days after the report is resolved.
|
||||||
:::
|
:::
|
||||||
|
|
||||||
## Access rules
|
## Access rules
|
||||||
|
|||||||
@@ -1063,15 +1063,11 @@ With feeds off, Fluxer checks none of this data. The first worker start with fee
|
|||||||
|
|
||||||
#### `FLUXER_REPORT_RETENTION_DAYS`
|
#### `FLUXER_REPORT_RETENTION_DAYS`
|
||||||
|
|
||||||
Default `365`. Read by `worker`. The daily [report retention](/admin-api/reports/#report-retention) job deletes each report this many days after it was filed, pending or resolved, with its evidence copies and search entry, unless a legal hold is in place. Integer 1 to 36500, checked at startup.
|
Default `365`. Read by `worker`. The daily [report retention](/admin-api/reports/#report-retention) job deletes each resolved report this many days after it was resolved, with its evidence copies and search entry. Pending reports are kept until they are resolved. A legal hold, a `child_safety` category or a linked enforcement decision can keep a report longer. Integer 1 to 36500, checked at startup.
|
||||||
|
|
||||||
#### `FLUXER_RESOLVED_REPORT_RETENTION_DAYS`
|
|
||||||
|
|
||||||
Default unset, which leaves resolved reports to `FLUXER_REPORT_RETENTION_DAYS`. Read by `worker`. When set, a resolved report is deleted this many days after it was resolved, if that comes before the end of its retention period. Pending reports are not affected, and a legal hold still keeps a report. Integer 1 to 36500, checked at startup.
|
|
||||||
|
|
||||||
#### `FLUXER_REPORT_RETENTION_DRY_RUN`
|
#### `FLUXER_REPORT_RETENTION_DRY_RUN`
|
||||||
|
|
||||||
Default `false`. Read by `worker`. With `true`, each run only counts what it would delete and deletes nothing. `true` or `false`, checked at startup. Deleted evidence cannot be restored. Check the counts in the `Processed report retention` log line before the first real run.
|
Default `false`. Read by `worker`. With `true`, each run only counts what it would delete or clear and changes nothing. `true` or `false`, checked at startup. Deleted evidence cannot be restored. Check the counts in the `Processed report retention` log line before the first real run.
|
||||||
|
|
||||||
#### `FLUXER_BREACHED_PASSWORD_CHECK_ENABLED`
|
#### `FLUXER_BREACHED_PASSWORD_CHECK_ENABLED`
|
||||||
|
|
||||||
|
|||||||
@@ -280,7 +280,6 @@ function defaultConfig(): MasterConfig {
|
|||||||
inactivity_deletion_threshold_days: 365,
|
inactivity_deletion_threshold_days: 365,
|
||||||
report_retention: {
|
report_retention: {
|
||||||
days: 365,
|
days: 365,
|
||||||
resolved_days: null,
|
|
||||||
dry_run: false,
|
dry_run: false,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
@@ -451,14 +450,6 @@ function validateApiWorkerConfig(config: MasterConfig): void {
|
|||||||
function validateReportRetentionConfig(config: MasterConfig): void {
|
function validateReportRetentionConfig(config: MasterConfig): void {
|
||||||
const retention = config.report_retention;
|
const retention = config.report_retention;
|
||||||
assertIntegerInRange(retention.days, 'FLUXER_REPORT_RETENTION_DAYS', 1, MAX_REPORT_RETENTION_DAYS);
|
assertIntegerInRange(retention.days, 'FLUXER_REPORT_RETENTION_DAYS', 1, MAX_REPORT_RETENTION_DAYS);
|
||||||
if (retention.resolved_days !== null) {
|
|
||||||
assertIntegerInRange(
|
|
||||||
retention.resolved_days,
|
|
||||||
'FLUXER_RESOLVED_REPORT_RETENTION_DAYS',
|
|
||||||
1,
|
|
||||||
MAX_REPORT_RETENTION_DAYS,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
assertBoolean(retention.dry_run, 'FLUXER_REPORT_RETENTION_DRY_RUN');
|
assertBoolean(retention.dry_run, 'FLUXER_REPORT_RETENTION_DRY_RUN');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -334,7 +334,6 @@ export interface MasterConfig {
|
|||||||
inactivity_deletion_threshold_days: number;
|
inactivity_deletion_threshold_days: number;
|
||||||
report_retention: {
|
report_retention: {
|
||||||
days: number;
|
days: number;
|
||||||
resolved_days: number | null;
|
|
||||||
dry_run: boolean;
|
dry_run: boolean;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -781,42 +781,22 @@ describe('report retention settings', () => {
|
|||||||
vi.unstubAllEnvs();
|
vi.unstubAllEnvs();
|
||||||
});
|
});
|
||||||
|
|
||||||
test('deletes reports for real after 365 days with no resolved-report rule by default', async () => {
|
test('deletes resolved reports for real after 365 days by default', async () => {
|
||||||
stubMinimalEnv();
|
stubMinimalEnv();
|
||||||
const config = await loadConfig();
|
const config = await loadConfig();
|
||||||
expect(config.report_retention).toEqual({days: 365, resolved_days: null, dry_run: false});
|
expect(config.report_retention).toEqual({days: 365, dry_run: false});
|
||||||
});
|
});
|
||||||
|
|
||||||
test('reads both day settings and the dry run switch from the environment', async () => {
|
test('reads the day setting and the dry run switch from the environment', async () => {
|
||||||
stubMinimalEnv({
|
stubMinimalEnv({FLUXER_REPORT_RETENTION_DAYS: ' 730 ', FLUXER_REPORT_RETENTION_DRY_RUN: 'TRUE'});
|
||||||
FLUXER_REPORT_RETENTION_DAYS: '730',
|
|
||||||
FLUXER_RESOLVED_REPORT_RETENTION_DAYS: ' 90 ',
|
|
||||||
FLUXER_REPORT_RETENTION_DRY_RUN: 'TRUE',
|
|
||||||
});
|
|
||||||
const config = await loadConfig();
|
const config = await loadConfig();
|
||||||
expect(config.report_retention).toEqual({days: 730, resolved_days: 90, dry_run: true});
|
expect(config.report_retention).toEqual({days: 730, dry_run: true});
|
||||||
});
|
|
||||||
|
|
||||||
test('an empty resolved-report setting leaves the rule off', async () => {
|
|
||||||
stubMinimalEnv({FLUXER_RESOLVED_REPORT_RETENTION_DAYS: '', FLUXER_REPORT_RETENTION_DRY_RUN: 'false'});
|
|
||||||
const config = await loadConfig();
|
|
||||||
expect(config.report_retention).toEqual({days: 365, resolved_days: null, dry_run: false});
|
|
||||||
});
|
});
|
||||||
|
|
||||||
test.each([
|
test.each([
|
||||||
['FLUXER_REPORT_RETENTION_DAYS', '0', 'FLUXER_REPORT_RETENTION_DAYS must be an integer between 1 and 36500'],
|
['FLUXER_REPORT_RETENTION_DAYS', '0', 'FLUXER_REPORT_RETENTION_DAYS must be an integer between 1 and 36500'],
|
||||||
['FLUXER_REPORT_RETENTION_DAYS', '36501', 'FLUXER_REPORT_RETENTION_DAYS must be an integer between 1 and 36500'],
|
['FLUXER_REPORT_RETENTION_DAYS', '36501', 'FLUXER_REPORT_RETENTION_DAYS must be an integer between 1 and 36500'],
|
||||||
['FLUXER_REPORT_RETENTION_DAYS', '1.5', 'FLUXER_REPORT_RETENTION_DAYS must be an integer, got "1.5"'],
|
['FLUXER_REPORT_RETENTION_DAYS', '1.5', 'FLUXER_REPORT_RETENTION_DAYS must be an integer, got "1.5"'],
|
||||||
[
|
|
||||||
'FLUXER_RESOLVED_REPORT_RETENTION_DAYS',
|
|
||||||
'0',
|
|
||||||
'FLUXER_RESOLVED_REPORT_RETENTION_DAYS must be an integer between 1 and 36500',
|
|
||||||
],
|
|
||||||
[
|
|
||||||
'FLUXER_RESOLVED_REPORT_RETENTION_DAYS',
|
|
||||||
'-30',
|
|
||||||
'FLUXER_RESOLVED_REPORT_RETENTION_DAYS must be an integer between 1 and 36500',
|
|
||||||
],
|
|
||||||
['FLUXER_REPORT_RETENTION_DRY_RUN', '1', 'FLUXER_REPORT_RETENTION_DRY_RUN must be true or false'],
|
['FLUXER_REPORT_RETENTION_DRY_RUN', '1', 'FLUXER_REPORT_RETENTION_DRY_RUN must be true or false'],
|
||||||
])('refuses %s=%j', async (name, value, message) => {
|
])('refuses %s=%j', async (name, value, message) => {
|
||||||
stubMinimalEnv({[name]: value});
|
stubMinimalEnv({[name]: value});
|
||||||
|
|||||||
@@ -97,11 +97,10 @@ describe('the shipped compose stack forwards settings from .env', () => {
|
|||||||
expect(renamed).toEqual([]);
|
expect(renamed).toEqual([]);
|
||||||
});
|
});
|
||||||
|
|
||||||
test.each([
|
test.each(['FLUXER_REPORT_RETENTION_DAYS', 'FLUXER_REPORT_RETENTION_DRY_RUN'])(
|
||||||
'FLUXER_REPORT_RETENTION_DAYS',
|
'the worker is handed %s and gets no value when .env leaves it out',
|
||||||
'FLUXER_RESOLVED_REPORT_RETENTION_DAYS',
|
(name) => {
|
||||||
'FLUXER_REPORT_RETENTION_DRY_RUN',
|
expect(serviceEnvironment('worker')[name]).toBe(`\${${name}:-}`);
|
||||||
])('the worker is handed %s and gets no value when .env leaves it out', (name) => {
|
},
|
||||||
expect(serviceEnvironment('worker')[name]).toBe(`\${${name}:-}`);
|
);
|
||||||
});
|
|
||||||
});
|
});
|
||||||
@@ -301,7 +301,6 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
|
|||||||
FLUXER_DISCOVERY_MIN_MEMBER_COUNT: {path: ['discovery', 'min_member_count'], parse: parseInteger},
|
FLUXER_DISCOVERY_MIN_MEMBER_COUNT: {path: ['discovery', 'min_member_count'], parse: parseInteger},
|
||||||
FLUXER_DELETION_GRACE_PERIOD_HOURS: {path: ['deletion_grace_period_hours'], parse: parseInteger},
|
FLUXER_DELETION_GRACE_PERIOD_HOURS: {path: ['deletion_grace_period_hours'], parse: parseInteger},
|
||||||
FLUXER_REPORT_RETENTION_DAYS: {path: ['report_retention', 'days'], parse: parseInteger},
|
FLUXER_REPORT_RETENTION_DAYS: {path: ['report_retention', 'days'], parse: parseInteger},
|
||||||
FLUXER_RESOLVED_REPORT_RETENTION_DAYS: {path: ['report_retention', 'resolved_days'], parse: parseInteger},
|
|
||||||
FLUXER_REPORT_RETENTION_DRY_RUN: {path: ['report_retention', 'dry_run'], parse: parseBoolean},
|
FLUXER_REPORT_RETENTION_DRY_RUN: {path: ['report_retention', 'dry_run'], parse: parseBoolean},
|
||||||
FLUXER_RELAX_REGISTRATION_RATE_LIMITS: {path: ['dev', 'relax_registration_rate_limits'], parse: parseBoolean},
|
FLUXER_RELAX_REGISTRATION_RATE_LIMITS: {path: ['dev', 'relax_registration_rate_limits'], parse: parseBoolean},
|
||||||
FLUXER_DISABLE_RATE_LIMITS: {path: ['dev', 'disable_rate_limits'], parse: parseBoolean},
|
FLUXER_DISABLE_RATE_LIMITS: {path: ['dev', 'disable_rate_limits'], parse: parseBoolean},
|
||||||
|
|||||||
Reference in new issue
Block a user