mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-18 21:45:31 +09:00
feat(reports): delete reports and configure retention (#3338)
This commit is contained in:
1 parent
93d1f8af46
commit
82abd03a37
106 files changed
+1457
-170
No files matched your search
@@ -2,31 +2,29 @@
|
||||
|
||||
import type {ReportID} from '@app/api/BrandedTypes';
|
||||
import {Config} from '@app/api/Config';
|
||||
import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';
|
||||
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
|
||||
import {Logger} from '@app/api/Logger';
|
||||
import type {IARSubmission, IReportRepository} from '@app/api/report/IReportRepository';
|
||||
import {type IARSubmission, ReportStatus} from '@app/api/report/IReportRepository';
|
||||
import {
|
||||
deleteReportWithEvidence,
|
||||
forEachStoredReport,
|
||||
isReportUnderLegalHold,
|
||||
type ReportDeletionDeps,
|
||||
referencedObjectKeys,
|
||||
} from '@app/api/report/ReportDeletion';
|
||||
import {getReportSearchService} from '@app/api/SearchFactory';
|
||||
import type {IReportSearchService} from '@app/api/search/IReportSearchService';
|
||||
import {readOptionalEnv} from '@app/api/utils/IntegerOptions';
|
||||
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
|
||||
import {listReportProfileSnapshotAssets} from '@fluxer/schema/src/domains/report/ReportProfileSnapshotSchemas';
|
||||
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
|
||||
import {ms} from 'itty-time';
|
||||
|
||||
const REPORT_RETENTION_DAYS = 365;
|
||||
const REPORT_RETENTION_MS = REPORT_RETENTION_DAYS * ms('1 day');
|
||||
const DEFAULT_SCAN_PAGE_SIZE = 500;
|
||||
|
||||
export interface ReportRetentionDeps {
|
||||
reportRepository: IReportRepository;
|
||||
storageService: IStorageService;
|
||||
reportSearchService: IReportSearchService | null;
|
||||
export interface ReportRetentionPolicy {
|
||||
retentionDays: number;
|
||||
resolvedRetentionDays: number | null;
|
||||
}
|
||||
|
||||
export interface ReportRetentionOptions {
|
||||
now: Date;
|
||||
dryRun: boolean;
|
||||
policy: ReportRetentionPolicy;
|
||||
pageSize?: number;
|
||||
}
|
||||
|
||||
@@ -43,63 +41,36 @@ export interface ReportRetentionSummary {
|
||||
|
||||
type ReportRetentionState = 'expired' | 'held' | 'retained';
|
||||
|
||||
export function resolveReportRetentionDryRun(raw: string | undefined): boolean {
|
||||
const value = raw?.trim().toLowerCase();
|
||||
return value !== 'false' && value !== '0';
|
||||
function reportExpiresAt(report: IARSubmission, policy: ReportRetentionPolicy): number {
|
||||
const reportedAt = report.reportedAt instanceof Date ? report.reportedAt.getTime() : Number.NaN;
|
||||
const byAge = reportedAt + policy.retentionDays * ms('1 day');
|
||||
if (policy.resolvedRetentionDays === null || report.status !== ReportStatus.RESOLVED) {
|
||||
return byAge;
|
||||
}
|
||||
const resolvedAt = report.resolvedAt instanceof Date ? report.resolvedAt.getTime() : Number.NaN;
|
||||
if (Number.isNaN(resolvedAt)) {
|
||||
return byAge;
|
||||
}
|
||||
return Math.min(byAge, resolvedAt + policy.resolvedRetentionDays * ms('1 day'));
|
||||
}
|
||||
|
||||
function reportRetentionState(report: IARSubmission, now: Date): ReportRetentionState {
|
||||
const reportedAt = report.reportedAt instanceof Date ? report.reportedAt.getTime() : Number.NaN;
|
||||
if (Number.isNaN(reportedAt) || reportedAt > now.getTime() - REPORT_RETENTION_MS) {
|
||||
function reportRetentionState(report: IARSubmission, now: Date, policy: ReportRetentionPolicy): ReportRetentionState {
|
||||
const expiresAt = reportExpiresAt(report, policy);
|
||||
if (Number.isNaN(expiresAt) || expiresAt > now.getTime()) {
|
||||
return 'retained';
|
||||
}
|
||||
if (report.legalHoldUntil && report.legalHoldUntil.getTime() > now.getTime()) {
|
||||
if (isReportUnderLegalHold(report, now)) {
|
||||
return 'held';
|
||||
}
|
||||
return 'expired';
|
||||
}
|
||||
|
||||
function reportProfilePrefix(reportId: ReportID): string {
|
||||
return `reports/${reportId}/profile/`;
|
||||
}
|
||||
|
||||
function referencedObjectKeys(report: IARSubmission): Array<string> {
|
||||
const keys: Array<string> = [];
|
||||
for (const message of report.messageContext ?? []) {
|
||||
const channelId = message.channelId ?? report.reportedChannelId;
|
||||
if (!channelId) continue;
|
||||
for (const attachment of message.attachments) {
|
||||
if (attachment.attachment_id == null || !attachment.filename) continue;
|
||||
keys.push(makeAttachmentCdnKey(channelId, attachment.attachment_id, String(attachment.filename)));
|
||||
}
|
||||
}
|
||||
for (const asset of listReportProfileSnapshotAssets(report.reportedProfileSnapshot)) {
|
||||
if (asset.key) {
|
||||
keys.push(asset.key);
|
||||
}
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
async function storedObjectKeys(storageService: IStorageService, report: IARSubmission): Promise<Set<string>> {
|
||||
const keys = new Set(referencedObjectKeys(report));
|
||||
const listed = await storageService.listObjects({
|
||||
bucket: Config.s3.buckets.reports,
|
||||
prefix: reportProfilePrefix(report.reportId),
|
||||
});
|
||||
for (const object of listed) {
|
||||
keys.add(object.key);
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
export async function processReportRetention(
|
||||
deps: ReportRetentionDeps,
|
||||
deps: ReportDeletionDeps,
|
||||
options: ReportRetentionOptions,
|
||||
): Promise<ReportRetentionSummary> {
|
||||
const {reportRepository, storageService, reportSearchService} = deps;
|
||||
const {now, dryRun} = options;
|
||||
const pageSize = options.pageSize ?? DEFAULT_SCAN_PAGE_SIZE;
|
||||
const {reportRepository} = deps;
|
||||
const {now, dryRun, policy} = options;
|
||||
const summary: ReportRetentionSummary = {
|
||||
dryRun,
|
||||
scanned: 0,
|
||||
@@ -112,15 +83,14 @@ export async function processReportRetention(
|
||||
};
|
||||
const retainedKeys = new Set<string>();
|
||||
const expiredIds: Array<ReportID> = [];
|
||||
let cursor: ReportID | undefined;
|
||||
while (true) {
|
||||
const page = await reportRepository.listAllReportsPaginated(pageSize, cursor);
|
||||
for (const report of page) {
|
||||
await forEachStoredReport(
|
||||
reportRepository,
|
||||
(report) => {
|
||||
summary.scanned++;
|
||||
const state = reportRetentionState(report, now);
|
||||
const state = reportRetentionState(report, now, policy);
|
||||
if (state === 'expired') {
|
||||
expiredIds.push(report.reportId);
|
||||
continue;
|
||||
return;
|
||||
}
|
||||
if (state === 'held') {
|
||||
summary.held++;
|
||||
@@ -128,59 +98,47 @@ export async function processReportRetention(
|
||||
for (const key of referencedObjectKeys(report)) {
|
||||
retainedKeys.add(key);
|
||||
}
|
||||
}
|
||||
if (page.length < pageSize) break;
|
||||
cursor = page[page.length - 1]!.reportId;
|
||||
}
|
||||
},
|
||||
options.pageSize,
|
||||
);
|
||||
summary.expired = expiredIds.length;
|
||||
const deletedKeys = new Set<string>();
|
||||
const ledger = {retainedKeys, deletedKeys: new Set<string>()};
|
||||
for (const reportId of expiredIds) {
|
||||
try {
|
||||
const report = await reportRepository.getReport(reportId);
|
||||
if (!report) continue;
|
||||
if (reportRetentionState(report, now) !== 'expired') {
|
||||
if (reportRetentionState(report, now, policy) !== 'expired') {
|
||||
for (const key of referencedObjectKeys(report)) {
|
||||
retainedKeys.add(key);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
for (const key of await storedObjectKeys(storageService, report)) {
|
||||
if (retainedKeys.has(key)) {
|
||||
summary.sharedObjectsKept++;
|
||||
continue;
|
||||
}
|
||||
if (deletedKeys.has(key)) continue;
|
||||
const metadata = await storageService.getObjectMetadata(Config.s3.buckets.reports, key);
|
||||
if (metadata?.lastModified && metadata.lastModified.getTime() >= now.getTime()) {
|
||||
retainedKeys.add(key);
|
||||
summary.sharedObjectsKept++;
|
||||
continue;
|
||||
}
|
||||
if (!dryRun) {
|
||||
await storageService.deleteObject(Config.s3.buckets.reports, key);
|
||||
}
|
||||
deletedKeys.add(key);
|
||||
summary.objectsDeleted++;
|
||||
}
|
||||
if (!dryRun) {
|
||||
await reportSearchService?.deleteReport(reportId);
|
||||
await reportRepository.deleteReport(reportId);
|
||||
}
|
||||
const result = await deleteReportWithEvidence(deps, report, ledger, {now, dryRun});
|
||||
summary.objectsDeleted += result.objectsDeleted;
|
||||
summary.sharedObjectsKept += result.sharedObjectsKept;
|
||||
summary.deleted++;
|
||||
} catch (error) {
|
||||
summary.failed++;
|
||||
Logger.error({error, reportId: reportId.toString()}, 'Failed to delete an expired report');
|
||||
}
|
||||
}
|
||||
Logger.info({...summary, retentionDays: REPORT_RETENTION_DAYS}, 'Processed report retention');
|
||||
Logger.info(
|
||||
{
|
||||
...summary,
|
||||
retentionDays: policy.retentionDays,
|
||||
resolvedRetentionDays: policy.resolvedRetentionDays,
|
||||
},
|
||||
'Processed report retention',
|
||||
);
|
||||
return summary;
|
||||
}
|
||||
|
||||
const expireReportSnapshots: WorkerTaskHandler = async () => {
|
||||
const {reportRepository, storageService} = getWorkerDependencies();
|
||||
const {days, resolvedDays, dryRun} = Config.reportRetention;
|
||||
await processReportRetention(
|
||||
{reportRepository, storageService, reportSearchService: getReportSearchService()},
|
||||
{now: new Date(), dryRun: resolveReportRetentionDryRun(readOptionalEnv('FLUXER_REPORT_RETENTION_DRY_RUN'))},
|
||||
{now: new Date(), dryRun, policy: {retentionDays: days, resolvedRetentionDays: resolvedDays}},
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {createAttachmentID, createChannelID, createReportID, type ReportID} from '@app/api/BrandedTypes';
|
||||
import {createAttachmentID, createChannelID, createReportID, createUserID, type ReportID} from '@app/api/BrandedTypes';
|
||||
import {Config} from '@app/api/Config';
|
||||
import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';
|
||||
import type {MessageAttachment} from '@app/api/database/types/MessageTypes';
|
||||
@@ -10,22 +10,28 @@ import {
|
||||
type IARSubmissionRow,
|
||||
} from '@app/api/database/types/ReportTypes';
|
||||
import type {IARSubmission} from '@app/api/report/IReportRepository';
|
||||
import type {ReportDeletionDeps} from '@app/api/report/ReportDeletion';
|
||||
import {ReportRepository} from '@app/api/report/ReportRepository';
|
||||
import type {IReportSearchService} from '@app/api/search/IReportSearchService';
|
||||
import {MockStorageService} from '@app/api/test/mocks/MockStorageService';
|
||||
import {NoopLogger} from '@app/api/test/mocks/NoopLogger';
|
||||
import {InMemorySearchProvider} from '@app/api/test/search/InMemorySearchProvider';
|
||||
import {
|
||||
import expireReportSnapshots, {
|
||||
processReportRetention,
|
||||
type ReportRetentionDeps,
|
||||
resolveReportRetentionDryRun,
|
||||
type ReportRetentionOptions,
|
||||
type ReportRetentionPolicy,
|
||||
} from '@app/api/worker/tasks/ExpireReportSnapshots';
|
||||
import {clearWorkerDependencies, setWorkerDependenciesForTest} from '@app/api/worker/WorkerContext';
|
||||
import {serializeReportProfileSnapshot} from '@fluxer/schema/src/domains/report/ReportProfileSnapshotSchemas';
|
||||
import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
|
||||
import {ms} from 'itty-time';
|
||||
import {beforeEach, describe, expect, it} from 'vitest';
|
||||
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
|
||||
|
||||
const NOW = new Date('2027-06-01T12:00:00.000Z');
|
||||
const DAY_MS = ms('1 day');
|
||||
const REPORTS_BUCKET = Config.s3.buckets.reports;
|
||||
const WORKER_HELPERS = {logger: new NoopLogger()} as unknown as WorkerTaskHelpers;
|
||||
const DEFAULT_POLICY: ReportRetentionPolicy = {retentionDays: 365, resolvedRetentionDays: null};
|
||||
let sequence = 1_480_000_000_000_000_000n;
|
||||
|
||||
function nextId(): bigint {
|
||||
@@ -92,6 +98,7 @@ interface SeedOptions {
|
||||
profile?: boolean;
|
||||
legalHoldUntil?: Date | null;
|
||||
status?: number;
|
||||
resolvedDaysAgo?: number;
|
||||
snapshotJson?: string;
|
||||
}
|
||||
|
||||
@@ -113,7 +120,7 @@ describe('report retention', () => {
|
||||
let repository: ReportRepository;
|
||||
let storage: MockStorageService;
|
||||
let search: IReportSearchService;
|
||||
let deps: ReportRetentionDeps;
|
||||
let deps: ReportDeletionDeps;
|
||||
|
||||
beforeEach(() => {
|
||||
repository = new ReportRepository();
|
||||
@@ -158,7 +165,8 @@ describe('report retention', () => {
|
||||
...Object.fromEntries(IAR_SUBMISSION_COLUMNS.map((column) => [column, null])),
|
||||
report_id: id,
|
||||
reported_at: daysAgo(options.ageDays),
|
||||
status: options.status ?? 0,
|
||||
status: options.status ?? (options.resolvedDaysAgo === undefined ? 0 : 1),
|
||||
resolved_at: options.resolvedDaysAgo === undefined ? null : daysAgo(options.resolvedDaysAgo),
|
||||
report_type: 0,
|
||||
category: 'other',
|
||||
reported_channel_id: options.attachment?.channelId ?? null,
|
||||
@@ -195,7 +203,7 @@ describe('report retention', () => {
|
||||
const expired = await seed({ageDays: 365, attachment: newAttachment(), profile: true});
|
||||
const kept = await seed({ageDays: 364, attachment: newAttachment(), profile: true});
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false});
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
|
||||
await expectGone(expired);
|
||||
expect(storage.hasObject(REPORTS_BUCKET, expired.attachmentKey!)).toBe(false);
|
||||
@@ -216,7 +224,7 @@ describe('report retention', () => {
|
||||
const pending = await seed({ageDays: 400, status: 0});
|
||||
const resolved = await seed({ageDays: 400, status: 1});
|
||||
|
||||
await processReportRetention(deps, {now: NOW, dryRun: false});
|
||||
await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
|
||||
await expectGone(pending);
|
||||
await expectGone(resolved);
|
||||
@@ -227,7 +235,7 @@ describe('report retention', () => {
|
||||
const expired = await seed({ageDays: 500, attachment: shared});
|
||||
const retained = await seed({ageDays: 10, attachment: shared});
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false});
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
|
||||
await expectGone(expired);
|
||||
await expectKept(retained);
|
||||
@@ -241,7 +249,7 @@ describe('report retention', () => {
|
||||
const first = await seed({ageDays: 500, attachment: shared});
|
||||
const second = await seed({ageDays: 450, attachment: shared});
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false});
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
|
||||
await expectGone(first);
|
||||
await expectGone(second);
|
||||
@@ -261,7 +269,7 @@ describe('report retention', () => {
|
||||
const holdEnded = await seed({ageDays: 500, profile: true, legalHoldUntil: daysAgo(1)});
|
||||
const sharingWithHeld = await seed({ageDays: 500, attachment: shared});
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false});
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
|
||||
await expectKept(held);
|
||||
expect((await repository.getReport(held.reportId))?.legalHoldReason).toBe('Preservation request');
|
||||
@@ -275,7 +283,7 @@ describe('report retention', () => {
|
||||
const expired = await seed({ageDays: 366, attachment: newAttachment(), profile: true});
|
||||
const kept = await seed({ageDays: 1, profile: true});
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: true});
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: true, policy: DEFAULT_POLICY});
|
||||
|
||||
await expectKept(expired);
|
||||
await expectKept(kept);
|
||||
@@ -305,7 +313,10 @@ describe('report retention', () => {
|
||||
};
|
||||
|
||||
await expect(
|
||||
processReportRetention({...deps, reportRepository: failingRepository}, {now: NOW, dryRun: false, pageSize: 2}),
|
||||
processReportRetention(
|
||||
{...deps, reportRepository: failingRepository},
|
||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY, pageSize: 2},
|
||||
),
|
||||
).rejects.toThrow('read timeout');
|
||||
|
||||
await expectKept(expired);
|
||||
@@ -321,7 +332,7 @@ describe('report retention', () => {
|
||||
kept.push(await seed({ageDays: index, profile: true}));
|
||||
}
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, pageSize: 3});
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY, pageSize: 3});
|
||||
|
||||
expect(summary).toMatchObject({scanned: 14, expired: 7, deleted: 7, objectsDeleted: 7, failed: 0});
|
||||
for (const report of expired) await expectGone(report);
|
||||
@@ -332,14 +343,14 @@ describe('report retention', () => {
|
||||
const expired = await seed({ageDays: 400, attachment: newAttachment(), profile: true});
|
||||
storage.configure({shouldFailDelete: true});
|
||||
|
||||
const failed = await processReportRetention(deps, {now: NOW, dryRun: false});
|
||||
const failed = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
|
||||
expect(failed).toMatchObject({expired: 1, deleted: 0, failed: 1});
|
||||
expect(await repository.getReport(expired.reportId)).not.toBeNull();
|
||||
expect(await indexedIds()).toContain(expired.reportId.toString());
|
||||
|
||||
storage.configure({shouldFailDelete: false});
|
||||
const retried = await processReportRetention(deps, {now: NOW, dryRun: false});
|
||||
const retried = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
|
||||
expect(retried).toMatchObject({expired: 1, deleted: 1, failed: 0});
|
||||
await expectGone(expired);
|
||||
@@ -355,7 +366,7 @@ describe('report retention', () => {
|
||||
|
||||
const summary = await processReportRetention(
|
||||
{...deps, reportSearchService: failingSearch},
|
||||
{now: NOW, dryRun: false},
|
||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
|
||||
);
|
||||
|
||||
expect(summary).toMatchObject({deleted: 0, failed: 1});
|
||||
@@ -365,7 +376,10 @@ describe('report retention', () => {
|
||||
it('deletes rows when search is not configured', async () => {
|
||||
const expired = await seed({ageDays: 400, profile: true});
|
||||
|
||||
const summary = await processReportRetention({...deps, reportSearchService: null}, {now: NOW, dryRun: false});
|
||||
const summary = await processReportRetention(
|
||||
{...deps, reportSearchService: null},
|
||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
|
||||
);
|
||||
|
||||
expect(summary).toMatchObject({deleted: 1, failed: 0});
|
||||
expect(await repository.getReport(expired.reportId)).toBeNull();
|
||||
@@ -394,7 +408,7 @@ describe('report retention', () => {
|
||||
|
||||
const summary = await processReportRetention(
|
||||
{...deps, reportRepository: holdingRepository},
|
||||
{now: NOW, dryRun: false},
|
||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
|
||||
);
|
||||
|
||||
await expectKept(first);
|
||||
@@ -417,7 +431,7 @@ describe('report retention', () => {
|
||||
|
||||
const summary = await processReportRetention(
|
||||
{...deps, storageService: rewritingStorage},
|
||||
{now: NOW, dryRun: false},
|
||||
{now: NOW, dryRun: false, policy: DEFAULT_POLICY},
|
||||
);
|
||||
|
||||
await expectGone(expired);
|
||||
@@ -428,22 +442,156 @@ describe('report retention', () => {
|
||||
it('deletes profile images stored for a report whose snapshot cannot be read', async () => {
|
||||
const expired = await seed({ageDays: 400, profile: true, snapshotJson: '{"not":"a snapshot"'});
|
||||
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false});
|
||||
const summary = await processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY});
|
||||
|
||||
expect(summary).toMatchObject({deleted: 1, objectsDeleted: 1});
|
||||
await expectGone(expired);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[undefined, true],
|
||||
['', true],
|
||||
['true', true],
|
||||
['1', true],
|
||||
['yes', true],
|
||||
['false', false],
|
||||
['FALSE', false],
|
||||
[' 0 ', false],
|
||||
])('FLUXER_REPORT_RETENTION_DRY_RUN=%j gives a dry run of %s', (raw, expected) => {
|
||||
expect(resolveReportRetentionDryRun(raw)).toBe(expected);
|
||||
function run(options: Partial<ReportRetentionOptions> = {}) {
|
||||
return processReportRetention(deps, {now: NOW, dryRun: false, policy: DEFAULT_POLICY, ...options});
|
||||
}
|
||||
|
||||
it('deletes earlier when the retention period is shorter', async () => {
|
||||
const expired = await seed({ageDays: 30, profile: true});
|
||||
const kept = await seed({ageDays: 29, profile: true});
|
||||
|
||||
const summary = await run({policy: {retentionDays: 30, resolvedRetentionDays: null}});
|
||||
|
||||
await expectGone(expired);
|
||||
await expectKept(kept);
|
||||
expect(summary).toMatchObject({scanned: 2, expired: 1, deleted: 1});
|
||||
});
|
||||
|
||||
it('deletes a resolved report once the resolved-report period has passed since it was resolved', async () => {
|
||||
const policy = {retentionDays: 365, resolvedRetentionDays: 30};
|
||||
const resolvedLongAgo = await seed({ageDays: 100, resolvedDaysAgo: 30, attachment: newAttachment()});
|
||||
const resolvedRecently = await seed({ageDays: 100, resolvedDaysAgo: 29, profile: true});
|
||||
const pending = await seed({ageDays: 100, profile: true});
|
||||
const resolvedWithoutTime = await seed({ageDays: 100, status: 1, profile: true});
|
||||
|
||||
const summary = await run({policy});
|
||||
|
||||
await expectGone(resolvedLongAgo);
|
||||
expect(storage.hasObject(REPORTS_BUCKET, resolvedLongAgo.attachmentKey!)).toBe(false);
|
||||
await expectKept(resolvedRecently);
|
||||
await expectKept(pending);
|
||||
await expectKept(resolvedWithoutTime);
|
||||
expect(summary).toMatchObject({scanned: 4, expired: 1, deleted: 1, held: 0});
|
||||
});
|
||||
|
||||
it('applies whichever rule ends first', async () => {
|
||||
const policy = {retentionDays: 100, resolvedRetentionDays: 300};
|
||||
const byAge = await seed({ageDays: 100, resolvedDaysAgo: 10});
|
||||
const neither = await seed({ageDays: 99, resolvedDaysAgo: 90});
|
||||
|
||||
await run({policy});
|
||||
|
||||
await expectGone(byAge);
|
||||
await expectKept(neither);
|
||||
});
|
||||
|
||||
it('keeps a held resolved report past the resolved-report period and keeps its evidence', async () => {
|
||||
const shared = newAttachment();
|
||||
const held = await seed({
|
||||
ageDays: 50,
|
||||
resolvedDaysAgo: 40,
|
||||
attachment: shared,
|
||||
profile: true,
|
||||
legalHoldUntil: new Date(NOW.getTime() + DAY_MS),
|
||||
});
|
||||
const sharingWithHeld = await seed({ageDays: 50, resolvedDaysAgo: 40, attachment: shared});
|
||||
|
||||
const summary = await run({policy: {retentionDays: 365, resolvedRetentionDays: 7}});
|
||||
|
||||
await expectKept(held);
|
||||
await expectGone(sharingWithHeld);
|
||||
expect(storage.hasObject(REPORTS_BUCKET, attachmentKey(shared))).toBe(true);
|
||||
expect(summary).toMatchObject({expired: 1, held: 1, deleted: 1, sharedObjectsKept: 1});
|
||||
});
|
||||
|
||||
it('frees the reporter to report the same account again once the report is deleted', async () => {
|
||||
const reporterId = nextId();
|
||||
const reportedUserId = nextId();
|
||||
const reportId = createReportID(nextId());
|
||||
await repository.createReport({
|
||||
...Object.fromEntries(IAR_SUBMISSION_COLUMNS.map((column) => [column, null])),
|
||||
report_id: BigInt(reportId),
|
||||
reporter_id: reporterId,
|
||||
reported_user_id: reportedUserId,
|
||||
reported_at: daysAgo(400),
|
||||
status: 0,
|
||||
report_type: 1,
|
||||
category: 'harassment',
|
||||
} as IARSubmissionRow);
|
||||
const reservation = {
|
||||
reporter_id: createUserID(reporterId),
|
||||
reported_user_id: createUserID(reportedUserId),
|
||||
report_id: reportId,
|
||||
reported_at: daysAgo(400),
|
||||
};
|
||||
expect(await repository.reserveUserReportByReporter(reservation)).toBe(true);
|
||||
|
||||
await run();
|
||||
|
||||
expect(await repository.getReport(reportId)).toBeNull();
|
||||
expect(await repository.reserveUserReportByReporter({...reservation, report_id: createReportID(nextId())})).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
describe('the daily task', () => {
|
||||
const configured = {...Config.reportRetention};
|
||||
|
||||
beforeEach(() => {
|
||||
setWorkerDependenciesForTest({reportRepository: repository, storageService: storage});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
Object.assign(Config.reportRetention, configured);
|
||||
clearWorkerDependencies();
|
||||
});
|
||||
|
||||
async function runDailyTask() {
|
||||
vi.useFakeTimers({now: NOW, toFake: ['Date']});
|
||||
await expireReportSnapshots({}, WORKER_HELPERS);
|
||||
}
|
||||
|
||||
it('deletes for real with the default settings', async () => {
|
||||
expect(configured).toEqual({days: 365, resolvedDays: null, dryRun: false});
|
||||
const expired = await seed({ageDays: 366, attachment: newAttachment(), profile: true});
|
||||
const kept = await seed({ageDays: 364, profile: true});
|
||||
|
||||
await runDailyTask();
|
||||
|
||||
expect(await repository.getReport(expired.reportId)).toBeNull();
|
||||
expect(storage.hasObject(REPORTS_BUCKET, expired.attachmentKey!)).toBe(false);
|
||||
expect(storage.hasObject(REPORTS_BUCKET, expired.profileKey!)).toBe(false);
|
||||
expect(await repository.getReport(kept.reportId)).not.toBeNull();
|
||||
});
|
||||
|
||||
it('only counts what it would delete when FLUXER_REPORT_RETENTION_DRY_RUN is true', async () => {
|
||||
Object.assign(Config.reportRetention, {dryRun: true});
|
||||
const expired = await seed({ageDays: 366, attachment: newAttachment(), profile: true});
|
||||
|
||||
await runDailyTask();
|
||||
|
||||
expect(await repository.getReport(expired.reportId)).not.toBeNull();
|
||||
expect(storage.deleteObjectSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('reads both day settings from the config', async () => {
|
||||
Object.assign(Config.reportRetention, {days: 60, resolvedDays: 5});
|
||||
const byAge = await seed({ageDays: 60});
|
||||
const byResolution = await seed({ageDays: 20, resolvedDaysAgo: 5});
|
||||
const kept = await seed({ageDays: 20, resolvedDaysAgo: 4});
|
||||
|
||||
await runDailyTask();
|
||||
|
||||
expect(await repository.getReport(byAge.reportId)).toBeNull();
|
||||
expect(await repository.getReport(byResolution.reportId)).toBeNull();
|
||||
expect(await repository.getReport(kept.reportId)).not.toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user