fix(media): stream large files and honour size settings (#3387)

fluxer_api/src/api/attachment/AttachmentDecayReconcileRun.ts

@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {randomUUID} from 'node:crypto';
+import {Config} from '@app/api/Config';
+import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig';
+import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
+import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';
+import {seconds} from 'itty-time';
+
+const RUN_KEY = 'attachment_decay:reconcile_run';
+const RUN_TTL_SECONDS = seconds('1 day');
+
+export function attachmentDecayReconcileJobKey(runId: string, chunk: number): string {
+	return `${runId}:${chunk}`;
+}
+
+export async function startAttachmentDecayReconcile(
+	kvClient: IKVProvider,
+	workerService: IWorkerService<WorkerTaskName>,
+): Promise<void> {
+	if (!Config.instance.selfHosted) return;
+	const runId = randomUUID();
+	await kvClient.setex(RUN_KEY, RUN_TTL_SECONDS, runId);
+	await workerService.addJob(
+		'reconcileAttachmentDecay',
+		{runId, chunk: 0},
+		{jobKey: attachmentDecayReconcileJobKey(runId, 0)},
+	);
+}
+
+export async function holdAttachmentDecayReconcileRun(kvClient: IKVProvider, runId: string): Promise<boolean> {
+	return (
+		(await kvClient.extendLock(RUN_KEY, runId, RUN_TTL_SECONDS)) ||
+		(await kvClient.acquireLock(RUN_KEY, runId, RUN_TTL_SECONDS))
+	);
+}
+
+export async function finishAttachmentDecayReconcileRun(kvClient: IKVProvider, runId: string): Promise<void> {
+	await kvClient.releaseLock(RUN_KEY, runId);
+}
+
+export async function isAttachmentDecayReconcileRunning(kvClient: IKVProvider): Promise<boolean> {
+	if (!Config.instance.selfHosted) return false;
+	return (await kvClient.exists(RUN_KEY)) > 0;
+}

fluxer_api/src/api/attachment/AttachmentDecayRepository.ts

@@ -7,9 +7,11 @@ import {
 	fetchMany,
 	fetchManyInChunks,
 	fetchOne,
+	fetchPage,
 } from '@app/api/database/CassandraQueryExecution';
-import {AttachmentDecayByExpiry, AttachmentDecayById} from '@app/api/Tables';
+import {AttachmentDecayByExpiry, AttachmentDecayById, ChannelMessageBuckets} from '@app/api/Tables';
 import type {AttachmentDecayRow} from '@app/api/types/AttachmentDecayTypes';
+import {BACKGROUND_READ_TIMEOUT_MS} from '@pkgs/cassandra/src/Client';
 
 interface AttachmentDecayExpiryRow {
 	expiry_bucket: number;
@@ -26,6 +28,7 @@ const FETCH_BY_ID_CQL = AttachmentDecayById.selectCql({
 const FETCH_BY_IDS_CQL = AttachmentDecayById.selectCql({
 	where: AttachmentDecayById.where.in('attachment_id', 'attachment_ids'),
 });
+const SCAN_CHANNELS_WITH_MESSAGES_CQL = ChannelMessageBuckets.selectCql({columns: ['channel_id']});
 const createFetchExpiredByBucketQuery = (limit: number) =>
 	AttachmentDecayByExpiry.select({
 		where: [
@@ -74,6 +77,18 @@ export class AttachmentDecayRepository {
 		return map;
 	}
 
+	async scanChannelsWithMessagesPage(
+		limit: number,
+		pageState: string | null,
+	): Promise<{channelIds: Array<ChannelID>; pageState: string | null}> {
+		const page = await fetchPage<{channel_id: ChannelID}>(
+			SCAN_CHANNELS_WITH_MESSAGES_CQL,
+			{},
+			{pageSize: limit, pageState, readTimeout: BACKGROUND_READ_TIMEOUT_MS},
+		);
+		return {channelIds: [...new Set(page.rows.map((row) => row.channel_id))], pageState: page.pageState};
+	}
+
 	async fetchExpiredByBucket(bucket: number, currentTime: Date, limit = 200): Promise<Array<AttachmentDecayExpiryRow>> {
 		const query = createFetchExpiredByBucketQuery(limit);
 		return fetchMany(query.bind({expiry_bucket: bucket, current_time: currentTime}));

fluxer_api/src/api/attachment/AttachmentDecayService.test.ts

@@ -0,0 +1,186 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import type {AttachmentDecayRepository} from '@app/api/attachment/AttachmentDecayRepository';
+import {type AttachmentDecayPayload, AttachmentDecayService} from '@app/api/attachment/AttachmentDecayService';
+import type {AttachmentID} from '@app/api/BrandedTypes';
+import {createAttachmentID, createChannelID, createMessageID} from '@app/api/BrandedTypes';
+import type {InstanceAttachmentDecayEffectiveConfig} from '@app/api/instance/InstanceConfigRepository';
+import type {AttachmentDecayRow} from '@app/api/types/AttachmentDecayTypes';
+import {ms} from 'itty-time';
+import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
+
+const NOW = new Date('2026-10-10T00:00:00.000Z');
+const ATTACHMENT_ID = createAttachmentID(7001n);
+const CONFIG: InstanceAttachmentDecayEffectiveConfig = {
+	enabled: true,
+	min_size_mb: 5,
+	max_size_mb: 500,
+	max_eligible_size_mb: 500,
+	min_lifetime_days: 14,
+	max_lifetime_days: 365,
+	curve: 0.5,
+	renew_threshold_days: 30,
+	renew_window_days: 30,
+};
+
+function daysFromNow(days: number): Date {
+	return new Date(NOW.getTime() + days * ms('1 day'));
+}
+
+function payload(uploadedAt: Date, sizeBytes = 1024n): AttachmentDecayPayload {
+	return {
+		attachmentId: ATTACHMENT_ID,
+		channelId: createChannelID(7002n),
+		messageId: createMessageID(7003n),
+		filename: 'file.png',
+		sizeBytes,
+		uploadedAt,
+	};
+}
+
+function createService(config: InstanceAttachmentDecayEffectiveConfig = CONFIG) {
+	const rows = new Map<AttachmentID, AttachmentDecayRow>();
+	const repo = {
+		async fetchByIds(ids: Array<AttachmentID>) {
+			return new Map(ids.flatMap((id) => (rows.has(id) ? [[id, rows.get(id)!] as const] : [])));
+		},
+		async upsert(record: AttachmentDecayRow) {
+			rows.set(record.attachment_id, record);
+		},
+		async deleteRecords({attachment_id}: {attachment_id: AttachmentID}) {
+			rows.delete(attachment_id);
+		},
+	} as unknown as AttachmentDecayRepository;
+	return {rows, service: new AttachmentDecayService(repo, async () => config)};
+}
+
+async function fileExists(): Promise<boolean> {
+	return true;
+}
+
+async function fileMissing(): Promise<boolean> {
+	return false;
+}
+
+function seed(rows: Map<AttachmentID, AttachmentDecayRow>, uploadedAt: Date, expiresAt: Date): void {
+	rows.set(ATTACHMENT_ID, {
+		attachment_id: ATTACHMENT_ID,
+		channel_id: createChannelID(7002n),
+		message_id: createMessageID(7003n),
+		filename: 'file.png',
+		size_bytes: 1024n,
+		uploaded_at: uploadedAt,
+		expires_at: expiresAt,
+		last_accessed_at: uploadedAt,
+		cost: 1,
+		lifetime_days: 1,
+		status: null,
+	});
+}
+
+describe('AttachmentDecayService.reconcile', () => {
+	beforeEach(() => {
+		vi.useFakeTimers();
+		vi.setSystemTime(NOW);
+	});
+
+	afterEach(() => {
+		vi.useRealTimers();
+	});
+
+	it('writes nothing while expiry is off', async () => {
+		const {rows, service} = createService({...CONFIG, enabled: false});
+
+		expect(await service.reconcile([payload(daysFromNow(-10))], fileExists)).toBe(0);
+		expect(rows.size).toBe(0);
+	});
+
+	it('gives an untracked attachment the expiry its upload date implies', async () => {
+		const {rows, service} = createService();
+
+		expect(await service.reconcile([payload(daysFromNow(-100))], fileExists)).toBe(1);
+		expect(rows.get(ATTACHMENT_ID)?.expires_at).toEqual(daysFromNow(265));
+	});
+
+	it('gives an untracked attachment that is already overdue the renew window', async () => {
+		const {rows, service} = createService();
+
+		await service.reconcile([payload(daysFromNow(-1000))], fileExists);
+
+		expect(rows.get(ATTACHMENT_ID)?.expires_at).toEqual(daysFromNow(30));
+	});
+
+	it('gives an attachment whose expiry passed unnoticed the renew window', async () => {
+		const {rows, service} = createService();
+		seed(rows, daysFromNow(-1000), daysFromNow(-200));
+
+		await service.reconcile([payload(daysFromNow(-1000))], fileExists);
+
+		expect(rows.get(ATTACHMENT_ID)?.expires_at).toEqual(daysFromNow(30));
+	});
+
+	it('leaves an untracked attachment alone when its file is gone', async () => {
+		const {rows, service} = createService();
+
+		expect(await service.reconcile([payload(daysFromNow(-100))], fileMissing)).toBe(0);
+		expect(rows.size).toBe(0);
+	});
+
+	it('keeps a passed expiry when the file is gone', async () => {
+		const {rows, service} = createService();
+		seed(rows, daysFromNow(-1000), daysFromNow(-200));
+
+		expect(await service.reconcile([payload(daysFromNow(-1000))], fileMissing)).toBe(0);
+		expect(rows.get(ATTACHMENT_ID)?.expires_at).toEqual(daysFromNow(-200));
+	});
+
+	it('does not look for the file of an attachment that is still pending', async () => {
+		const {rows, service} = createService({...CONFIG, max_lifetime_days: 730});
+		seed(rows, daysFromNow(-100), daysFromNow(265));
+
+		await service.reconcile([payload(daysFromNow(-100))], fileMissing);
+
+		expect(rows.get(ATTACHMENT_ID)?.expires_at).toEqual(daysFromNow(630));
+	});
+
+	it('extends a tracked attachment when the lifetime grows', async () => {
+		const {rows, service} = createService({...CONFIG, max_lifetime_days: 730});
+		seed(rows, daysFromNow(-100), daysFromNow(265));
+
+		await service.reconcile([payload(daysFromNow(-100))], fileExists);
+
+		expect(rows.get(ATTACHMENT_ID)?.expires_at).toEqual(daysFromNow(630));
+	});
+
+	it('shortens a tracked attachment no further than the renew window', async () => {
+		const {rows, service} = createService({...CONFIG, max_lifetime_days: 60});
+		seed(rows, daysFromNow(-100), daysFromNow(265));
+
+		await service.reconcile([payload(daysFromNow(-100))], fileExists);
+
+		expect(rows.get(ATTACHMENT_ID)?.expires_at).toEqual(daysFromNow(30));
+	});
+
+	it('keeps an expiry that already falls inside the renew window', async () => {
+		const {rows, service} = createService({...CONFIG, max_lifetime_days: 60});
+		seed(rows, daysFromNow(-100), daysFromNow(12));
+
+		expect(await service.reconcile([payload(daysFromNow(-100))], fileExists)).toBe(0);
+		expect(rows.get(ATTACHMENT_ID)?.expires_at).toEqual(daysFromNow(12));
+	});
+
+	it('drops the record of an attachment that is no longer eligible', async () => {
+		const {rows, service} = createService({...CONFIG, max_eligible_size_mb: 1});
+		seed(rows, daysFromNow(-100), daysFromNow(265));
+
+		expect(await service.reconcile([payload(daysFromNow(-100), 2n * 1024n * 1024n)], fileExists)).toBe(1);
+		expect(rows.size).toBe(0);
+	});
+
+	it('settles after one pass', async () => {
+		const {service} = createService({...CONFIG, max_lifetime_days: 60});
+
+		expect(await service.reconcile([payload(daysFromNow(-100))], fileExists)).toBe(1);
+		expect(await service.reconcile([payload(daysFromNow(-100))], fileExists)).toBe(0);
+	});
+});

fluxer_api/src/api/attachment/AttachmentDecayService.ts

@@ -15,7 +15,7 @@ import {
 } from '@app/api/utils/AttachmentDecay';
 import {ms} from 'itty-time';
 
-interface AttachmentDecayPayload {
+export interface AttachmentDecayPayload {
 	attachmentId: AttachmentID;
 	channelId: ChannelID;
 	messageId: MessageID;
@@ -27,7 +27,7 @@ interface AttachmentDecayPayload {
 
 type AttachmentDecayConfigResolver = () => Promise<InstanceAttachmentDecayEffectiveConfig>;
 
-async function resolveDefaultAttachmentDecayConfig(): Promise<InstanceAttachmentDecayEffectiveConfig> {
+export async function resolveDefaultAttachmentDecayConfig(): Promise<InstanceAttachmentDecayEffectiveConfig> {
 	const {getInstanceConfigRepository} = await import('@app/api/middleware/ServiceSingletons');
 	return getInstanceConfigRepository().getEffectiveAttachmentDecayConfig();
 }
@@ -108,26 +108,85 @@ export class AttachmentDecayService {
 					expiresAt = renewed;
 				}
 			}
-			const lifetimeDays = Math.round((expiresAt.getTime() - uploadedAt.getTime()) / ms('1 day'));
-			const cost = computeCost({
-				sizeBytes: attachment.sizeBytes,
-				lifetimeDays,
-				pricePerTBPerMonth: DEFAULT_DECAY_CONSTANTS.PRICE_PER_TB_PER_MONTH,
-			});
-			await this.repo.upsert({
-				attachment_id: attachment.attachmentId,
-				channel_id: attachment.channelId,
-				message_id: attachment.messageId,
-				filename: attachment.filename,
-				size_bytes: attachment.sizeBytes,
-				uploaded_at: uploadedAt,
-				expires_at: expiresAt,
-				last_accessed_at: now,
-				cost,
-				lifetime_days: lifetimeDays,
+			await this.writeRecord(attachment, {
+				uploadedAt,
+				expiresAt,
+				lastAccessedAt: now,
 				status: existing?.status ?? null,
-				expiry_bucket: getExpiryBucket(expiresAt),
 			});
 		}
 	}
+
+	async reconcile(
+		attachments: Array<AttachmentDecayPayload>,
+		fileExists: (attachment: AttachmentDecayPayload) => Promise<boolean>,
+	): Promise<number> {
+		if (attachments.length === 0) return 0;
+		const config = await this.resolveConfig();
+		if (!config.enabled) return 0;
+		const rules = buildDecayRules(config);
+		const existingRecords = await this.repo.fetchByIds(attachments.map((a) => a.attachmentId));
+		const now = new Date();
+		const windowEnd = new Date(now.getTime() + config.renew_window_days * ms('1 day'));
+		const seen = new Set<AttachmentID>();
+		let changed = 0;
+		for (const attachment of attachments) {
+			if (seen.has(attachment.attachmentId)) continue;
+			seen.add(attachment.attachmentId);
+			const existing = existingRecords.get(attachment.attachmentId);
+			const uploadedAt = existing?.uploaded_at ?? attachment.uploadedAt;
+			const decay = computeDecay({sizeBytes: attachment.sizeBytes, uploadedAt, rules});
+			if (!decay) {
+				if (existing) {
+					await this.repo.deleteRecords({
+						expiry_bucket: getExpiryBucket(existing.expires_at),
+						expires_at: existing.expires_at,
+						attachment_id: existing.attachment_id,
+					});
+					changed++;
+				}
+				continue;
+			}
+			const stillPending = existing != null && existing.expires_at.getTime() > now.getTime();
+			if (!stillPending && !(await fileExists(attachment))) continue;
+			const floor =
+				stillPending && existing.expires_at.getTime() < windowEnd.getTime() ? existing.expires_at : windowEnd;
+			const expiresAt = extendExpiry(floor, decay.expiresAt);
+			if (existing && existing.expires_at.getTime() === expiresAt.getTime()) continue;
+			await this.writeRecord(attachment, {
+				uploadedAt,
+				expiresAt,
+				lastAccessedAt: existing?.last_accessed_at ?? uploadedAt,
+				status: existing?.status ?? null,
+			});
+			changed++;
+		}
+		return changed;
+	}
+
+	private async writeRecord(
+		attachment: AttachmentDecayPayload,
+		state: {uploadedAt: Date; expiresAt: Date; lastAccessedAt: Date; status: string | null},
+	): Promise<void> {
+		const lifetimeDays = Math.round((state.expiresAt.getTime() - state.uploadedAt.getTime()) / ms('1 day'));
+		const cost = computeCost({
+			sizeBytes: attachment.sizeBytes,
+			lifetimeDays,
+			pricePerTBPerMonth: DEFAULT_DECAY_CONSTANTS.PRICE_PER_TB_PER_MONTH,
+		});
+		await this.repo.upsert({
+			attachment_id: attachment.attachmentId,
+			channel_id: attachment.channelId,
+			message_id: attachment.messageId,
+			filename: attachment.filename,
+			size_bytes: attachment.sizeBytes,
+			uploaded_at: state.uploadedAt,
+			expires_at: state.expiresAt,
+			last_accessed_at: state.lastAccessedAt,
+			cost,
+			lifetime_days: lifetimeDays,
+			status: state.status,
+			expiry_bucket: getExpiryBucket(state.expiresAt),
+		});
+	}
 }