diff --git a/Cargo.lock b/Cargo.lock index 128d8524b..96ca26596 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1803,6 +1803,7 @@ dependencies = [ "http-body-util", "hyper", "libc", + "memmap2", "moka", "parking_lot", "percent-encoding", @@ -2892,6 +2893,15 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + [[package]] name = "mime" version = "0.3.17" diff --git a/fluxer_admin/src/templates/pages/instance_config.rs b/fluxer_admin/src/templates/pages/instance_config.rs index 9239f9875..dfcc033c3 100644 --- a/fluxer_admin/src/templates/pages/instance_config.rs +++ b/fluxer_admin/src/templates/pages/instance_config.rs @@ -200,7 +200,12 @@ pub fn instance_config_page( "Media & retention", "Attachment expiry rules that can be changed without editing environment variables.", html! { - (media_config_section(base, csrf_token, &instance_config.media)) + (media_config_section( + base, + csrf_token, + &instance_config.media, + instance_config.self_hosted, + )) }, )) (config_group( @@ -752,7 +757,12 @@ fn integrations_config_section( ) } -fn media_config_section(base: &str, csrf_token: &str, media: &InstanceMediaResponse) -> Markup { +fn media_config_section( + base: &str, + csrf_token: &str, + media: &InstanceMediaResponse, + self_hosted: bool, +) -> Markup { let decay = &media.attachment_decay; let effective = &decay.effective; let enabled = decay.enabled.unwrap_or(effective.enabled); @@ -780,9 +790,14 @@ fn media_config_section(base: &str, csrf_token: &str, media: &InstanceMediaRespo .renew_window_days .unwrap_or(effective.renew_window_days) .to_string(); + let description = if self_hosted { + "Expire eligible attachments using size-based lifetimes. This is enabled by default and applies at runtime. On a self-hosted instance, changes also apply to attachments uploaded earlier, which are kept for at least the renew window before a new or shorter expiry deletes them." + } else { + "Expire eligible attachments using size-based lifetimes. This is enabled by default and applies at runtime. Changes apply to attachments uploaded from then on." + }; section_card_with_description( "Media Expiry", - "Expire eligible attachments using size-based lifetimes. This is disabled by default and applies at runtime.", + description, html! { form method="post" action={(base) "/instance-config?action=update_media"} { (csrf_input(csrf_token)) diff --git a/fluxer_api/src/api/admin/controllers/InstanceConfigAdminController.ts b/fluxer_api/src/api/admin/controllers/InstanceConfigAdminController.ts index 85bb92b22..e350acccb 100644 --- a/fluxer_api/src/api/admin/controllers/InstanceConfigAdminController.ts +++ b/fluxer_api/src/api/admin/controllers/InstanceConfigAdminController.ts @@ -2,9 +2,11 @@ import {AdminAuditReadActions} from '@app/api/admin/AdminAuditActions'; import {recordAdminRead, recordAdminWrite} from '@app/api/admin/AdminAuditRecorder'; +import {startAttachmentDecayReconcile} from '@app/api/attachment/AttachmentDecayReconcileRun'; import {createUserID} from '@app/api/BrandedTypes'; import {Config} from '@app/api/Config'; import { + type InstanceAttachmentDecayEffectiveConfig, type InstancePolicyConfig, REGISTRATION_PENDING_APPROVAL_TRAIT, REGISTRATION_REJECTED_TRAIT, @@ -13,6 +15,7 @@ import {deriveSsoRedirectUri, normalizeAndValidateSsoConfig} from '@app/api/inst import {requireAdminACL} from '@app/api/middleware/AdminMiddleware'; import {RateLimitMiddleware} from '@app/api/middleware/RateLimitMiddleware'; import {OpenAPI} from '@app/api/middleware/ResponseTypeMiddleware'; +import {getKVClient, getWorkerService} from '@app/api/middleware/ServiceRegistry'; import { getGatewayRolloutConfigPublisher, getInstanceConfigRepository, @@ -47,6 +50,22 @@ import type {Context} from 'hono'; import {createMiddleware} from 'hono/factory'; const INSTANCE_BRANDING_ENTITY_ID = 0n; +const ATTACHMENT_EXPIRY_RULE_FIELDS = [ + 'enabled', + 'min_size_mb', + 'max_size_mb', + 'max_eligible_size_mb', + 'min_lifetime_days', + 'max_lifetime_days', + 'curve', +] as const satisfies ReadonlyArray; + +function attachmentExpiryRulesChanged( + previous: InstanceAttachmentDecayEffectiveConfig, + next: InstanceAttachmentDecayEffectiveConfig, +): boolean { + return ATTACHMENT_EXPIRY_RULE_FIELDS.some((field) => previous[field] !== next[field]); +} function readOptionalField(value: T, key: K): T[K] | undefined { return Object.hasOwn(value, key) ? value[key] : undefined; @@ -556,6 +575,7 @@ export function InstanceConfigAdminController(app: HonoApp) { }); } if (data.media) { + const previousAttachmentDecay = await instanceConfigRepository.getEffectiveAttachmentDecayConfig(); await instanceConfigRepository.setInstanceMediaConfig({ attachment_decay: data.media.attachment_decay ? omitUndefinedFields({ @@ -571,6 +591,10 @@ export function InstanceConfigAdminController(app: HonoApp) { }) : undefined, }); + const attachmentDecay = await instanceConfigRepository.getEffectiveAttachmentDecayConfig(); + if (attachmentDecay.enabled && attachmentExpiryRulesChanged(previousAttachmentDecay, attachmentDecay)) { + await startAttachmentDecayReconcile(getKVClient(), getWorkerService()); + } } if (data.policy) { await applyInstancePolicyUpdate( diff --git a/fluxer_api/src/api/attachment/AttachmentDecayReconcileRun.ts b/fluxer_api/src/api/attachment/AttachmentDecayReconcileRun.ts new file mode 100644 index 000000000..8a1854db3 --- /dev/null +++ b/fluxer_api/src/api/attachment/AttachmentDecayReconcileRun.ts @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {randomUUID} from 'node:crypto'; +import {Config} from '@app/api/Config'; +import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig'; +import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider'; +import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService'; +import {seconds} from 'itty-time'; + +const RUN_KEY = 'attachment_decay:reconcile_run'; +const RUN_TTL_SECONDS = seconds('1 day'); + +export function attachmentDecayReconcileJobKey(runId: string, chunk: number): string { + return `${runId}:${chunk}`; +} + +export async function startAttachmentDecayReconcile( + kvClient: IKVProvider, + workerService: IWorkerService, +): Promise { + if (!Config.instance.selfHosted) return; + const runId = randomUUID(); + await kvClient.setex(RUN_KEY, RUN_TTL_SECONDS, runId); + await workerService.addJob( + 'reconcileAttachmentDecay', + {runId, chunk: 0}, + {jobKey: attachmentDecayReconcileJobKey(runId, 0)}, + ); +} + +export async function holdAttachmentDecayReconcileRun(kvClient: IKVProvider, runId: string): Promise { + return ( + (await kvClient.extendLock(RUN_KEY, runId, RUN_TTL_SECONDS)) || + (await kvClient.acquireLock(RUN_KEY, runId, RUN_TTL_SECONDS)) + ); +} + +export async function finishAttachmentDecayReconcileRun(kvClient: IKVProvider, runId: string): Promise { + await kvClient.releaseLock(RUN_KEY, runId); +} + +export async function isAttachmentDecayReconcileRunning(kvClient: IKVProvider): Promise { + if (!Config.instance.selfHosted) return false; + return (await kvClient.exists(RUN_KEY)) > 0; +} diff --git a/fluxer_api/src/api/attachment/AttachmentDecayRepository.ts b/fluxer_api/src/api/attachment/AttachmentDecayRepository.ts index 238728679..dcb3a3de5 100644 --- a/fluxer_api/src/api/attachment/AttachmentDecayRepository.ts +++ b/fluxer_api/src/api/attachment/AttachmentDecayRepository.ts @@ -7,9 +7,11 @@ import { fetchMany, fetchManyInChunks, fetchOne, + fetchPage, } from '@app/api/database/CassandraQueryExecution'; -import {AttachmentDecayByExpiry, AttachmentDecayById} from '@app/api/Tables'; +import {AttachmentDecayByExpiry, AttachmentDecayById, ChannelMessageBuckets} from '@app/api/Tables'; import type {AttachmentDecayRow} from '@app/api/types/AttachmentDecayTypes'; +import {BACKGROUND_READ_TIMEOUT_MS} from '@pkgs/cassandra/src/Client'; interface AttachmentDecayExpiryRow { expiry_bucket: number; @@ -26,6 +28,7 @@ const FETCH_BY_ID_CQL = AttachmentDecayById.selectCql({ const FETCH_BY_IDS_CQL = AttachmentDecayById.selectCql({ where: AttachmentDecayById.where.in('attachment_id', 'attachment_ids'), }); +const SCAN_CHANNELS_WITH_MESSAGES_CQL = ChannelMessageBuckets.selectCql({columns: ['channel_id']}); const createFetchExpiredByBucketQuery = (limit: number) => AttachmentDecayByExpiry.select({ where: [ @@ -74,6 +77,18 @@ export class AttachmentDecayRepository { return map; } + async scanChannelsWithMessagesPage( + limit: number, + pageState: string | null, + ): Promise<{channelIds: Array; pageState: string | null}> { + const page = await fetchPage<{channel_id: ChannelID}>( + SCAN_CHANNELS_WITH_MESSAGES_CQL, + {}, + {pageSize: limit, pageState, readTimeout: BACKGROUND_READ_TIMEOUT_MS}, + ); + return {channelIds: [...new Set(page.rows.map((row) => row.channel_id))], pageState: page.pageState}; + } + async fetchExpiredByBucket(bucket: number, currentTime: Date, limit = 200): Promise> { const query = createFetchExpiredByBucketQuery(limit); return fetchMany(query.bind({expiry_bucket: bucket, current_time: currentTime})); diff --git a/fluxer_api/src/api/attachment/AttachmentDecayService.test.ts b/fluxer_api/src/api/attachment/AttachmentDecayService.test.ts new file mode 100644 index 000000000..d98d12666 --- /dev/null +++ b/fluxer_api/src/api/attachment/AttachmentDecayService.test.ts @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {AttachmentDecayRepository} from '@app/api/attachment/AttachmentDecayRepository'; +import {type AttachmentDecayPayload, AttachmentDecayService} from '@app/api/attachment/AttachmentDecayService'; +import type {AttachmentID} from '@app/api/BrandedTypes'; +import {createAttachmentID, createChannelID, createMessageID} from '@app/api/BrandedTypes'; +import type {InstanceAttachmentDecayEffectiveConfig} from '@app/api/instance/InstanceConfigRepository'; +import type {AttachmentDecayRow} from '@app/api/types/AttachmentDecayTypes'; +import {ms} from 'itty-time'; +import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; + +const NOW = new Date('2026-10-10T00:00:00.000Z'); +const ATTACHMENT_ID = createAttachmentID(7001n); +const CONFIG: InstanceAttachmentDecayEffectiveConfig = { + enabled: true, + min_size_mb: 5, + max_size_mb: 500, + max_eligible_size_mb: 500, + min_lifetime_days: 14, + max_lifetime_days: 365, + curve: 0.5, + renew_threshold_days: 30, + renew_window_days: 30, +}; + +function daysFromNow(days: number): Date { + return new Date(NOW.getTime() + days * ms('1 day')); +} + +function payload(uploadedAt: Date, sizeBytes = 1024n): AttachmentDecayPayload { + return { + attachmentId: ATTACHMENT_ID, + channelId: createChannelID(7002n), + messageId: createMessageID(7003n), + filename: 'file.png', + sizeBytes, + uploadedAt, + }; +} + +function createService(config: InstanceAttachmentDecayEffectiveConfig = CONFIG) { + const rows = new Map(); + const repo = { + async fetchByIds(ids: Array) { + return new Map(ids.flatMap((id) => (rows.has(id) ? [[id, rows.get(id)!] as const] : []))); + }, + async upsert(record: AttachmentDecayRow) { + rows.set(record.attachment_id, record); + }, + async deleteRecords({attachment_id}: {attachment_id: AttachmentID}) { + rows.delete(attachment_id); + }, + } as unknown as AttachmentDecayRepository; + return {rows, service: new AttachmentDecayService(repo, async () => config)}; +} + +async function fileExists(): Promise { + return true; +} + +async function fileMissing(): Promise { + return false; +} + +function seed(rows: Map, uploadedAt: Date, expiresAt: Date): void { + rows.set(ATTACHMENT_ID, { + attachment_id: ATTACHMENT_ID, + channel_id: createChannelID(7002n), + message_id: createMessageID(7003n), + filename: 'file.png', + size_bytes: 1024n, + uploaded_at: uploadedAt, + expires_at: expiresAt, + last_accessed_at: uploadedAt, + cost: 1, + lifetime_days: 1, + status: null, + }); +} + +describe('AttachmentDecayService.reconcile', () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('writes nothing while expiry is off', async () => { + const {rows, service} = createService({...CONFIG, enabled: false}); + + expect(await service.reconcile([payload(daysFromNow(-10))], fileExists)).toBe(0); + expect(rows.size).toBe(0); + }); + + it('gives an untracked attachment the expiry its upload date implies', async () => { + const {rows, service} = createService(); + + expect(await service.reconcile([payload(daysFromNow(-100))], fileExists)).toBe(1); + expect(rows.get(ATTACHMENT_ID)?.expires_at).toEqual(daysFromNow(265)); + }); + + it('gives an untracked attachment that is already overdue the renew window', async () => { + const {rows, service} = createService(); + + await service.reconcile([payload(daysFromNow(-1000))], fileExists); + + expect(rows.get(ATTACHMENT_ID)?.expires_at).toEqual(daysFromNow(30)); + }); + + it('gives an attachment whose expiry passed unnoticed the renew window', async () => { + const {rows, service} = createService(); + seed(rows, daysFromNow(-1000), daysFromNow(-200)); + + await service.reconcile([payload(daysFromNow(-1000))], fileExists); + + expect(rows.get(ATTACHMENT_ID)?.expires_at).toEqual(daysFromNow(30)); + }); + + it('leaves an untracked attachment alone when its file is gone', async () => { + const {rows, service} = createService(); + + expect(await service.reconcile([payload(daysFromNow(-100))], fileMissing)).toBe(0); + expect(rows.size).toBe(0); + }); + + it('keeps a passed expiry when the file is gone', async () => { + const {rows, service} = createService(); + seed(rows, daysFromNow(-1000), daysFromNow(-200)); + + expect(await service.reconcile([payload(daysFromNow(-1000))], fileMissing)).toBe(0); + expect(rows.get(ATTACHMENT_ID)?.expires_at).toEqual(daysFromNow(-200)); + }); + + it('does not look for the file of an attachment that is still pending', async () => { + const {rows, service} = createService({...CONFIG, max_lifetime_days: 730}); + seed(rows, daysFromNow(-100), daysFromNow(265)); + + await service.reconcile([payload(daysFromNow(-100))], fileMissing); + + expect(rows.get(ATTACHMENT_ID)?.expires_at).toEqual(daysFromNow(630)); + }); + + it('extends a tracked attachment when the lifetime grows', async () => { + const {rows, service} = createService({...CONFIG, max_lifetime_days: 730}); + seed(rows, daysFromNow(-100), daysFromNow(265)); + + await service.reconcile([payload(daysFromNow(-100))], fileExists); + + expect(rows.get(ATTACHMENT_ID)?.expires_at).toEqual(daysFromNow(630)); + }); + + it('shortens a tracked attachment no further than the renew window', async () => { + const {rows, service} = createService({...CONFIG, max_lifetime_days: 60}); + seed(rows, daysFromNow(-100), daysFromNow(265)); + + await service.reconcile([payload(daysFromNow(-100))], fileExists); + + expect(rows.get(ATTACHMENT_ID)?.expires_at).toEqual(daysFromNow(30)); + }); + + it('keeps an expiry that already falls inside the renew window', async () => { + const {rows, service} = createService({...CONFIG, max_lifetime_days: 60}); + seed(rows, daysFromNow(-100), daysFromNow(12)); + + expect(await service.reconcile([payload(daysFromNow(-100))], fileExists)).toBe(0); + expect(rows.get(ATTACHMENT_ID)?.expires_at).toEqual(daysFromNow(12)); + }); + + it('drops the record of an attachment that is no longer eligible', async () => { + const {rows, service} = createService({...CONFIG, max_eligible_size_mb: 1}); + seed(rows, daysFromNow(-100), daysFromNow(265)); + + expect(await service.reconcile([payload(daysFromNow(-100), 2n * 1024n * 1024n)], fileExists)).toBe(1); + expect(rows.size).toBe(0); + }); + + it('settles after one pass', async () => { + const {service} = createService({...CONFIG, max_lifetime_days: 60}); + + expect(await service.reconcile([payload(daysFromNow(-100))], fileExists)).toBe(1); + expect(await service.reconcile([payload(daysFromNow(-100))], fileExists)).toBe(0); + }); +}); diff --git a/fluxer_api/src/api/attachment/AttachmentDecayService.ts b/fluxer_api/src/api/attachment/AttachmentDecayService.ts index bafee7863..541e792af 100644 --- a/fluxer_api/src/api/attachment/AttachmentDecayService.ts +++ b/fluxer_api/src/api/attachment/AttachmentDecayService.ts @@ -15,7 +15,7 @@ import { } from '@app/api/utils/AttachmentDecay'; import {ms} from 'itty-time'; -interface AttachmentDecayPayload { +export interface AttachmentDecayPayload { attachmentId: AttachmentID; channelId: ChannelID; messageId: MessageID; @@ -27,7 +27,7 @@ interface AttachmentDecayPayload { type AttachmentDecayConfigResolver = () => Promise; -async function resolveDefaultAttachmentDecayConfig(): Promise { +export async function resolveDefaultAttachmentDecayConfig(): Promise { const {getInstanceConfigRepository} = await import('@app/api/middleware/ServiceSingletons'); return getInstanceConfigRepository().getEffectiveAttachmentDecayConfig(); } @@ -108,26 +108,85 @@ export class AttachmentDecayService { expiresAt = renewed; } } - const lifetimeDays = Math.round((expiresAt.getTime() - uploadedAt.getTime()) / ms('1 day')); - const cost = computeCost({ - sizeBytes: attachment.sizeBytes, - lifetimeDays, - pricePerTBPerMonth: DEFAULT_DECAY_CONSTANTS.PRICE_PER_TB_PER_MONTH, - }); - await this.repo.upsert({ - attachment_id: attachment.attachmentId, - channel_id: attachment.channelId, - message_id: attachment.messageId, - filename: attachment.filename, - size_bytes: attachment.sizeBytes, - uploaded_at: uploadedAt, - expires_at: expiresAt, - last_accessed_at: now, - cost, - lifetime_days: lifetimeDays, + await this.writeRecord(attachment, { + uploadedAt, + expiresAt, + lastAccessedAt: now, status: existing?.status ?? null, - expiry_bucket: getExpiryBucket(expiresAt), }); } } + + async reconcile( + attachments: Array, + fileExists: (attachment: AttachmentDecayPayload) => Promise, + ): Promise { + if (attachments.length === 0) return 0; + const config = await this.resolveConfig(); + if (!config.enabled) return 0; + const rules = buildDecayRules(config); + const existingRecords = await this.repo.fetchByIds(attachments.map((a) => a.attachmentId)); + const now = new Date(); + const windowEnd = new Date(now.getTime() + config.renew_window_days * ms('1 day')); + const seen = new Set(); + let changed = 0; + for (const attachment of attachments) { + if (seen.has(attachment.attachmentId)) continue; + seen.add(attachment.attachmentId); + const existing = existingRecords.get(attachment.attachmentId); + const uploadedAt = existing?.uploaded_at ?? attachment.uploadedAt; + const decay = computeDecay({sizeBytes: attachment.sizeBytes, uploadedAt, rules}); + if (!decay) { + if (existing) { + await this.repo.deleteRecords({ + expiry_bucket: getExpiryBucket(existing.expires_at), + expires_at: existing.expires_at, + attachment_id: existing.attachment_id, + }); + changed++; + } + continue; + } + const stillPending = existing != null && existing.expires_at.getTime() > now.getTime(); + if (!stillPending && !(await fileExists(attachment))) continue; + const floor = + stillPending && existing.expires_at.getTime() < windowEnd.getTime() ? existing.expires_at : windowEnd; + const expiresAt = extendExpiry(floor, decay.expiresAt); + if (existing && existing.expires_at.getTime() === expiresAt.getTime()) continue; + await this.writeRecord(attachment, { + uploadedAt, + expiresAt, + lastAccessedAt: existing?.last_accessed_at ?? uploadedAt, + status: existing?.status ?? null, + }); + changed++; + } + return changed; + } + + private async writeRecord( + attachment: AttachmentDecayPayload, + state: {uploadedAt: Date; expiresAt: Date; lastAccessedAt: Date; status: string | null}, + ): Promise { + const lifetimeDays = Math.round((state.expiresAt.getTime() - state.uploadedAt.getTime()) / ms('1 day')); + const cost = computeCost({ + sizeBytes: attachment.sizeBytes, + lifetimeDays, + pricePerTBPerMonth: DEFAULT_DECAY_CONSTANTS.PRICE_PER_TB_PER_MONTH, + }); + await this.repo.upsert({ + attachment_id: attachment.attachmentId, + channel_id: attachment.channelId, + message_id: attachment.messageId, + filename: attachment.filename, + size_bytes: attachment.sizeBytes, + uploaded_at: state.uploadedAt, + expires_at: state.expiresAt, + last_accessed_at: state.lastAccessedAt, + cost, + lifetime_days: lifetimeDays, + status: state.status, + expiry_bucket: getExpiryBucket(state.expiresAt), + }); + } } diff --git a/fluxer_api/src/api/channel/services/message/AttachmentProcessingService.ts b/fluxer_api/src/api/channel/services/message/AttachmentProcessingService.ts index c8b235b48..76e76d396 100644 --- a/fluxer_api/src/api/channel/services/message/AttachmentProcessingService.ts +++ b/fluxer_api/src/api/channel/services/message/AttachmentProcessingService.ts @@ -249,6 +249,7 @@ export class AttachmentProcessingService { uploadFilename: attachment.upload_filename, filename: attachment.filename, nsfwMode, + sizeBytes: uploadedFile.contentLength, }); if (metadata) { applyFinalObjectMetadata = true; @@ -406,14 +407,18 @@ export class AttachmentProcessingService { uploadFilename: string; filename: string; nsfwMode: MediaProxyNsfwMode; + sizeBytes: number; }): Promise { try { - const metadata = await this.mediaService.getMetadata({ - type: 'upload', - upload_filename: params.uploadFilename, - filename: params.filename, - nsfw: params.nsfwMode, - }); + const metadata = await this.mediaService.getMetadata( + { + type: 'upload', + upload_filename: params.uploadFilename, + filename: params.filename, + nsfw: params.nsfwMode, + }, + {sizeBytes: params.sizeBytes}, + ); if (metadata) { return metadata; } diff --git a/fluxer_api/src/api/channel/services/message/MessageHelpers.ts b/fluxer_api/src/api/channel/services/message/MessageHelpers.ts index 3ed416ea3..31e98c403 100644 --- a/fluxer_api/src/api/channel/services/message/MessageHelpers.ts +++ b/fluxer_api/src/api/channel/services/message/MessageHelpers.ts @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +import type {AttachmentDecayPayload} from '@app/api/attachment/AttachmentDecayService'; import type {AttachmentID, ChannelID, MessageID, UserID} from '@app/api/BrandedTypes'; import {createAttachmentID, userIdToChannelId} from '@app/api/BrandedTypes'; import {Config} from '@app/api/Config'; @@ -472,6 +473,22 @@ export function collectMessageAttachments(message: Message): Array { return [...message.attachments, ...message.messageSnapshots.flatMap((snapshot) => snapshot.attachments)]; } +export function attachmentDecayPayloadsForMessage(message: Message): Array { + const attachments = collectMessageAttachments(message); + if (attachments.length === 0) return []; + const ownerMessageId = isCrosspostCopy(message) ? (message.reference?.messageId ?? message.id) : message.id; + const uploadedAt = snowflakeToDate(ownerMessageId); + const storageChannelId = attachmentStorageChannelId(message); + return attachments.map((attachment) => ({ + attachmentId: attachment.id, + channelId: storageChannelId, + messageId: ownerMessageId, + filename: attachment.filename, + sizeBytes: attachment.size, + uploadedAt, + })); +} + export function countedThreadMessages(channel: Channel, messageIds: Array): number { if (!channel.isThread()) return 0; const starterId = channel.id.toString(); diff --git a/fluxer_api/src/api/channel/services/message/MessageResponseDataService.test.ts b/fluxer_api/src/api/channel/services/message/MessageResponseDataService.test.ts index 3e8590b32..455032ab7 100644 --- a/fluxer_api/src/api/channel/services/message/MessageResponseDataService.test.ts +++ b/fluxer_api/src/api/channel/services/message/MessageResponseDataService.test.ts @@ -5,11 +5,15 @@ import { MESSAGE_BUILD_BATCH_MAX_BYTES, MessageResponseDataService, } from '@app/api/channel/services/message/MessageResponseDataService'; +import { + type InstanceAttachmentDecayEffectiveConfig, + InstanceConfigRepository, +} from '@app/api/instance/InstanceConfigRepository'; import {Message} from '@app/api/models/Message'; import {MessageTypes} from '@fluxer/constants/src/ChannelConstants'; import type {NatsConnection} from '@nats-io/transport-node'; import type {INatsConnectionManager} from '@pkgs/nats/src/INatsConnectionManager'; -import {describe, expect, it} from 'vitest'; +import {afterEach, describe, expect, it, vi} from 'vitest'; const encoder = new TextEncoder(); const decoder = new TextDecoder(); @@ -129,6 +133,10 @@ function batchSizes(connectionManager: FakeConnectionManager): Array { } describe('MessageResponseDataService', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + it('omits reactions from broadcast message response requests', async () => { const connectionManager = new FakeConnectionManager(); const service = new MessageResponseDataService(connectionManager); @@ -161,6 +169,19 @@ describe('MessageResponseDataService', () => { }); }); + it('tells the shard whether attachment expiry is on', async () => { + const connectionManager = new FakeConnectionManager(); + const service = new MessageResponseDataService(connectionManager); + const decayConfig = vi.spyOn(InstanceConfigRepository.prototype, 'getEffectiveAttachmentDecayConfig'); + + for (const enabled of [true, false]) { + decayConfig.mockResolvedValue({enabled} as InstanceAttachmentDecayEffectiveConfig); + await service.buildMessageForChannel({channel: {guildId: null}, message: makeMessage()}); + } + + expect(connectionManager.payloads.map((payload) => payload.attachment_decay)).toEqual([true, false]); + }); + it('waits longer than the router shard timeout so the inner hop expires first', async () => { const connectionManager = new FakeConnectionManager(); const service = new MessageResponseDataService(connectionManager); diff --git a/fluxer_api/src/api/channel/services/message/MessageResponseDataService.ts b/fluxer_api/src/api/channel/services/message/MessageResponseDataService.ts index e33255a8c..a443791f6 100644 --- a/fluxer_api/src/api/channel/services/message/MessageResponseDataService.ts +++ b/fluxer_api/src/api/channel/services/message/MessageResponseDataService.ts @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +import {resolveDefaultAttachmentDecayConfig} from '@app/api/attachment/AttachmentDecayService'; import type {ChannelID, GuildID, MessageID, UserID} from '@app/api/BrandedTypes'; import {createUserID} from '@app/api/BrandedTypes'; import {Config} from '@app/api/Config'; @@ -49,6 +50,10 @@ type MessageServiceResponse = } | 'NotFound'; +async function attachmentDecayEnabled(): Promise { + return (await resolveDefaultAttachmentDecayConfig()).enabled; +} + function isMessageResponse(value: unknown): value is MessageResponse { return isJsonRecord(value) && typeof value.id === 'string'; } @@ -113,6 +118,7 @@ export class MessageResponseDataService { media_endpoint: Config.endpoints.media, media_proxy_secret_key: Config.mediaProxy.secretKey, attachment_url_secret_base64: Config.mediaProxy.attachmentUrls.secretsBase64[0], + attachment_decay: await attachmentDecayEnabled(), include_reactions: true, }); if (typeof response === 'object' && 'FoundApiMany' in response) { @@ -157,6 +163,7 @@ export class MessageResponseDataService { media_endpoint: Config.endpoints.media, media_proxy_secret_key: Config.mediaProxy.secretKey, attachment_url_secret_base64: Config.mediaProxy.attachmentUrls.secretsBase64[0], + attachment_decay: await attachmentDecayEnabled(), include_reactions: true, nonce: params.nonce, tts: params.tts, @@ -189,6 +196,7 @@ export class MessageResponseDataService { media_endpoint: Config.endpoints.media, media_proxy_secret_key: Config.mediaProxy.secretKey, attachment_url_secret_base64: Config.mediaProxy.attachmentUrls.secretsBase64[0], + attachment_decay: await attachmentDecayEnabled(), include_reactions: params.includeReactions ?? true, nonce: params.nonce, tts: params.tts, @@ -258,6 +266,7 @@ export class MessageResponseDataService { media_endpoint: Config.endpoints.media, media_proxy_secret_key: Config.mediaProxy.secretKey, attachment_url_secret_base64: Config.mediaProxy.attachmentUrls.secretsBase64[0], + attachment_decay: await attachmentDecayEnabled(), include_reactions: params.includeReactions ?? true, }); if (typeof response !== 'object' || !('FoundApiMany' in response)) { diff --git a/fluxer_api/src/api/channel/services/message/MessageRetrievalService.ts b/fluxer_api/src/api/channel/services/message/MessageRetrievalService.ts index 6d099dfc8..b118b9333 100644 --- a/fluxer_api/src/api/channel/services/message/MessageRetrievalService.ts +++ b/fluxer_api/src/api/channel/services/message/MessageRetrievalService.ts @@ -1,17 +1,13 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import {AttachmentDecayService} from '@app/api/attachment/AttachmentDecayService'; -import type {AttachmentID, ChannelID, MessageID, UserID} from '@app/api/BrandedTypes'; +import type {ChannelID, MessageID, UserID} from '@app/api/BrandedTypes'; import {mapChannelToResponse} from '@app/api/channel/ChannelMappers'; import type {ChannelRepository} from '@app/api/channel/ChannelRepository'; import type {AuthenticatedChannel} from '@app/api/channel/services/AuthenticatedChannel'; import {getDmChannelIdsForScope} from '@app/api/channel/services/message/DmScopeUtils'; import type {MessageChannelAuthService} from '@app/api/channel/services/message/MessageChannelAuthService'; -import { - attachmentStorageChannelId, - collectMessageAttachments, - isCrosspostCopy, -} from '@app/api/channel/services/message/MessageHelpers'; +import {attachmentDecayPayloadsForMessage} from '@app/api/channel/services/message/MessageHelpers'; import type {MessageProcessingService} from '@app/api/channel/services/message/MessageProcessingService'; import { createMessageResponseDataService, @@ -283,36 +279,11 @@ export class MessageRetrievalService { } private async extendAttachments(messages: Array): Promise { - const payloads = messages.flatMap((message) => { - return this.buildAttachmentDecayEntriesForMessage(message); - }); + const payloads = messages.flatMap(attachmentDecayPayloadsForMessage); if (payloads.length === 0) return; await this.attachmentDecayService.extendForAttachments(payloads); } - private buildAttachmentDecayEntriesForMessage(message: Message): Array<{ - attachmentId: AttachmentID; - channelId: ChannelID; - messageId: MessageID; - filename: string; - sizeBytes: bigint; - uploadedAt: Date; - }> { - const attachments = collectMessageAttachments(message); - if (attachments.length === 0) return []; - const ownerMessageId = isCrosspostCopy(message) ? (message.reference?.messageId ?? message.id) : message.id; - const uploadedAt = snowflakeToDate(ownerMessageId); - const storageChannelId = attachmentStorageChannelId(message); - return attachments.map((attachment) => ({ - attachmentId: attachment.id, - channelId: storageChannelId, - messageId: ownerMessageId, - filename: attachment.filename, - sizeBytes: attachment.size, - uploadedAt, - })); - } - private async applyDmScopeToSearchParams({ userId, channel, diff --git a/fluxer_api/src/api/channel/tests/AttachmentDecay.test.ts b/fluxer_api/src/api/channel/tests/AttachmentDecay.test.ts index ed5f7408f..7fe1e5520 100644 --- a/fluxer_api/src/api/channel/tests/AttachmentDecay.test.ts +++ b/fluxer_api/src/api/channel/tests/AttachmentDecay.test.ts @@ -1,17 +1,35 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import {createTestAccount} from '@app/api/auth/tests/AuthTestUtils'; +import {AttachmentDecayRepository} from '@app/api/attachment/AttachmentDecayRepository'; +import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils'; +import {createAttachmentID, createChannelID, createMessageID} from '@app/api/BrandedTypes'; +import {Config, getConfig} from '@app/api/Config'; import { createChannel, createGuild, loadFixture, sendMessageWithAttachments, } from '@app/api/channel/tests/AttachmentTestUtils'; +import { + getAssetDeletionQueue, + getChannelRepository, + getInstanceConfigRepository, +} from '@app/api/middleware/ServiceSingletons'; import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {NoopWorkerService} from '@app/api/test/NoopWorkerService'; import {HTTP_STATUS} from '@app/api/test/TestConstants'; import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder'; +import {getExpiryBucket} from '@app/api/utils/AttachmentDecay'; +import {processExpiredAttachments} from '@app/api/worker/tasks/ExpireAttachments'; +import reconcileAttachmentDecay, { + type ReconcileAttachmentDecayPayload, + reconcileAttachmentDecayChunk, +} from '@app/api/worker/tasks/ReconcileAttachmentDecay'; +import {clearWorkerDependencies, setWorkerDependenciesForTest} from '@app/api/worker/WorkerContext'; +import {AdminACLs} from '@fluxer/constants/src/AdminACLs'; import {MessageReferenceTypes} from '@fluxer/constants/src/ChannelConstants'; -import {afterEach, beforeEach, describe, expect, test} from 'vitest'; +import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask'; +import {afterEach, beforeEach, describe, expect, test, vi} from 'vitest'; interface AttachmentDecayRow { attachment_id: string; @@ -24,14 +42,325 @@ interface AttachmentDecayRow { status: string | null; } +interface ListedMessage { + id: string; + attachments?: Array<{id: string; url?: string | null; expires_at?: string | null}>; +} + describe('Attachment Decay', () => { let harness: ApiTestHarness; + const originalSelfHosted = getConfig().instance.selfHosted; beforeEach(async () => { + getConfig().instance.selfHosted = true; harness = await createApiTestHarness(); }); afterEach(async () => { + getConfig().instance.selfHosted = originalSelfHosted; + vi.restoreAllMocks(); + clearWorkerDependencies(); await harness?.shutdown(); }); + + async function createInstanceAdmin(): Promise { + return setUserACLs(harness, await createTestAccount(harness), [ + AdminACLs.AUTHENTICATE, + AdminACLs.INSTANCE_CONFIG_VIEW, + AdminACLs.INSTANCE_CONFIG_UPDATE, + ]); + } + + async function setAttachmentExpiry(admin: TestAccount, attachmentDecay: Record): Promise { + await createBuilder(harness, admin.token) + .patch('/admin/instance/config') + .body({media: {attachment_decay: attachmentDecay}}) + .expect(HTTP_STATUS.OK) + .execute(); + } + + async function listedAttachment(token: string, channelId: string, messageId: string) { + const messages = await createBuilder>(harness, token) + .get(`/channels/${channelId}/messages?limit=10`) + .execute(); + return messages.find((message) => message.id === messageId)?.attachments?.[0]; + } + + async function fetchDecayRow(attachmentId: string): Promise { + const {row} = await createBuilderWithoutAuth<{row: AttachmentDecayRow | null}>(harness) + .get(`/test/attachment-decay/${attachmentId}`) + .execute(); + return row; + } + + function installWorkerDependencies(): void { + setWorkerDependenciesForTest({ + assetDeletionQueue: getAssetDeletionQueue(), + channelRepository: getChannelRepository(), + instanceConfigRepository: getInstanceConfigRepository(), + kvClient: harness.kvProvider, + storageService: harness.storageService, + }); + } + + async function enableExpiryAndTakeReconcile(admin: TestAccount): Promise { + const addJob = vi.spyOn(NoopWorkerService.prototype, 'addJob'); + await setAttachmentExpiry(admin, {enabled: true}); + expect(addJob).toHaveBeenCalledTimes(1); + const [task, payload, options] = addJob.mock.calls[0] as unknown as [ + string, + ReconcileAttachmentDecayPayload, + {jobKey: string}, + ]; + expect(task).toBe('reconcileAttachmentDecay'); + expect(payload).toEqual({runId: expect.any(String), chunk: 0}); + expect(options).toEqual({jobKey: `${payload.runId}:0`}); + addJob.mockRestore(); + return payload; + } + + async function runReconcile( + first: ReconcileAttachmentDecayPayload, + limits?: {messagePageSize: number; messagePages: number}, + ): Promise { + let chunks = 0; + let next: ReconcileAttachmentDecayPayload | null = first; + while (next !== null) { + next = await reconcileAttachmentDecayChunk(next, limits); + chunks++; + } + return chunks; + } + + async function uploadWhileExpiryIsOff(token: string, channelId: string, filename: string) { + const {json} = await sendMessageWithAttachments( + harness, + token, + channelId, + {content: filename, attachments: [{id: 0, filename}]}, + [{index: 0, filename, data: loadFixture('yeah.png')}], + ); + return {messageId: json.id, attachmentId: json.attachments![0].id}; + } + + test('should stop reporting an expiry once attachment expiry is turned off', async () => { + const admin = await createInstanceAdmin(); + const guild = await createGuild(harness, admin.token, 'Expiry Off Guild'); + const channelId = guild.system_channel_id!; + await setAttachmentExpiry(admin, {enabled: true}); + const {json} = await sendMessageWithAttachments( + harness, + admin.token, + channelId, + {content: 'Uploaded while expiry is on', attachments: [{id: 0, filename: 'on.png'}]}, + [{index: 0, filename: 'on.png', data: loadFixture('yeah.png')}], + ); + expect((await listedAttachment(admin.token, channelId, json.id))?.expires_at).toBeTruthy(); + + await setAttachmentExpiry(admin, {enabled: false}); + + const listed = await listedAttachment(admin.token, channelId, json.id); + expect(listed?.expires_at ?? null).toBeNull(); + expect(listed?.url).toBeTruthy(); + expect(await fetchDecayRow(json.attachments![0].id)).not.toBeNull(); + }); + test('should apply an expiry to earlier attachments once attachment expiry is turned on', async () => { + const admin = await createInstanceAdmin(); + const guild = await createGuild(harness, admin.token, 'Expiry On Guild'); + const channelId = guild.system_channel_id!; + await setAttachmentExpiry(admin, {enabled: false}); + const {json} = await sendMessageWithAttachments( + harness, + admin.token, + channelId, + {content: 'Uploaded while expiry is off', attachments: [{id: 0, filename: 'off.png'}]}, + [{index: 0, filename: 'off.png', data: loadFixture('yeah.png')}], + ); + const attachmentId = json.attachments![0].id; + expect(await fetchDecayRow(attachmentId)).toBeNull(); + + const reconcile = await enableExpiryAndTakeReconcile(admin); + installWorkerDependencies(); + expect(await runReconcile(reconcile)).toBe(1); + const row = await fetchDecayRow(attachmentId); + expect(row?.channel_id).toBe(channelId); + expect(row?.message_id).toBe(json.id); + expect(new Date(row!.expires_at).getTime()).toBeGreaterThan(Date.now()); + }); + test('should only queue a reconcile when the expiry rules change', async () => { + const admin = await createInstanceAdmin(); + await setAttachmentExpiry(admin, {enabled: true, max_lifetime_days: 400}); + const addJob = vi.spyOn(NoopWorkerService.prototype, 'addJob'); + + await setAttachmentExpiry(admin, {enabled: true, max_lifetime_days: 400, renew_window_days: 10}); + expect(addJob).not.toHaveBeenCalled(); + + await setAttachmentExpiry(admin, {max_lifetime_days: 200}); + expect(addJob).toHaveBeenCalledTimes(1); + + await setAttachmentExpiry(admin, {enabled: false}); + expect(addJob).toHaveBeenCalledTimes(1); + }); + test('should not queue a reconcile on the hosted instance', async () => { + const admin = await createInstanceAdmin(); + await setAttachmentExpiry(admin, {enabled: false}); + getConfig().instance.selfHosted = false; + const addJob = vi.spyOn(NoopWorkerService.prototype, 'addJob'); + + await setAttachmentExpiry(admin, {enabled: true}); + await setAttachmentExpiry(admin, {max_lifetime_days: 200}); + + expect(addJob).not.toHaveBeenCalled(); + expect(await harness.kvProvider.exists('attachment_decay:reconcile_run')).toBe(0); + }); + test('should not run a queued reconcile job on the hosted instance', async () => { + const admin = await createInstanceAdmin(); + const guild = await createGuild(harness, admin.token, 'Hosted Guild'); + await setAttachmentExpiry(admin, {enabled: false}); + const upload = await uploadWhileExpiryIsOff(admin.token, guild.system_channel_id!, 'hosted.png'); + const reconcile = await enableExpiryAndTakeReconcile(admin); + installWorkerDependencies(); + getConfig().instance.selfHosted = false; + const addJob = vi.fn(); + + await reconcileAttachmentDecay({...reconcile, __jobId: '1'}, {addJob} as unknown as WorkerTaskHelpers); + + expect(addJob).not.toHaveBeenCalled(); + expect(await fetchDecayRow(upload.attachmentId)).toBeNull(); + getConfig().instance.selfHosted = true; + expect(await runReconcile(reconcile)).toBe(1); + expect(await fetchDecayRow(upload.attachmentId)).not.toBeNull(); + }); + test('should reconcile in resumable chunks', async () => { + const admin = await createInstanceAdmin(); + const guild = await createGuild(harness, admin.token, 'Chunked Guild'); + const second = await createChannel(harness, admin.token, guild.id, 'second'); + await setAttachmentExpiry(admin, {enabled: false}); + const uploads = [ + await uploadWhileExpiryIsOff(admin.token, guild.system_channel_id!, 'a.png'), + await uploadWhileExpiryIsOff(admin.token, guild.system_channel_id!, 'b.png'), + await uploadWhileExpiryIsOff(admin.token, guild.system_channel_id!, 'c.png'), + await uploadWhileExpiryIsOff(admin.token, second.id, 'd.png'), + await uploadWhileExpiryIsOff(admin.token, second.id, 'e.png'), + ]; + const reconcile = await enableExpiryAndTakeReconcile(admin); + installWorkerDependencies(); + + const next = await reconcileAttachmentDecayChunk(reconcile, {messagePageSize: 1, messagePages: 1}); + + expect(next).toMatchObject({runId: reconcile.runId, chunk: 1, cursor: {beforeMessageId: expect.any(String)}}); + const tracked = await Promise.all(uploads.map((upload) => fetchDecayRow(upload.attachmentId))); + expect(tracked.filter((row) => row !== null)).toHaveLength(1); + expect(await runReconcile(next!, {messagePageSize: 1, messagePages: 1})).toBeGreaterThan(3); + for (const upload of uploads) { + expect((await fetchDecayRow(upload.attachmentId))?.message_id).toBe(upload.messageId); + } + }); + test('should run a queued reconcile job to the end', async () => { + const admin = await createInstanceAdmin(); + const guild = await createGuild(harness, admin.token, 'Queued Job Guild'); + await setAttachmentExpiry(admin, {enabled: false}); + const upload = await uploadWhileExpiryIsOff(admin.token, guild.system_channel_id!, 'queued.png'); + const reconcile = await enableExpiryAndTakeReconcile(admin); + installWorkerDependencies(); + const addJob = vi.fn(); + + await reconcileAttachmentDecay({...reconcile, __jobId: '1'}, {addJob} as unknown as WorkerTaskHelpers); + + expect(addJob).not.toHaveBeenCalled(); + expect(await fetchDecayRow(upload.attachmentId)).not.toBeNull(); + expect(await harness.kvProvider.exists('attachment_decay:reconcile_run')).toBe(0); + }); + test('should stop a reconcile once a newer one starts', async () => { + const admin = await createInstanceAdmin(); + const guild = await createGuild(harness, admin.token, 'Superseded Guild'); + await setAttachmentExpiry(admin, {enabled: false}); + const upload = await uploadWhileExpiryIsOff(admin.token, guild.system_channel_id!, 'old.png'); + const first = await enableExpiryAndTakeReconcile(admin); + const addJob = vi.spyOn(NoopWorkerService.prototype, 'addJob'); + await setAttachmentExpiry(admin, {max_lifetime_days: 200}); + expect(addJob).toHaveBeenCalledTimes(1); + installWorkerDependencies(); + + expect(await reconcileAttachmentDecayChunk(first)).toBeNull(); + + expect(await fetchDecayRow(upload.attachmentId)).toBeNull(); + }); + test('should read the stored settings instead of a stale cache', async () => { + const admin = await createInstanceAdmin(); + const guild = await createGuild(harness, admin.token, 'Stale Cache Guild'); + await setAttachmentExpiry(admin, {enabled: false}); + const upload = await uploadWhileExpiryIsOff(admin.token, guild.system_channel_id!, 'stale.png'); + const reconcile = await enableExpiryAndTakeReconcile(admin); + installWorkerDependencies(); + const current = await getInstanceConfigRepository().getEffectiveAttachmentDecayConfig(); + vi.spyOn(getInstanceConfigRepository(), 'getEffectiveAttachmentDecayConfig').mockResolvedValue({ + ...current, + enabled: false, + }); + + await runReconcile(reconcile); + + expect(await fetchDecayRow(upload.attachmentId)).not.toBeNull(); + }); + test('should not give a new expiry to an attachment whose file is gone', async () => { + const admin = await createInstanceAdmin(); + const guild = await createGuild(harness, admin.token, 'Purged Guild'); + const channelId = guild.system_channel_id!; + await setAttachmentExpiry(admin, {enabled: false}); + const purged = await uploadWhileExpiryIsOff(admin.token, channelId, 'purged.png'); + const kept = await uploadWhileExpiryIsOff(admin.token, channelId, 'kept.png'); + await harness.storageService.deleteObject( + Config.s3.buckets.cdn, + `attachments/${channelId}/${purged.attachmentId}/purged.png`, + ); + const reconcile = await enableExpiryAndTakeReconcile(admin); + installWorkerDependencies(); + + await runReconcile(reconcile); + + expect(await fetchDecayRow(purged.attachmentId)).toBeNull(); + expect(await fetchDecayRow(kept.attachmentId)).not.toBeNull(); + }); + test('should not delete expired attachments while a reconcile is running', async () => { + const admin = await createInstanceAdmin(); + const guild = await createGuild(harness, admin.token, 'Paused Expiry Guild'); + const channelId = guild.system_channel_id!; + await setAttachmentExpiry(admin, {enabled: true}); + const {json} = await sendMessageWithAttachments( + harness, + admin.token, + channelId, + {content: 'Lapsed while expiry was off', attachments: [{id: 0, filename: 'lapsed.png'}]}, + [{index: 0, filename: 'lapsed.png', data: loadFixture('yeah.png')}], + ); + const attachmentId = json.attachments![0].id; + await setAttachmentExpiry(admin, {enabled: false}); + const expiresAt = new Date(Date.now() - 60_000); + await new AttachmentDecayRepository().upsert({ + attachment_id: createAttachmentID(BigInt(attachmentId)), + channel_id: createChannelID(BigInt(channelId)), + message_id: createMessageID(BigInt(json.id)), + filename: 'lapsed.png', + size_bytes: 1024n, + uploaded_at: expiresAt, + expires_at: expiresAt, + last_accessed_at: expiresAt, + cost: 1, + lifetime_days: 1, + status: null, + expiry_bucket: getExpiryBucket(expiresAt), + }); + const reconcile = await enableExpiryAndTakeReconcile(admin); + installWorkerDependencies(); + + await processExpiredAttachments(); + + expect(new Date((await fetchDecayRow(attachmentId))!.expires_at).getTime()).toBeLessThan(Date.now()); + await runReconcile(reconcile); + await processExpiredAttachments(); + expect(new Date((await fetchDecayRow(attachmentId))!.expires_at).getTime()).toBeGreaterThan(Date.now()); + const key = `attachments/${channelId}/${attachmentId}/lapsed.png`; + expect(harness.storageService.hasObject(Config.s3.buckets.cdn, key)).toBe(true); + }); test('should create decay metadata when uploading attachment', async () => { const account = await createTestAccount(harness); const guild = await createGuild(harness, account.token, 'Decay Test Guild'); diff --git a/fluxer_api/src/api/channel/tests/CrosspostPropagation.test.ts b/fluxer_api/src/api/channel/tests/CrosspostPropagation.test.ts index bf8583aca..f0e7e355c 100644 --- a/fluxer_api/src/api/channel/tests/CrosspostPropagation.test.ts +++ b/fluxer_api/src/api/channel/tests/CrosspostPropagation.test.ts @@ -713,6 +713,7 @@ describe('Crosspost propagation', () => { queued.push(item); }, } as unknown as IAssetDeletionQueue, + kvClient: harness.kvProvider, instanceConfigRepository: { async getEffectiveAttachmentDecayConfig() { return {enabled: true}; diff --git a/fluxer_api/src/api/infrastructure/AvatarService.ts b/fluxer_api/src/api/infrastructure/AvatarService.ts index c3e0e0754..f1778ac14 100644 --- a/fluxer_api/src/api/infrastructure/AvatarService.ts +++ b/fluxer_api/src/api/infrastructure/AvatarService.ts @@ -95,13 +95,13 @@ export class AvatarService { return null; } const base64Data = base64Image.includes(',') ? base64Image.split(',')[1] : base64Image; - const imageBuffer = new Uint8Array(Buffer.from(base64Data, 'base64')); const maxAvatarSize = this.resolveSizeLimit('avatar_max_size', AVATAR_MAX_SIZE); - if (imageBuffer.length > maxAvatarSize) { + if (Buffer.byteLength(base64Data, 'base64') > maxAvatarSize) { throw InputValidationError.fromCode(errorPath, ValidationErrorCodes.IMAGE_SIZE_EXCEEDS_LIMIT, { maxSize: maxAvatarSize, }); } + const imageBuffer = new Uint8Array(Buffer.from(base64Data, 'base64')); const kind = this.prefixToKind(prefix); const metadata = this.requireAllowedMetadata({ metadata: await this.mediaService.getMetadata({ @@ -143,13 +143,13 @@ export class AvatarService { }> { const {errorPath, base64Image, guildFeatures} = params; const base64Data = base64Image.includes(',') ? base64Image.split(',')[1] : base64Image; - const imageBuffer = new Uint8Array(Buffer.from(base64Data, 'base64')); const maxEmojiSize = this.resolveSizeLimit('emoji_max_size', EMOJI_MAX_SIZE, guildFeatures); - if (imageBuffer.length > maxEmojiSize) { + if (Buffer.byteLength(base64Data, 'base64') > maxEmojiSize) { throw InputValidationError.fromCode(errorPath, ValidationErrorCodes.IMAGE_SIZE_EXCEEDS_LIMIT, { maxSize: maxEmojiSize, }); } + const imageBuffer = new Uint8Array(Buffer.from(base64Data, 'base64')); const metadata = this.requireAllowedMetadata({ metadata: await this.mediaService.getMetadata({ type: 'base64', @@ -244,13 +244,13 @@ export class AvatarService { }> { const {errorPath, base64Image, guildFeatures} = params; const base64Data = base64Image.includes(',') ? base64Image.split(',')[1] : base64Image; - const imageBuffer = new Uint8Array(Buffer.from(base64Data, 'base64')); const maxStickerSize = this.resolveSizeLimit('sticker_max_size', STICKER_MAX_SIZE, guildFeatures); - if (imageBuffer.length > maxStickerSize) { + if (Buffer.byteLength(base64Data, 'base64') > maxStickerSize) { throw InputValidationError.fromCode(errorPath, ValidationErrorCodes.IMAGE_SIZE_EXCEEDS_LIMIT, { maxSize: maxStickerSize, }); } + const imageBuffer = new Uint8Array(Buffer.from(base64Data, 'base64')); const metadata = this.requireAllowedMetadata({ metadata: await this.mediaService.getMetadata({ type: 'base64', diff --git a/fluxer_api/src/api/infrastructure/EntityAssetService.ts b/fluxer_api/src/api/infrastructure/EntityAssetService.ts index 46e8ff9d7..5c6a1a88a 100644 --- a/fluxer_api/src/api/infrastructure/EntityAssetService.ts +++ b/fluxer_api/src/api/infrastructure/EntityAssetService.ts @@ -291,13 +291,13 @@ export class EntityAssetService { animated: boolean; }> { const base64Data = base64Image.includes(',') ? base64Image.split(',')[1] : base64Image; - const imageBuffer = new Uint8Array(Buffer.from(base64Data, 'base64')); const maxAvatarSize = this.resolveAvatarSizeLimit(); - if (imageBuffer.length > maxAvatarSize) { + if (Buffer.byteLength(base64Data, 'base64') > maxAvatarSize) { throw InputValidationError.fromCode(errorPath, ValidationErrorCodes.IMAGE_SIZE_EXCEEDS_LIMIT, { maxSize: maxAvatarSize, }); } + const imageBuffer = new Uint8Array(Buffer.from(base64Data, 'base64')); const kind = this.mapAssetTypeToAssetKind(assetType); const metadata = this.requireAllowedMetadata({ metadata: await this.mediaService.getMetadata({ diff --git a/fluxer_api/src/api/infrastructure/IMediaService.ts b/fluxer_api/src/api/infrastructure/IMediaService.ts index e65aeb512..c20b71db8 100644 --- a/fluxer_api/src/api/infrastructure/IMediaService.ts +++ b/fluxer_api/src/api/infrastructure/IMediaService.ts @@ -42,6 +42,10 @@ export type MediaProxyMetadataRequest = | MediaProxyMetadataBase64Request | MediaProxyMetadataS3Request; +export interface MediaProxyMetadataOptions { + sizeBytes?: number; +} + export interface MediaProxyMetadataResponse { format: string; content_type: string; @@ -83,7 +87,10 @@ export interface MediaProxyFrameResponse { } export abstract class IMediaService { - abstract getMetadata(request: MediaProxyMetadataRequest): Promise; + abstract getMetadata( + request: MediaProxyMetadataRequest, + options?: MediaProxyMetadataOptions, + ): Promise; abstract getExternalMediaProxyURL(url: string): string; diff --git a/fluxer_api/src/api/infrastructure/MediaService.ts b/fluxer_api/src/api/infrastructure/MediaService.ts index 51682a312..ffc5aa456 100644 --- a/fluxer_api/src/api/infrastructure/MediaService.ts +++ b/fluxer_api/src/api/infrastructure/MediaService.ts @@ -7,6 +7,7 @@ import { MEDIA_PROXY_METADATA_REQUEST_VERSION, type MediaProxyFrameRequest, type MediaProxyFrameResponse, + type MediaProxyMetadataOptions, type MediaProxyMetadataRequest, type MediaProxyMetadataResponse, type MediaProxyNsfwMode, @@ -36,6 +37,17 @@ const MEDIA_PROXY_FRAMES_MAX_BYTES = 512 * 1024; const MEDIA_PROXY_SNIFF_MAX_BYTES = 1024; const MEDIA_PROXY_REQUEST_TIMEOUT_MS = ms('30 seconds'); const MEDIA_PROXY_SNIFF_TIMEOUT_MS = ms('2 seconds'); +const MEDIA_PROXY_BUFFERED_INPUT_MAX_BYTES = 500 * 1024 * 1024; +const MEDIA_PROXY_SPOOLED_INPUT_BYTES_PER_SECOND = 16 * 1024 * 1024; + +export function metadataRequestTimeoutMs(sizeBytes: number | undefined): number { + if (sizeBytes === undefined || sizeBytes <= MEDIA_PROXY_BUFFERED_INPUT_MAX_BYTES) { + return MEDIA_PROXY_REQUEST_TIMEOUT_MS; + } + return ( + MEDIA_PROXY_REQUEST_TIMEOUT_MS + Math.ceil(sizeBytes / MEDIA_PROXY_SPOOLED_INPUT_BYTES_PER_SECOND) * ms('1 second') + ); +} function isMediaProxySniffResponse(value: unknown): value is MediaProxySniffResponse { return isJsonRecord(value) && (value.content_type === null || typeof value.content_type === 'string'); @@ -105,9 +117,16 @@ export class MediaService extends IMediaService { this.proxyURL = new URL(Config.endpoints.media); } - async getMetadata(request: MediaProxyMetadataRequest): Promise { + async getMetadata( + request: MediaProxyMetadataRequest, + options: MediaProxyMetadataOptions = {}, + ): Promise { const nsfwMode = this.getNsfwMode(request); - const response = await this.makeRequest('/_metadata', this.toMetadataWireRequest(request, nsfwMode)); + const response = await this.makeRequest( + '/_metadata', + this.toMetadataWireRequest(request, nsfwMode), + metadataRequestTimeoutMs(options.sizeBytes), + ); if (!response) { return null; } diff --git a/fluxer_api/src/api/infrastructure/MediaServiceMetadataTimeout.test.ts b/fluxer_api/src/api/infrastructure/MediaServiceMetadataTimeout.test.ts new file mode 100644 index 000000000..960e83a15 --- /dev/null +++ b/fluxer_api/src/api/infrastructure/MediaServiceMetadataTimeout.test.ts @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {metadataRequestTimeoutMs} from '@app/api/infrastructure/MediaService'; +import {describe, expect, it} from 'vitest'; + +const MIB = 1024 * 1024; + +describe('metadataRequestTimeoutMs', () => { + it.each([[undefined], [0], [100 * MIB], [500 * MIB]])( + 'keeps the 30 second window for an input of %s bytes that media-proxy buffers', + (sizeBytes) => { + expect(metadataRequestTimeoutMs(sizeBytes)).toBe(30_000); + }, + ); + + it.each([ + [500 * MIB + 1, 62_000], + [700 * MIB, 74_000], + [2 * 1024 * MIB, 158_000], + [8 * 1024 * MIB, 542_000], + ])('widens the window for an input of %s bytes that media-proxy copies to disk first', (sizeBytes, timeoutMs) => { + expect(metadataRequestTimeoutMs(sizeBytes)).toBe(timeoutMs); + }); +}); diff --git a/fluxer_api/src/api/infrastructure/StorageObjectHelpers.ts b/fluxer_api/src/api/infrastructure/StorageObjectHelpers.ts index 89f811faa..c41354642 100644 --- a/fluxer_api/src/api/infrastructure/StorageObjectHelpers.ts +++ b/fluxer_api/src/api/infrastructure/StorageObjectHelpers.ts @@ -228,15 +228,59 @@ async function stripMetadataWithExiftool(data: Uint8Array, extension: string): P } } -async function stripVideoMetadataInPlace(inputPath: string, outputPath: string): Promise { - await execFilePromise('ffmpeg', [ +function ffmpegMuxerForContentType(contentType: string): string | null { + if (contentType.includes('3gpp')) return '3gp'; + if (contentType.includes('mp4')) return 'mp4'; + if (contentType.includes('webm')) return 'webm'; + if (contentType.includes('quicktime')) return 'mov'; + if (contentType.includes('x-matroska')) return 'matroska'; + if (contentType.includes('x-msvideo')) return 'avi'; + if (contentType.includes('x-flv')) return 'flv'; + if (contentType.includes('mp2t')) return 'mpegts'; + if (contentType.includes('mpeg')) return contentType.startsWith('audio/') ? 'mp3' : 'mpeg'; + if (contentType.includes('x-ms-wmv')) return 'asf'; + if (contentType.includes('wav')) return 'wav'; + if (contentType.includes('flac')) return 'flac'; + if (contentType.includes('aac')) return 'adts'; + if (contentType.includes('aiff')) return 'aiff'; + if (contentType.includes('ogg')) return 'ogg'; + return null; +} + +const FFMPEG_FAST_START_MUXERS = new Set(['mp4', 'mov', '3gp']); +const FFMPEG_DEMUXER_FOR_MUXER: Readonly> = { + '3gp': 'mp4', + mov: 'mp4', + webm: 'matroska', + adts: 'aac', +}; +const FFMPEG_STRIP_TIMEOUT_MS = 10 * 60 * 1000; +const FFMPEG_MAX_BUFFER_BYTES = 8 * 1024 * 1024; +const FFPROBE_TIMEOUT_MS = 30 * 1000; +const FFPROBE_MAX_BUFFER_BYTES = 64 * 1024; + +function requireFfmpegMuxer(contentType: string): string { + const muxer = ffmpegMuxerForContentType(contentType); + if (!muxer) { + throw new Error(`No ffmpeg output format for content type ${contentType}`); + } + return muxer; +} + +export function ffmpegMetadataStripArgs(inputPath: string, outputPath: string, contentType: string): Array { + const muxer = requireFfmpegMuxer(contentType); + return [ '-hide_banner', '-loglevel', - 'warning', + 'error', '-i', inputPath, '-map', - '0', + '0:v?', + '-map', + '0:a?', + '-map', + '0:s?', '-map_metadata', '-1', '-map_metadata:s', @@ -247,9 +291,29 @@ async function stripVideoMetadataInPlace(inputPath: string, outputPath: string): '-1', '-c', 'copy', + ...(FFMPEG_FAST_START_MUXERS.has(muxer) ? ['-movflags', '+faststart'] : []), + '-f', + muxer, '-y', outputPath, - ]); + ]; +} + +export function ffmpegContainerMatchesContentType(formatNames: string, contentType: string): boolean { + const muxer = requireFfmpegMuxer(contentType); + return formatNames + .trim() + .split(',') + .includes(FFMPEG_DEMUXER_FOR_MUXER[muxer] ?? muxer); +} + +async function probeFfmpegContainer(inputPath: string): Promise { + const {stdout} = await execFilePromise( + 'ffprobe', + ['-v', 'error', '-show_entries', 'format=format_name', '-of', 'default=nw=1:nk=1', inputPath], + {timeout: FFPROBE_TIMEOUT_MS, maxBuffer: FFPROBE_MAX_BUFFER_BYTES}, + ); + return stdout; } async function cleanupTempFiles(paths: ReadonlyArray): Promise< @@ -392,8 +456,16 @@ async function stripVideoFileToFile( outputPath: string, contentType: string, ): Promise<{contentType: string; width?: number; height?: number}> { - await stripVideoMetadataInPlace(inputPath, outputPath); - return {contentType: normalizeContentType(contentType)}; + const normalizedContentType = normalizeContentType(contentType); + const container = await probeFfmpegContainer(inputPath); + if (!ffmpegContainerMatchesContentType(container, normalizedContentType)) { + throw new Error(`Container ${container.trim()} does not match content type ${normalizedContentType}`); + } + await execFilePromise('ffmpeg', ffmpegMetadataStripArgs(inputPath, outputPath, normalizedContentType), { + timeout: FFMPEG_STRIP_TIMEOUT_MS, + maxBuffer: FFMPEG_MAX_BUFFER_BYTES, + }); + return {contentType: normalizedContentType}; } export async function processMediaFile( diff --git a/fluxer_api/src/api/infrastructure/StorageObjectHelpersFfmpegFormat.test.ts b/fluxer_api/src/api/infrastructure/StorageObjectHelpersFfmpegFormat.test.ts new file mode 100644 index 000000000..0367b60bf --- /dev/null +++ b/fluxer_api/src/api/infrastructure/StorageObjectHelpersFfmpegFormat.test.ts @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {execFileSync, spawnSync} from 'node:child_process'; +import fs from 'node:fs'; +import { + ffmpegContainerMatchesContentType, + ffmpegMetadataStripArgs, + processMediaFile, +} from '@app/api/infrastructure/StorageObjectHelpers'; +import {temporaryFile} from 'tempy'; +import {afterEach, describe, expect, it} from 'vitest'; + +const INPUT_PATH = '/tmp/0123456789abcdef'; +const OUTPUT_PATH = '/tmp/fedcba9876543210'; + +function outputFormat(contentType: string): string | undefined { + const args = ffmpegMetadataStripArgs(INPUT_PATH, OUTPUT_PATH, contentType); + const flag = args.indexOf('-f'); + return flag === -1 ? undefined : args[flag + 1]; +} + +describe('ffmpegMetadataStripArgs', () => { + it.each([ + ['video/mp4', 'mp4'], + ['video/quicktime', 'mov'], + ['video/3gpp', '3gp'], + ['video/webm', 'webm'], + ['video/x-matroska', 'matroska'], + ['video/x-msvideo', 'avi'], + ['video/x-flv', 'flv'], + ['video/mp2t', 'mpegts'], + ['video/mpeg', 'mpeg'], + ['video/x-ms-wmv', 'asf'], + ['video/ogg', 'ogg'], + ['audio/mpeg', 'mp3'], + ['audio/mp4', 'mp4'], + ['audio/ogg', 'ogg'], + ['audio/webm', 'webm'], + ['audio/wav', 'wav'], + ['audio/flac', 'flac'], + ['audio/aac', 'adts'], + ['audio/aiff', 'aiff'], + ])('names the %s output format because the output path has no extension', (contentType, muxer) => { + expect(outputFormat(contentType)).toBe(muxer); + }); + + it('writes to the output path last and reads the input path', () => { + const args = ffmpegMetadataStripArgs(INPUT_PATH, OUTPUT_PATH, 'video/mp4'); + expect(args[args.indexOf('-i') + 1]).toBe(INPUT_PATH); + expect(args.at(-1)).toBe(OUTPUT_PATH); + }); + + it.each([ + ['video/mp4', true], + ['video/quicktime', true], + ['video/webm', false], + ['audio/mpeg', false], + ])('moves the %s index to the front of the file: %s', (contentType, fastStart) => { + const args = ffmpegMetadataStripArgs(INPUT_PATH, OUTPUT_PATH, contentType); + expect(args.includes('+faststart')).toBe(fastStart); + }); + + it('refuses a container it has no output format for', () => { + expect(() => ffmpegMetadataStripArgs(INPUT_PATH, OUTPUT_PATH, 'video/x-unknown-container')).toThrow( + /output format/, + ); + }); + + it('keeps video, audio and subtitle streams and leaves data streams behind', () => { + const args = ffmpegMetadataStripArgs(INPUT_PATH, OUTPUT_PATH, 'video/quicktime'); + const maps = args.flatMap((arg, index) => (arg === '-map' ? [args[index + 1]] : [])); + expect(maps).toEqual(['0:v?', '0:a?', '0:s?']); + }); +}); + +describe('ffmpegContainerMatchesContentType', () => { + it.each([ + ['mov,mp4,m4a,3gp,3g2,mj2\n', 'video/mp4'], + ['mov,mp4,m4a,3gp,3g2,mj2\n', 'video/quicktime'], + ['mov,mp4,m4a,3gp,3g2,mj2\n', 'video/3gpp'], + ['mov,mp4,m4a,3gp,3g2,mj2\n', 'audio/mp4'], + ['matroska,webm\n', 'video/webm'], + ['matroska,webm\n', 'video/x-matroska'], + ['matroska,webm\n', 'audio/webm'], + ['avi\n', 'video/x-msvideo'], + ['flv\n', 'video/x-flv'], + ['mpegts\n', 'video/mp2t'], + ['mpeg\n', 'video/mpeg'], + ['asf\n', 'video/x-ms-wmv'], + ['ogg\n', 'video/ogg'], + ['mp3\n', 'audio/mpeg'], + ['ogg\n', 'audio/ogg'], + ['wav\n', 'audio/wav'], + ['flac\n', 'audio/flac'], + ['aac\n', 'audio/aac'], + ['aiff\n', 'audio/aiff'], + ])('accepts the %s container for %s', (formatNames, contentType) => { + expect(ffmpegContainerMatchesContentType(formatNames, contentType)).toBe(true); + }); + + it.each([ + ['mov,mp4,m4a,3gp,3g2,mj2\n', 'video/x-matroska'], + ['matroska,webm\n', 'video/mp4'], + ['mp3\n', 'video/mpeg'], + ['mpeg\n', 'audio/mpeg'], + ['mpegts\n', 'video/mpeg'], + ['wav\n', 'audio/flac'], + ])('refuses the %s container for %s', (formatNames, contentType) => { + expect(ffmpegContainerMatchesContentType(formatNames, contentType)).toBe(false); + }); +}); + +const hasFfmpeg = ['ffmpeg', 'ffprobe'].every((tool) => spawnSync(tool, ['-version']).status === 0); + +describe.skipIf(!hasFfmpeg)('processMediaFile with ffmpeg', () => { + const paths: Array = []; + + function tempPath(): string { + const path = temporaryFile(); + paths.push(path); + return path; + } + + function generate(args: ReadonlyArray): string { + const path = tempPath(); + execFileSync('ffmpeg', [ + '-loglevel', + 'error', + '-f', + 'lavfi', + '-i', + 'testsrc=s=64x48:r=5', + '-f', + 'lavfi', + '-i', + 'sine=f=440', + '-t', + '1', + '-c:v', + 'mpeg4', + '-c:a', + 'aac', + '-metadata', + 'title=private-title', + ...args, + '-y', + path, + ]); + return path; + } + + function probe(path: string): { + format: {format_name: string; tags?: Record}; + streams: Array<{codec_type: string}>; + } { + return JSON.parse( + execFileSync('ffprobe', ['-v', 'error', '-show_format', '-show_streams', '-of', 'json', path]).toString(), + ); + } + + afterEach(() => { + for (const path of paths.splice(0)) { + fs.rmSync(path, {force: true}); + } + }); + + it('rewrites a QuickTime file without its tags and data stream at paths that have no extension', async () => { + const input = generate(['-timecode', '01:00:00:00', '-f', 'mov']); + expect(probe(input).format.tags?.title).toBe('private-title'); + expect(probe(input).streams.map((stream) => stream.codec_type)).toContain('data'); + const output = tempPath(); + + const processed = await processMediaFile(input, output, 'video/quicktime'); + + const stripped = probe(output); + expect(processed?.contentType).toBe('video/quicktime'); + expect(processed?.contentLength).toBe(fs.statSync(output).size); + expect(stripped.format.format_name).toContain('mov'); + expect(stripped.format.tags?.title).toBeUndefined(); + expect(stripped.streams.map((stream) => stream.codec_type).sort()).toEqual(['audio', 'video']); + }); + + it('refuses a file whose container is not the one its content type names', async () => { + const input = generate(['-f', 'matroska']); + + await expect(processMediaFile(input, tempPath(), 'video/mp4')).rejects.toThrow(/does not match content type/); + }); +}); diff --git a/fluxer_api/src/api/instance/InstanceConfigRepository.ts b/fluxer_api/src/api/instance/InstanceConfigRepository.ts index 9027c987f..33d490f21 100644 --- a/fluxer_api/src/api/instance/InstanceConfigRepository.ts +++ b/fluxer_api/src/api/instance/InstanceConfigRepository.ts @@ -950,6 +950,32 @@ function decodeInstanceMediaConfig(value: unknown): InstanceMediaConfig { return validateStoredConfig(StoredInstanceMediaSchema, value, 'media'); } +function resolveEffectiveAttachmentDecayConfig(media: InstanceMediaConfig): InstanceAttachmentDecayEffectiveConfig { + const attachmentDecay = media.attachment_decay; + const minSizeMb = attachmentDecay.min_size_mb ?? DEFAULT_DECAY_CONSTANTS.MIN_MB; + const configuredMaxSizeMb = attachmentDecay.max_size_mb ?? DEFAULT_DECAY_CONSTANTS.MAX_MB; + const maxSizeMb = computeAttachmentDecayMaxSize(minSizeMb, configuredMaxSizeMb); + const configuredMaxEligibleSizeMb = attachmentDecay.max_eligible_size_mb ?? DEFAULT_DECAY_CONSTANTS.PLAN_MB; + const maxEligibleSizeMb = Math.max(maxSizeMb, configuredMaxEligibleSizeMb); + const minLifetimeDays = attachmentDecay.min_lifetime_days ?? DEFAULT_DECAY_CONSTANTS.MIN_DAYS; + const configuredMaxLifetimeDays = attachmentDecay.max_lifetime_days ?? DEFAULT_DECAY_CONSTANTS.MAX_DAYS; + const maxLifetimeDays = + configuredMaxLifetimeDays >= minLifetimeDays + ? configuredMaxLifetimeDays + : Math.max(DEFAULT_DECAY_CONSTANTS.MAX_DAYS, minLifetimeDays); + return { + enabled: attachmentDecay.enabled ?? Config.attachmentDecayEnabled, + min_size_mb: minSizeMb, + max_size_mb: maxSizeMb, + max_eligible_size_mb: maxEligibleSizeMb, + min_lifetime_days: minLifetimeDays, + max_lifetime_days: maxLifetimeDays, + curve: attachmentDecay.curve ?? DEFAULT_DECAY_CONSTANTS.CURVE, + renew_threshold_days: attachmentDecay.renew_threshold_days ?? DEFAULT_RENEWAL_CONSTANTS.RENEW_THRESHOLD_DAYS, + renew_window_days: attachmentDecay.renew_window_days ?? DEFAULT_RENEWAL_CONSTANTS.RENEW_WINDOW_DAYS, + }; +} + function parseStoredInstanceMediaConfig(raw: string | null): InstanceMediaConfig { return decodeInstanceMediaConfig(parseStoredConfigValue(raw, 'media')); } @@ -1881,30 +1907,12 @@ export class InstanceConfigRepository { } async getEffectiveAttachmentDecayConfig(): Promise { - const media = await this.getInstanceMediaConfig(); - const attachmentDecay = media.attachment_decay; - const minSizeMb = attachmentDecay.min_size_mb ?? DEFAULT_DECAY_CONSTANTS.MIN_MB; - const configuredMaxSizeMb = attachmentDecay.max_size_mb ?? DEFAULT_DECAY_CONSTANTS.MAX_MB; - const maxSizeMb = computeAttachmentDecayMaxSize(minSizeMb, configuredMaxSizeMb); - const configuredMaxEligibleSizeMb = attachmentDecay.max_eligible_size_mb ?? DEFAULT_DECAY_CONSTANTS.PLAN_MB; - const maxEligibleSizeMb = Math.max(maxSizeMb, configuredMaxEligibleSizeMb); - const minLifetimeDays = attachmentDecay.min_lifetime_days ?? DEFAULT_DECAY_CONSTANTS.MIN_DAYS; - const configuredMaxLifetimeDays = attachmentDecay.max_lifetime_days ?? DEFAULT_DECAY_CONSTANTS.MAX_DAYS; - const maxLifetimeDays = - configuredMaxLifetimeDays >= minLifetimeDays - ? configuredMaxLifetimeDays - : Math.max(DEFAULT_DECAY_CONSTANTS.MAX_DAYS, minLifetimeDays); - return { - enabled: attachmentDecay.enabled ?? Config.attachmentDecayEnabled, - min_size_mb: minSizeMb, - max_size_mb: maxSizeMb, - max_eligible_size_mb: maxEligibleSizeMb, - min_lifetime_days: minLifetimeDays, - max_lifetime_days: maxLifetimeDays, - curve: attachmentDecay.curve ?? DEFAULT_DECAY_CONSTANTS.CURVE, - renew_threshold_days: attachmentDecay.renew_threshold_days ?? DEFAULT_RENEWAL_CONSTANTS.RENEW_THRESHOLD_DAYS, - renew_window_days: attachmentDecay.renew_window_days ?? DEFAULT_RENEWAL_CONSTANTS.RENEW_WINDOW_DAYS, - }; + return resolveEffectiveAttachmentDecayConfig(await this.getInstanceMediaConfig()); + } + + async fetchStoredEffectiveAttachmentDecayConfig(): Promise { + const raw = await this.fetchConfigFromDatabase(INSTANCE_MEDIA_CONFIG_KEY); + return resolveEffectiveAttachmentDecayConfig(parseStoredInstanceMediaConfig(raw)); } async getInstanceMediaAdminConfig(): Promise { diff --git a/fluxer_api/src/api/limits/tests/ConfiguredImageSizeLimits.test.ts b/fluxer_api/src/api/limits/tests/ConfiguredImageSizeLimits.test.ts new file mode 100644 index 000000000..dcd61a0cb --- /dev/null +++ b/fluxer_api/src/api/limits/tests/ConfiguredImageSizeLimits.test.ts @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils'; +import {VALID_GIF_BASE64, VALID_PNG_BASE64} from '@app/api/emoji/tests/EmojiTestUtils'; +import {createGuild} from '@app/api/guild/tests/GuildTestUtils'; +import {ensureSessionStarted} from '@app/api/message/tests/MessageTestUtils'; +import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness'; +import {HTTP_STATUS} from '@app/api/test/TestConstants'; +import {createBuilder} from '@app/api/test/TestRequestBuilder'; +import {AdminACLs} from '@fluxer/constants/src/AdminACLs'; +import type {LimitKey} from '@fluxer/constants/src/LimitConfigMetadata'; +import {AVATAR_MAX_SIZE, EMOJI_MAX_SIZE, STICKER_MAX_SIZE} from '@fluxer/constants/src/LimitConstants'; +import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes'; +import type {LimitConfigSnapshot} from '@fluxer/limits/src/LimitTypes'; +import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest'; + +const RAISED_EXPRESSION_SIZE = EMOJI_MAX_SIZE * 4; +const RAISED_AVATAR_SIZE = AVATAR_MAX_SIZE + 2 * 1024 * 1024; + +interface LimitConfigReadResponse { + limit_config: LimitConfigSnapshot; +} + +interface ValidationErrorResponse { + errors?: Array<{path?: string; code?: string; message?: string}>; +} + +function paddedImage(mime: string, base64: string, byteLength: number): string { + const image = Buffer.from(base64, 'base64'); + const padded = Buffer.concat([image, Buffer.alloc(byteLength - image.length)]); + return `data:${mime};base64,${padded.toString('base64')}`; +} + +describe('configured image size limits', () => { + let harness: ApiTestHarness; + + beforeAll(async () => { + harness = await createApiTestHarness(); + }); + + beforeEach(async () => { + await harness.reset(); + }); + + afterAll(async () => { + await harness?.shutdown(); + }); + + const raiseLimits = async (limits: Partial>): Promise => { + const admin = await setUserACLs(harness, await createTestAccount(harness), [ + AdminACLs.AUTHENTICATE, + AdminACLs.INSTANCE_LIMIT_CONFIG_VIEW, + AdminACLs.INSTANCE_LIMIT_CONFIG_UPDATE, + ]); + const current = await createBuilder(harness, admin.token) + .get('/admin/limit-config') + .expect(HTTP_STATUS.OK) + .execute(); + await createBuilder(harness, admin.token) + .put('/admin/limit-config') + .body({ + limit_config: { + traitDefinitions: current.limit_config.traitDefinitions, + rules: current.limit_config.rules.map((rule) => ({ + id: rule.id, + filters: rule.filters, + limits: rule.id === 'very_large_guild' ? rule.limits : {...rule.limits, ...limits}, + })), + }, + }) + .expect(HTTP_STATUS.OK) + .execute(); + }; + + const createOwner = async (): Promise => { + const owner = await createTestAccount(harness); + await ensureSessionStarted(harness, owner.token); + return owner; + }; + + it('accepts emoji and sticker uploads above the stock size once the limits are raised', async () => { + await raiseLimits({emoji_max_size: RAISED_EXPRESSION_SIZE, sticker_max_size: RAISED_EXPRESSION_SIZE}); + const owner = await createOwner(); + const guild = await createGuild(harness, owner.token, 'Raised expression limits'); + const image = paddedImage('image/gif', VALID_GIF_BASE64, Math.max(EMOJI_MAX_SIZE, STICKER_MAX_SIZE) * 2); + + await createBuilder(harness, owner.token) + .post(`/guilds/${guild.id}/emojis`) + .body({name: 'large_emoji', image}) + .expect(HTTP_STATUS.OK) + .execute(); + await createBuilder(harness, owner.token) + .post(`/guilds/${guild.id}/stickers`) + .body({name: 'large_sticker', description: 'large sticker', tags: [], image}) + .expect(HTTP_STATUS.OK) + .execute(); + }); + + it('rejects an emoji over a raised limit with the configured size', async () => { + await raiseLimits({emoji_max_size: RAISED_EXPRESSION_SIZE}); + const owner = await createOwner(); + const guild = await createGuild(harness, owner.token, 'Raised emoji ceiling'); + + const json = await createBuilder(harness, owner.token) + .post(`/guilds/${guild.id}/emojis`) + .body({name: 'too_large', image: paddedImage('image/gif', VALID_GIF_BASE64, RAISED_EXPRESSION_SIZE + 1)}) + .expect(HTTP_STATUS.BAD_REQUEST) + .execute(); + expect(json.errors?.[0]).toMatchObject({ + path: 'image', + code: ValidationErrorCodes.IMAGE_SIZE_EXCEEDS_LIMIT, + message: `Image size exceeds ${RAISED_EXPRESSION_SIZE} bytes.`, + }); + }); + + it('accepts an avatar above the stock size once avatar_max_size is raised', async () => { + await raiseLimits({avatar_max_size: RAISED_AVATAR_SIZE}); + const account = await createOwner(); + + const json = await createBuilder<{avatar: string | null}>(harness, account.token) + .patch('/users/@me') + .body({avatar: paddedImage('image/png', VALID_PNG_BASE64, AVATAR_MAX_SIZE + 1024 * 1024)}) + .expect(HTTP_STATUS.OK) + .execute(); + expect(json.avatar).toBeTruthy(); + }); + + it('rejects an avatar over a raised limit with the configured size', async () => { + await raiseLimits({avatar_max_size: RAISED_AVATAR_SIZE}); + const account = await createOwner(); + + const json = await createBuilder(harness, account.token) + .patch('/users/@me') + .body({avatar: paddedImage('image/png', VALID_PNG_BASE64, RAISED_AVATAR_SIZE + 1)}) + .expect(HTTP_STATUS.BAD_REQUEST) + .execute(); + expect(json.errors?.[0]).toMatchObject({ + path: 'avatar', + code: ValidationErrorCodes.IMAGE_SIZE_EXCEEDS_LIMIT, + message: `Image size exceeds ${RAISED_AVATAR_SIZE} bytes.`, + }); + }); +}); diff --git a/fluxer_api/src/api/test/mocks/RepositoryBackedMessageResponseDataService.ts b/fluxer_api/src/api/test/mocks/RepositoryBackedMessageResponseDataService.ts index 21042ee87..27a4c3d52 100644 --- a/fluxer_api/src/api/test/mocks/RepositoryBackedMessageResponseDataService.ts +++ b/fluxer_api/src/api/test/mocks/RepositoryBackedMessageResponseDataService.ts @@ -1,7 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import {AttachmentDecayRepository} from '@app/api/attachment/AttachmentDecayRepository'; -import {AttachmentDecayService} from '@app/api/attachment/AttachmentDecayService'; +import {AttachmentDecayService, resolveDefaultAttachmentDecayConfig} from '@app/api/attachment/AttachmentDecayService'; import {makeSignedAttachmentCdnUrl, signAttachmentUrl} from '@app/api/attachment/AttachmentUrls'; import type {ChannelID, GuildID, MessageID, UserID} from '@app/api/BrandedTypes'; import {createUserID} from '@app/api/BrandedTypes'; @@ -378,9 +378,10 @@ export class RepositoryBackedMessageResponseDataService extends MessageResponseD uploadedAt: snowflakeToDate(ownerMessageId), })), ); + const decayEnabled = (await resolveDefaultAttachmentDecayConfig()).enabled; return Promise.all( attachments.map(async (attachment) => { - const decay = await this.attachmentDecayRepository.fetchById(attachment.id); + const decay = decayEnabled ? await this.attachmentDecayRepository.fetchById(attachment.id) : null; const url = this.mapAttachmentUrl(message, attachment); return { id: attachment.id.toString(), diff --git a/fluxer_api/src/api/worker/WorkerLaneConfig.ts b/fluxer_api/src/api/worker/WorkerLaneConfig.ts index 4c360f036..abf6e2ef8 100644 --- a/fluxer_api/src/api/worker/WorkerLaneConfig.ts +++ b/fluxer_api/src/api/worker/WorkerLaneConfig.ts @@ -69,6 +69,7 @@ const LANE_CONFIG = { consumerName: 'workers_batch', tasks: [ 'expireAttachments', + 'reconcileAttachmentDecay', 'expireReportSnapshots', 'clearAuthenticatedReporterEmails', 'expireStaleJobs', diff --git a/fluxer_api/src/api/worker/WorkerTaskRegistry.ts b/fluxer_api/src/api/worker/WorkerTaskRegistry.ts index 0586831c0..82d0ed265 100644 --- a/fluxer_api/src/api/worker/WorkerTaskRegistry.ts +++ b/fluxer_api/src/api/worker/WorkerTaskRegistry.ts @@ -40,6 +40,7 @@ import processPremiumStateReconciliationQueue from '@app/api/worker/tasks/Proces import processStorePurchaseRefreshQueue from '@app/api/worker/tasks/ProcessStorePurchaseRefreshQueue'; import processStripeWebhook from '@app/api/worker/tasks/ProcessStripeWebhook'; import prunePostgresKvTtl from '@app/api/worker/tasks/PrunePostgresKvTtl'; +import reconcileAttachmentDecay from '@app/api/worker/tasks/ReconcileAttachmentDecay'; import reconcileUserPayments from '@app/api/worker/tasks/ReconcileUserPayments'; import refreshSearchIndex from '@app/api/worker/tasks/RefreshSearchIndex'; import refreshStorePurchase from '@app/api/worker/tasks/RefreshStorePurchase'; @@ -104,6 +105,7 @@ export const workerTasks: Record = { processInactivityDeletions, processPendingBulkMessageDeletions, processPremiumStateReconciliationQueue, + reconcileAttachmentDecay, reconcileUserPayments, processAppStoreNotification, processGooglePlayNotification, diff --git a/fluxer_api/src/api/worker/tasks/ExpireAttachments.ts b/fluxer_api/src/api/worker/tasks/ExpireAttachments.ts index 32340a7f4..e9a02895f 100644 --- a/fluxer_api/src/api/worker/tasks/ExpireAttachments.ts +++ b/fluxer_api/src/api/worker/tasks/ExpireAttachments.ts @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +import {isAttachmentDecayReconcileRunning} from '@app/api/attachment/AttachmentDecayReconcileRun'; import {AttachmentDecayRepository} from '@app/api/attachment/AttachmentDecayRepository'; import {makeAttachmentCdnKey, makeAttachmentCdnUrl} from '@app/api/channel/services/message/MessageHelpers'; import {Logger} from '@app/api/Logger'; @@ -11,12 +12,16 @@ const BUCKET_LOOKBACK_DAYS = 3; const FETCH_LIMIT = 200; export async function processExpiredAttachments(now = new Date()): Promise { - const {assetDeletionQueue, instanceConfigRepository} = getWorkerDependencies(); + const {assetDeletionQueue, instanceConfigRepository, kvClient} = getWorkerDependencies(); const attachmentDecay = await instanceConfigRepository.getEffectiveAttachmentDecayConfig(); if (!attachmentDecay.enabled) { Logger.info('Attachment decay disabled; skipping expireAttachments task'); return; } + if (await isAttachmentDecayReconcileRunning(kvClient)) { + Logger.info('Attachment decay reconcile in progress; skipping expireAttachments task'); + return; + } const repo = new AttachmentDecayRepository(); let totalQueued = 0; let totalDeletedRows = 0; diff --git a/fluxer_api/src/api/worker/tasks/ReconcileAttachmentDecay.ts b/fluxer_api/src/api/worker/tasks/ReconcileAttachmentDecay.ts new file mode 100644 index 000000000..fdb69afd0 --- /dev/null +++ b/fluxer_api/src/api/worker/tasks/ReconcileAttachmentDecay.ts @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import { + attachmentDecayReconcileJobKey, + finishAttachmentDecayReconcileRun, + holdAttachmentDecayReconcileRun, +} from '@app/api/attachment/AttachmentDecayReconcileRun'; +import {AttachmentDecayRepository} from '@app/api/attachment/AttachmentDecayRepository'; +import {type AttachmentDecayPayload, AttachmentDecayService} from '@app/api/attachment/AttachmentDecayService'; +import type {ChannelID, MessageID} from '@app/api/BrandedTypes'; +import {createChannelID, createMessageID} from '@app/api/BrandedTypes'; +import {Config} from '@app/api/Config'; +import { + attachmentDecayPayloadsForMessage, + makeAttachmentCdnKey, +} from '@app/api/channel/services/message/MessageHelpers'; +import {Logger} from '@app/api/Logger'; +import {getWorkerDependencies} from '@app/api/worker/WorkerContext'; +import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask'; +import {z} from 'zod'; + +const CHANNEL_PAGE_SIZE = 1000; + +const CursorSchema = z.object({ + pageState: z.string().nullable(), + channelId: z.string().nullable(), + beforeMessageId: z.string().nullable(), +}); + +const PayloadSchema = z.object({ + runId: z.string(), + chunk: z.number().int().nonnegative(), + cursor: CursorSchema.optional(), +}); + +type ReconcileCursor = z.infer; +export type ReconcileAttachmentDecayPayload = z.infer; + +interface ReconcileChunkLimits { + messagePageSize: number; + messagePages: number; +} + +const DEFAULT_CHUNK_LIMITS: ReconcileChunkLimits = {messagePageSize: 500, messagePages: 20}; + +interface ChunkProgress { + messagePageSize: number; + messagePagesLeft: number; + channels: number; + changed: number; +} + +async function attachmentFileExists(attachment: AttachmentDecayPayload): Promise { + const {storageService} = getWorkerDependencies(); + const key = makeAttachmentCdnKey(attachment.channelId, attachment.attachmentId, attachment.filename); + return (await storageService.getObjectMetadata(Config.s3.buckets.cdn, key)) !== null; +} + +async function reconcileChannel( + service: AttachmentDecayService, + channelId: ChannelID, + beforeMessageId: MessageID | undefined, + progress: ChunkProgress, +): Promise { + const {channelRepository} = getWorkerDependencies(); + let cursor = beforeMessageId; + while (true) { + if (progress.messagePagesLeft <= 0 && cursor !== undefined) return cursor; + const messages = await channelRepository.messages.listMessages(channelId, cursor, progress.messagePageSize); + progress.messagePagesLeft--; + if (messages.length === 0) break; + progress.changed += await service.reconcile( + messages.flatMap(attachmentDecayPayloadsForMessage), + attachmentFileExists, + ); + if (messages.length < progress.messagePageSize) break; + cursor = messages[messages.length - 1]!.id; + } + progress.channels++; + return null; +} + +async function reconcileFromCursor( + service: AttachmentDecayService, + repo: AttachmentDecayRepository, + cursor: ReconcileCursor, + progress: ChunkProgress, +): Promise { + let {pageState, channelId} = cursor; + if (channelId !== null && cursor.beforeMessageId !== null) { + const beforeMessageId = await reconcileChannel( + service, + createChannelID(BigInt(channelId)), + createMessageID(BigInt(cursor.beforeMessageId)), + progress, + ); + if (beforeMessageId !== null) return {pageState, channelId, beforeMessageId: beforeMessageId.toString()}; + } + while (true) { + const page = await repo.scanChannelsWithMessagesPage(CHANNEL_PAGE_SIZE, pageState); + const handled = channelId === null ? -1 : page.channelIds.findIndex((id) => id.toString() === channelId); + for (const nextChannelId of page.channelIds.slice(handled + 1)) { + if (progress.messagePagesLeft <= 0) return {pageState, channelId, beforeMessageId: null}; + channelId = nextChannelId.toString(); + const beforeMessageId = await reconcileChannel(service, nextChannelId, undefined, progress); + if (beforeMessageId !== null) return {pageState, channelId, beforeMessageId: beforeMessageId.toString()}; + } + if (page.pageState === null) return null; + pageState = page.pageState; + } +} + +export async function reconcileAttachmentDecayChunk( + payload: ReconcileAttachmentDecayPayload, + limits = DEFAULT_CHUNK_LIMITS, +): Promise { + const {runId, chunk} = payload; + if (!Config.instance.selfHosted) { + Logger.info({runId, chunk}, 'Attachment decay reconcile only runs on self-hosted instances; stopping'); + return null; + } + const {instanceConfigRepository, kvClient} = getWorkerDependencies(); + if (!(await holdAttachmentDecayReconcileRun(kvClient, runId))) { + Logger.info({runId, chunk}, 'Attachment decay reconcile superseded by a newer run; stopping'); + return null; + } + const config = await instanceConfigRepository.fetchStoredEffectiveAttachmentDecayConfig(); + if (!config.enabled) { + await finishAttachmentDecayReconcileRun(kvClient, runId); + Logger.info({runId, chunk}, 'Attachment decay disabled; stopping reconcileAttachmentDecay task'); + return null; + } + const repo = new AttachmentDecayRepository(); + const service = new AttachmentDecayService(repo, async () => config); + const progress: ChunkProgress = { + messagePageSize: limits.messagePageSize, + messagePagesLeft: limits.messagePages, + channels: 0, + changed: 0, + }; + const cursor = await reconcileFromCursor( + service, + repo, + payload.cursor ?? {pageState: null, channelId: null, beforeMessageId: null}, + progress, + ); + const summary = {runId, chunk, channels: progress.channels, changed: progress.changed}; + if (cursor !== null) { + Logger.info(summary, 'Reconciled a chunk of attachment decay records'); + return {runId, chunk: chunk + 1, cursor}; + } + await finishAttachmentDecayReconcileRun(kvClient, runId); + Logger.info(summary, 'Reconciled attachment decay records with the instance settings'); + return null; +} + +const reconcileAttachmentDecay: WorkerTaskHandler = async (payload, helpers) => { + const next = await reconcileAttachmentDecayChunk(PayloadSchema.parse(payload)); + if (next === null) return; + await helpers.addJob('reconcileAttachmentDecay', next, { + jobKey: attachmentDecayReconcileJobKey(next.runId, next.chunk), + skipLedger: true, + }); +}; + +export default reconcileAttachmentDecay; diff --git a/fluxer_api/src/api/worker/tests/ExpireAttachments.test.ts b/fluxer_api/src/api/worker/tests/ExpireAttachments.test.ts index 2b2f7ae22..3341945f6 100644 --- a/fluxer_api/src/api/worker/tests/ExpireAttachments.test.ts +++ b/fluxer_api/src/api/worker/tests/ExpireAttachments.test.ts @@ -2,8 +2,10 @@ import {AttachmentDecayRepository} from '@app/api/attachment/AttachmentDecayRepository'; import {createAttachmentID, createChannelID, createMessageID} from '@app/api/BrandedTypes'; +import {getConfig} from '@app/api/Config'; import type {IAssetDeletionQueue, QueuedAssetDeletion} from '@app/api/infrastructure/IAssetDeletionQueue'; import type {InstanceConfigRepository} from '@app/api/instance/InstanceConfigRepository'; +import {MockKVProvider} from '@app/api/test/mocks/MockKVProvider'; import {getExpiryBucket} from '@app/api/utils/AttachmentDecay'; import {processExpiredAttachments} from '@app/api/worker/tasks/ExpireAttachments'; import {clearWorkerDependencies, setWorkerDependenciesForTest} from '@app/api/worker/WorkerContext'; @@ -28,9 +30,10 @@ function createQueue(): {queue: IAssetDeletionQueue; queued: Array { + const originalSelfHosted = getConfig().instance.selfHosted; afterEach(() => { + getConfig().instance.selfHosted = originalSelfHosted; clearWorkerDependencies(); }); + it.each([ + [true, []], + [false, [`attachments/${CHANNEL_ID}/${ATTACHMENT_ID}/${FILENAME}`]], + ])('pauses deletion during a reconcile run only when self-hosted (%s)', async (selfHosted, expectedKeys) => { + getConfig().instance.selfHosted = selfHosted; + const repository = new AttachmentDecayRepository(); + const {queue, queued} = createQueue(); + const kvClient = new MockKVProvider(); + await kvClient.setex('attachment_decay:reconcile_run', 60, 'run'); + installDependencies(queue, kvClient); + await writeDecayRecord(repository, FIRST_EXPIRY); + + await processExpiredAttachments(new Date(FIRST_EXPIRY.getTime() + ms('1 day'))); + + expect(queued.map((item) => item.s3Key)).toEqual(expectedKeys); + }); + it('keeps the decay record when it clears a superseded expiry row', async () => { const repository = new AttachmentDecayRepository(); const {queue} = createQueue(); diff --git a/fluxer_app/src/features/app/components/setup/SelfHostedSetupWizardGate.module.css b/fluxer_app/src/features/app/components/setup/SelfHostedSetupWizardGate.module.css index 3c51378ba..98d869789 100644 --- a/fluxer_app/src/features/app/components/setup/SelfHostedSetupWizardGate.module.css +++ b/fluxer_app/src/features/app/components/setup/SelfHostedSetupWizardGate.module.css @@ -35,8 +35,13 @@ padding: 1.25rem; } +.contentScroller > .contentFrame { + flex: 1 0 auto; +} + .content { display: flex; + flex: 1 0 auto; flex-direction: column; justify-content: center; min-height: min(33rem, calc(100svh - 12.5rem)); diff --git a/fluxer_app/src/features/app/components/setup/SelfHostedSetupWizardGate.tsx b/fluxer_app/src/features/app/components/setup/SelfHostedSetupWizardGate.tsx index 567f51878..421aa0951 100644 --- a/fluxer_app/src/features/app/components/setup/SelfHostedSetupWizardGate.tsx +++ b/fluxer_app/src/features/app/components/setup/SelfHostedSetupWizardGate.tsx @@ -1004,7 +1004,11 @@ export const SelfHostedSetupWizardGate = observer(() => { hideCloseButton data-flx="app.self-hosted-setup-wizard-gate.header" /> - +
{loadError ? (
diff --git a/fluxer_app/src/features/app/config/I18nDisplayConstants.ts b/fluxer_app/src/features/app/config/I18nDisplayConstants.ts index 362a8607b..d0d5987b9 100644 --- a/fluxer_app/src/features/app/config/I18nDisplayConstants.ts +++ b/fluxer_app/src/features/app/config/I18nDisplayConstants.ts @@ -67,7 +67,6 @@ export const STATIC_IMAGE_WITH_AVIF_FORMATS = formatAssetUploadExtensions('splas export const ANIMATED_IMAGE_FORMATS = formatAssetUploadExtensions('avatar'); export const ANIMATED_AVATAR_FORMATS = formatKnownAnimatedAssetExtensions('avatar'); export const AVIF_FORMAT_LABEL = 'AVIF'; -export const IMAGE_MAX_SIZE_BYTES = 10 * 1024 * 1024; export const BACKGROUND_MEDIA_MAX_SIZE_BYTES = 10 * 1024 * 1024; export const CUSTOM_SOUND_MAX_SIZE_BYTES = 2 * 1024 * 1024; export const AVATAR_RECOMMENDED_SIZE_LABEL = '512×512px'; diff --git a/fluxer_app/src/features/app/utils/GlobalLimits.ts b/fluxer_app/src/features/app/utils/GlobalLimits.ts index ab3c6d6ca..ac97a0a1c 100644 --- a/fluxer_app/src/features/app/utils/GlobalLimits.ts +++ b/fluxer_app/src/features/app/utils/GlobalLimits.ts @@ -1,25 +1,14 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import {LimitResolver} from '@app/features/app/utils/LimitResolverAdapter'; - -const FALLBACKS = { - emoji_max_size: 384 * 1024, - sticker_max_size: 512 * 1024, - avatar_max_size: 10 * 1024 * 1024, -} as const; +import {AVATAR_MAX_SIZE} from '@fluxer/constants/src/LimitConstants'; class GlobalLimitsClass { - getEmojiMaxSize(): number { + getAvatarMaxSize(): number { return LimitResolver.resolve({ - key: 'emoji_max_size', - fallback: FALLBACKS.emoji_max_size, - }); - } - - getStickerMaxSize(): number { - return LimitResolver.resolve({ - key: 'sticker_max_size', - fallback: FALLBACKS.sticker_max_size, + key: 'avatar_max_size', + fallback: AVATAR_MAX_SIZE, + context: {traits: [], guildFeatures: []}, }); } } diff --git a/fluxer_app/src/features/app/utils/ImageSizeLimits.test.ts b/fluxer_app/src/features/app/utils/ImageSizeLimits.test.ts new file mode 100644 index 000000000..875696d5b --- /dev/null +++ b/fluxer_app/src/features/app/utils/ImageSizeLimits.test.ts @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {AVATAR_MAX_SIZE, EMOJI_MAX_SIZE, STICKER_MAX_SIZE} from '@fluxer/constants/src/LimitConstants'; +import type {LimitConfigSnapshot} from '@fluxer/limits/src/LimitTypes'; +import type {Guild as WireGuild} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas'; +import {beforeEach, describe, expect, it, vi} from 'vitest'; + +const RAISED = 5 * 1024 * 1024; +const state: {limits: LimitConfigSnapshot} = {limits: {traitDefinitions: [], rules: []}}; + +vi.mock('@app/features/app/state/RuntimeConfig', () => ({ + default: { + localInstanceDomain: 'fluxer.test', + get limits() { + return state.limits; + }, + getSnapshotOrNull: () => ({limits: state.limits}), + }, +})); +vi.mock('@app/features/app/state/InstanceSnapshotStore', () => ({default: {getLimitsForInstance: () => null}})); +vi.mock('@app/features/user/state/Users', () => ({default: {getCurrentUser: () => ({traits: ['premium']})}})); +vi.mock('@app/features/channel/state/Channels', () => ({default: {getChannel: () => null}})); +vi.mock('@app/features/guild/state/Guilds', () => ({default: {getGuild: () => null}})); +vi.mock('@app/features/navigation/state/SelectedChannel', () => ({default: {currentChannelId: null}})); +vi.mock('@app/features/navigation/state/SelectedGuild', () => ({default: {selectedGuildId: null}})); +vi.mock('@app/features/theme/fonts/ScriptFontLoader', () => ({noteText: () => {}})); + +const {Guild} = await import('@app/features/guild/models/Guild'); +const {GlobalLimits} = await import('@app/features/app/utils/GlobalLimits'); + +function guild(features: Array = []) { + return new Guild({id: '1', name: 'guild', owner_id: '2', features, roles: []} as unknown as WireGuild); +} + +beforeEach(() => { + state.limits = {traitDefinitions: ['premium'], rules: [{id: 'default', limits: {}}]}; +}); + +describe('configured image size limits', () => { + it('falls back to the stock sizes when nothing is configured', () => { + expect(guild().maxEmojiSize).toBe(EMOJI_MAX_SIZE); + expect(guild().maxStickerSize).toBe(STICKER_MAX_SIZE); + expect(GlobalLimits.getAvatarMaxSize()).toBe(AVATAR_MAX_SIZE); + }); + + it('follows a raised default rule', () => { + state.limits.rules[0].limits = {emoji_max_size: RAISED, sticker_max_size: RAISED, avatar_max_size: RAISED * 6}; + expect(guild().maxEmojiSize).toBe(RAISED); + expect(guild().maxStickerSize).toBe(RAISED); + expect(GlobalLimits.getAvatarMaxSize()).toBe(RAISED * 6); + }); + + it('applies guild feature rules to emoji and sticker sizes', () => { + state.limits.rules.push({ + id: 'boosted', + filters: {guildFeatures: ['MORE_EMOJI']}, + limits: {emoji_max_size: RAISED, sticker_max_size: RAISED}, + }); + expect(guild().maxEmojiSize).toBe(EMOJI_MAX_SIZE); + expect(guild(['MORE_EMOJI']).maxEmojiSize).toBe(RAISED); + expect(guild(['MORE_EMOJI']).maxStickerSize).toBe(RAISED); + }); + + it('ignores trait rules the server does not apply', () => { + state.limits.rules.push({ + id: 'premium', + filters: {traits: ['premium']}, + limits: {emoji_max_size: RAISED, sticker_max_size: RAISED, avatar_max_size: RAISED * 6}, + }); + expect(guild().maxEmojiSize).toBe(EMOJI_MAX_SIZE); + expect(guild().maxStickerSize).toBe(STICKER_MAX_SIZE); + expect(GlobalLimits.getAvatarMaxSize()).toBe(AVATAR_MAX_SIZE); + }); +}); diff --git a/fluxer_app/src/features/channel/components/modals/useEditGroupForm.tsx b/fluxer_app/src/features/channel/components/modals/useEditGroupForm.tsx index 7065e63ee..8ec57c97f 100644 --- a/fluxer_app/src/features/channel/components/modals/useEditGroupForm.tsx +++ b/fluxer_app/src/features/channel/components/modals/useEditGroupForm.tsx @@ -1,11 +1,8 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import { - AVATAR_RECOMMENDED_SIZE_LABEL, - IMAGE_MAX_SIZE_BYTES, - STATIC_IMAGE_FORMATS, -} from '@app/features/app/config/I18nDisplayConstants'; +import {AVATAR_RECOMMENDED_SIZE_LABEL, STATIC_IMAGE_FORMATS} from '@app/features/app/config/I18nDisplayConstants'; import {useFormSubmit} from '@app/features/app/hooks/useFormSubmit'; +import {GlobalLimits} from '@app/features/app/utils/GlobalLimits'; import Authentication from '@app/features/auth/state/Authentication'; import * as ChannelCommands from '@app/features/channel/commands/ChannelCommands'; import {showChannelErrorModal} from '@app/features/channel/components/alerts/ChannelErrorModalUtils'; @@ -85,11 +82,11 @@ export function useEditGroupForm({ async (file: File | null) => { try { if (!file) return; - if (file.size > 10 * 1024 * 1024) { + if (file.size > GlobalLimits.getAvatarMaxSize()) { showChannelErrorModal({ title: i18n._(messages.iconFileTooLargeTitle), message: i18n._(messages.iconFileTooLarge, { - imageMaxSizeLabel: formatFileSize(i18n.locale, IMAGE_MAX_SIZE_BYTES), + imageMaxSizeLabel: formatFileSize(i18n.locale, GlobalLimits.getAvatarMaxSize()), }), dataFlx: `${dataFlx}.icon-file-too-large.generic-error-modal`, }); @@ -166,7 +163,7 @@ export function useEditGroupForm({ title: i18n._(messages.changeIcon), uploadHint: formatImageUploadRecommendedHint(i18n, { formats: STATIC_IMAGE_FORMATS, - maxSize: formatFileSize(i18n.locale, IMAGE_MAX_SIZE_BYTES), + maxSize: formatFileSize(i18n.locale, GlobalLimits.getAvatarMaxSize()), recommendedSize: AVATAR_RECOMMENDED_SIZE_LABEL, }), onPickUpload: handleIconUploadClick, diff --git a/fluxer_app/src/features/expressions/components/modals/AddGuildStickerModal.tsx b/fluxer_app/src/features/expressions/components/modals/AddGuildStickerModal.tsx index 9f1579d38..e75cb89a6 100644 --- a/fluxer_app/src/features/expressions/components/modals/AddGuildStickerModal.tsx +++ b/fluxer_app/src/features/expressions/components/modals/AddGuildStickerModal.tsx @@ -4,12 +4,12 @@ import {GenericErrorModal} from '@app/features/app/components/alerts/GenericErro import {ConfirmModal} from '@app/features/app/components/dialogs/ConfirmModal'; import * as Modal from '@app/features/app/components/dialogs/Modal'; import {useFormSubmit} from '@app/features/app/hooks/useFormSubmit'; -import {GlobalLimits} from '@app/features/app/utils/GlobalLimits'; import * as GuildStickerCommands from '@app/features/expressions/commands/GuildStickerCommands'; import styles from '@app/features/expressions/components/modals/AddGuildStickerModal.module.css'; import {StickerFormFields} from '@app/features/expressions/components/modals/sticker_form/StickerFormFields'; import {StickerPreview} from '@app/features/expressions/components/modals/sticker_form/StickerPreview'; import * as ImageCropUtils from '@app/features/expressions/utils/ImageCropUtils'; +import Guilds from '@app/features/guild/state/Guilds'; import {CREATE_DESCRIPTOR} from '@app/features/i18n/utils/CommonMessageDescriptors'; import {formatFileSize} from '@app/features/messaging/utils/FileUtils'; import {HttpError} from '@app/features/platform/types/EndpointError'; @@ -20,6 +20,7 @@ import * as ModalCommands from '@app/features/ui/commands/ModalCommands'; import {Form} from '@app/features/ui/components/form/Form'; import * as FormUtils from '@app/lib/forms'; import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; +import {STICKER_MAX_SIZE} from '@fluxer/constants/src/LimitConstants'; import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes'; import type {I18n} from '@lingui/core'; import {msg} from '@lingui/core/macro'; @@ -143,7 +144,7 @@ export const AddGuildStickerModal = observer(function AddGuildStickerModal({ }, [file]); const onSubmit = useCallback( async (data: FormInputs) => { - const maxStickerSize = GlobalLimits.getStickerMaxSize(); + const maxStickerSize = Guilds.getGuild(guildId)?.maxStickerSize ?? STICKER_MAX_SIZE; setIsProcessing(true); try { const base64Image = await ImageCropUtils.optimizeStickerImage(file, maxStickerSize, 320); diff --git a/fluxer_app/src/features/expressions/components/modals/AssetCropModal.tsx b/fluxer_app/src/features/expressions/components/modals/AssetCropModal.tsx index 7a51b910c..6abf00a41 100644 --- a/fluxer_app/src/features/expressions/components/modals/AssetCropModal.tsx +++ b/fluxer_app/src/features/expressions/components/modals/AssetCropModal.tsx @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +import {GlobalLimits} from '@app/features/app/utils/GlobalLimits'; import { getAspectRatioRange, type ImageDimensions, @@ -134,7 +135,6 @@ interface AssetConfig { maxHeight: number; minWidth: number; minHeight: number; - sizeLimitBytes: number; minHeightRatio?: number; maxHeightRatio?: number; } @@ -147,7 +147,6 @@ const ASSET_CONFIGS: Record = { maxHeight: 1024, minWidth: 256, minHeight: 256, - sizeLimitBytes: 10 * 1024 * 1024, }, [AssetType.GUILD_ICON]: { aspectRatio: 1, @@ -156,7 +155,6 @@ const ASSET_CONFIGS: Record = { maxHeight: 1024, minWidth: 256, minHeight: 256, - sizeLimitBytes: 10 * 1024 * 1024, }, [AssetType.CHANNEL_ICON]: { aspectRatio: 1, @@ -165,7 +163,6 @@ const ASSET_CONFIGS: Record = { maxHeight: 1024, minWidth: 256, minHeight: 256, - sizeLimitBytes: 10 * 1024 * 1024, }, [AssetType.GUILD_BANNER]: { aspectRatio: 16 / 9, @@ -174,7 +171,6 @@ const ASSET_CONFIGS: Record = { maxHeight: 1152, minWidth: 960, minHeight: 540, - sizeLimitBytes: 10 * 1024 * 1024, minHeightRatio: 0.5, maxHeightRatio: 1, }, @@ -185,7 +181,6 @@ const ASSET_CONFIGS: Record = { maxHeight: 723, minWidth: 680, minHeight: 240, - sizeLimitBytes: 10 * 1024 * 1024, }, [AssetType.SPLASH]: { aspectRatio: 16 / 9, @@ -194,7 +189,6 @@ const ASSET_CONFIGS: Record = { maxHeight: 1152, minWidth: 960, minHeight: 540, - sizeLimitBytes: 10 * 1024 * 1024, minHeightRatio: 0.5, maxHeightRatio: 1, }, @@ -205,7 +199,6 @@ const ASSET_CONFIGS: Record = { maxHeight: 1152, minWidth: 960, minHeight: 540, - sizeLimitBytes: 10 * 1024 * 1024, minHeightRatio: 0.5, maxHeightRatio: 1, }, @@ -317,7 +310,7 @@ export const AssetCropModal: React.FC = observer( cropShape={config.cropShape} maxWidth={config.maxWidth} maxHeight={config.maxHeight} - sizeLimitBytes={config.sizeLimitBytes} + sizeLimitBytes={GlobalLimits.getAvatarMaxSize()} minHeightRatio={config.minHeightRatio} maxHeightRatio={config.maxHeightRatio} canSkipOriginal={(dimensions) => canSkipOriginalAssetImage(assetType, dimensions)} diff --git a/fluxer_app/src/features/guild/components/modals/add_guild_modal/GuildCreateShared.tsx b/fluxer_app/src/features/guild/components/modals/add_guild_modal/GuildCreateShared.tsx index cfe37ba86..7725e8cdc 100644 --- a/fluxer_app/src/features/guild/components/modals/add_guild_modal/GuildCreateShared.tsx +++ b/fluxer_app/src/features/guild/components/modals/add_guild_modal/GuildCreateShared.tsx @@ -3,12 +3,9 @@ import {Routes} from '@app/app/Routes'; import {UserSettingsModal} from '@app/features/app/components/dialogs/LoadableSettingsModals'; import {ExternalLink} from '@app/features/app/components/shared/ExternalLink'; -import { - AVATAR_RECOMMENDED_SIZE_LABEL, - IMAGE_MAX_SIZE_BYTES, - STATIC_IMAGE_FORMATS, -} from '@app/features/app/config/I18nDisplayConstants'; +import {AVATAR_RECOMMENDED_SIZE_LABEL, STATIC_IMAGE_FORMATS} from '@app/features/app/config/I18nDisplayConstants'; import RuntimeConfig from '@app/features/app/state/RuntimeConfig'; +import {GlobalLimits} from '@app/features/app/utils/GlobalLimits'; import {openClaimAccountModal} from '@app/features/auth/components/modals/ClaimAccountModal'; import {AssetCropModal, AssetType} from '@app/features/expressions/components/modals/AssetCropModal'; import {openAssetSourceModal} from '@app/features/expressions/components/modals/AssetSourceModal'; @@ -95,10 +92,10 @@ export function useGuildCreateIconField({ try { const [file] = await openFilePicker({accept: getAcceptStringFiltered('guild_icon', false)}); if (!file) return; - if (file.size > 10 * 1024 * 1024) { + if (file.size > GlobalLimits.getAvatarMaxSize()) { showIconUploadErrorModal( i18n._(ICON_FILE_IS_TOO_LARGE_PLEASE_CHOOSE_A_DESCRIPTOR, { - imageMaxSizeLabel: formatFileSize(i18n.locale, IMAGE_MAX_SIZE_BYTES), + imageMaxSizeLabel: formatFileSize(i18n.locale, GlobalLimits.getAvatarMaxSize()), }), ); return; @@ -150,7 +147,7 @@ export function useGuildCreateIconField({ title: i18n._(messages.changeIcon), uploadHint: formatImageUploadRecommendedHint(i18n, { formats: STATIC_IMAGE_FORMATS, - maxSize: formatFileSize(i18n.locale, IMAGE_MAX_SIZE_BYTES), + maxSize: formatFileSize(i18n.locale, GlobalLimits.getAvatarMaxSize()), recommendedSize: AVATAR_RECOMMENDED_SIZE_LABEL, }), onPickUpload: handleIconUpload, diff --git a/fluxer_app/src/features/guild/components/modals/guild_tabs/GuildEmojiTab.tsx b/fluxer_app/src/features/guild/components/modals/guild_tabs/GuildEmojiTab.tsx index 6a82aa38c..50a719419 100644 --- a/fluxer_app/src/features/guild/components/modals/guild_tabs/GuildEmojiTab.tsx +++ b/fluxer_app/src/features/guild/components/modals/guild_tabs/GuildEmojiTab.tsx @@ -2,7 +2,6 @@ import {GenericErrorModal} from '@app/features/app/components/alerts/GenericErrorModal'; import {ConfirmModal} from '@app/features/app/components/dialogs/ConfirmModal'; -import {GlobalLimits} from '@app/features/app/utils/GlobalLimits'; import {EmojiListHeader, EmojiListItem} from '@app/features/emoji/components/emojis/EmojiListItem'; import {EmojiUploadModal} from '@app/features/emoji/components/modals/EmojiUploadModal'; import EmojiStickerLayout from '@app/features/emoji/state/EmojiStickerLayout'; @@ -33,7 +32,7 @@ import {Spinner} from '@app/features/ui/components/Spinner'; import Users from '@app/features/user/state/Users'; import {canCropFormat} from '@app/features/voice/utils/MediaCapabilities'; import {Permissions} from '@fluxer/constants/src/ChannelConstants'; -import {MAX_GUILD_EMOJIS} from '@fluxer/constants/src/LimitConstants'; +import {EMOJI_MAX_SIZE, MAX_GUILD_EMOJIS} from '@fluxer/constants/src/LimitConstants'; import type {GuildEmojiWithUser} from '@fluxer/schema/src/domains/guild/GuildEmojiSchemas'; import {sortBySnowflakeDesc} from '@fluxer/snowflake/src/SnowflakeUtils'; import {msg} from '@lingui/core/macro'; @@ -179,7 +178,8 @@ const GuildEmojiTab: React.FC<{guildId: string}> = observer(function GuildEmojiT }, [filteredEmojis]); const maxEmojis = guild?.maxEmojis ?? MAX_GUILD_EMOJIS; const currentEmojiCount = emojis.length; - const emojiMaxSizeLabel = formatFileSize(i18n.locale, GlobalLimits.getEmojiMaxSize()); + const maxEmojiSize = guild?.maxEmojiSize ?? EMOJI_MAX_SIZE; + const emojiMaxSizeLabel = formatFileSize(i18n.locale, maxEmojiSize); const canModifyEmoji = useCallback( (emoji: GuildEmojiWithUser): boolean => { if (canManageExpressions) return true; @@ -291,7 +291,6 @@ const GuildEmojiTab: React.FC<{guildId: string}> = observer(function GuildEmojiT ); const preparedEmojis: Array<{name: string; image: string; file: File}> = []; const preparationFailures: Array = []; - const maxEmojiSize = GlobalLimits.getEmojiMaxSize(); const maxEmojiSizeLabel = formatFileSize(i18n.locale, maxEmojiSize); for (const file of filesWithinSlots) { try { @@ -399,7 +398,7 @@ const GuildEmojiTab: React.FC<{guildId: string}> = observer(function GuildEmojiT ); } }, - [guildId, fetchEmojis, maxEmojis, currentEmojiCount, i18n, renderEmojiFailureDescription], + [guildId, fetchEmojis, maxEmojis, maxEmojiSize, currentEmojiCount, i18n, renderEmojiFailureDescription], ); const handleDrop = useCallback( async (files: Array) => { diff --git a/fluxer_app/src/features/guild/components/modals/guild_tabs/GuildStickersTab.tsx b/fluxer_app/src/features/guild/components/modals/guild_tabs/GuildStickersTab.tsx index 61178a313..26c5ca0d3 100644 --- a/fluxer_app/src/features/guild/components/modals/guild_tabs/GuildStickersTab.tsx +++ b/fluxer_app/src/features/guild/components/modals/guild_tabs/GuildStickersTab.tsx @@ -3,7 +3,6 @@ import {GenericErrorModal} from '@app/features/app/components/alerts/GenericErrorModal'; import {ConfirmModal} from '@app/features/app/components/dialogs/ConfirmModal'; import {StatusSlate} from '@app/features/app/components/dialogs/shared/StatusSlate'; -import {GlobalLimits} from '@app/features/app/utils/GlobalLimits'; import {StickerGridItem} from '@app/features/emoji/components/stickers/StickerGridItem'; import EmojiStickerLayout from '@app/features/emoji/state/EmojiStickerLayout'; import * as GuildStickerCommands from '@app/features/expressions/commands/GuildStickerCommands'; @@ -34,7 +33,7 @@ import {Spinner} from '@app/features/ui/components/Spinner'; import Users from '@app/features/user/state/Users'; import {canCropFormat} from '@app/features/voice/utils/MediaCapabilities'; import {Permissions} from '@fluxer/constants/src/ChannelConstants'; -import {MAX_GUILD_STICKERS} from '@fluxer/constants/src/LimitConstants'; +import {MAX_GUILD_STICKERS, STICKER_MAX_SIZE} from '@fluxer/constants/src/LimitConstants'; import type {GuildStickerWithUser} from '@fluxer/schema/src/domains/guild/GuildEmojiSchemas'; import {sortBySnowflakeDesc} from '@fluxer/snowflake/src/SnowflakeUtils'; import {msg} from '@lingui/core/macro'; @@ -227,7 +226,7 @@ const GuildStickersTab: React.FC<{guildId: string}> = observer(function GuildSti }, [canManageExpressions, canCreateExpressions, currentUserId], ); - const stickerMaxSizeLabel = formatFileSize(i18n.locale, GlobalLimits.getStickerMaxSize()); + const stickerMaxSizeLabel = formatFileSize(i18n.locale, guild?.maxStickerSize ?? STICKER_MAX_SIZE); return (
>; setHasCleared: React.Dispatch>; - maxBytes?: number; labelForMessages: string; gif?: GifPolicy; aspectRatio?: AspectRatioConfig; @@ -107,7 +106,6 @@ export function useGuildImageAssetField({ previewUrl, setPreviewUrl, setHasCleared, - maxBytes = MAX_IMAGE_BYTES, labelForMessages, gif, aspectRatio, @@ -115,6 +113,7 @@ export function useGuildImageAssetField({ uploadHint, }: UseGuildImageAssetFieldArgs): ImageAssetFieldController { const {i18n} = useLingui(); + const maxBytes = GlobalLimits.getAvatarMaxSize(); const label = labelForMessages; const requestIdRef = useRef(0); const [isProcessing, setIsProcessing] = useState(false); diff --git a/fluxer_app/src/features/guild/components/modals/guild_tabs/guild_overview_tab/utils/ImageAsset.ts b/fluxer_app/src/features/guild/components/modals/guild_tabs/guild_overview_tab/utils/ImageAsset.ts index 201a68746..cb1e9be10 100644 --- a/fluxer_app/src/features/guild/components/modals/guild_tabs/guild_overview_tab/utils/ImageAsset.ts +++ b/fluxer_app/src/features/guild/components/modals/guild_tabs/guild_overview_tab/utils/ImageAsset.ts @@ -1,7 +1,5 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -export const MAX_IMAGE_BYTES = 10 * 1024 * 1024; - export function revokeObjectUrl(url: string | null | undefined): void { if (!url) return; if (!url.startsWith('blob:')) return; diff --git a/fluxer_app/src/features/guild/models/Guild.ts b/fluxer_app/src/features/guild/models/Guild.ts index 8db7e75a0..cbf47b3c7 100644 --- a/fluxer_app/src/features/guild/models/Guild.ts +++ b/fluxer_app/src/features/guild/models/Guild.ts @@ -8,7 +8,12 @@ import {LARGE_GUILD_THRESHOLD} from '@fluxer/constants/src/GatewayConstants'; import type {GuildSplashCardAlignmentValue} from '@fluxer/constants/src/GuildConstants'; import {GuildFeatures, GuildNSFWLevel, GuildSplashCardAlignment} from '@fluxer/constants/src/GuildConstants'; import type {LimitKey} from '@fluxer/constants/src/LimitConfigMetadata'; -import {MAX_GUILD_EMOJIS, MAX_GUILD_STICKERS} from '@fluxer/constants/src/LimitConstants'; +import { + EMOJI_MAX_SIZE, + MAX_GUILD_EMOJIS, + MAX_GUILD_STICKERS, + STICKER_MAX_SIZE, +} from '@fluxer/constants/src/LimitConstants'; import {MessageNotifications} from '@fluxer/constants/src/NotificationConstants'; import {resolveLimit} from '@fluxer/limits/src/LimitResolver'; import type {Guild as WireGuild} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas'; @@ -348,6 +353,14 @@ export class Guild { return this.resolveGuildLimit('max_guild_stickers', MAX_GUILD_STICKERS); } + get maxEmojiSize(): number { + return this.resolveGuildLimit('emoji_max_size', EMOJI_MAX_SIZE); + } + + get maxStickerSize(): number { + return this.resolveGuildLimit('sticker_max_size', STICKER_MAX_SIZE); + } + private resolveGuildLimit(key: LimitKey, fallback: number): number { const resolved = resolveLimit( RuntimeConfig.limits, diff --git a/fluxer_app/src/features/user/components/modals/tabs/my_profile_tab/AvatarUploader.tsx b/fluxer_app/src/features/user/components/modals/tabs/my_profile_tab/AvatarUploader.tsx index 5bf18cc4c..a4c22a343 100644 --- a/fluxer_app/src/features/user/components/modals/tabs/my_profile_tab/AvatarUploader.tsx +++ b/fluxer_app/src/features/user/components/modals/tabs/my_profile_tab/AvatarUploader.tsx @@ -6,10 +6,10 @@ import { ANIMATED_AVATAR_FORMATS, ANIMATED_IMAGE_FORMATS, AVATAR_RECOMMENDED_SIZE_LABEL, - IMAGE_MAX_SIZE_BYTES, STATIC_IMAGE_FORMATS, } from '@app/features/app/config/I18nDisplayConstants'; import RuntimeConfig from '@app/features/app/state/RuntimeConfig'; +import {GlobalLimits} from '@app/features/app/utils/GlobalLimits'; import {LimitResolver} from '@app/features/app/utils/LimitResolverAdapter'; import {isLimitToggleEnabled} from '@app/features/app/utils/LimitUtils'; import type {Gif} from '@app/features/expressions/commands/GifCommands'; @@ -163,11 +163,11 @@ export const AvatarUploader = observer( ); const processAvatarFile = useCallback( async (file: File) => { - if (file.size > 10 * 1024 * 1024) { + if (file.size > GlobalLimits.getAvatarMaxSize()) { showUserErrorModal( i18n._(COULDN_T_UPLOAD_AVATAR_DESCRIPTOR), i18n._(AVATAR_FILE_IS_TOO_LARGE_PLEASE_CHOOSE_A_DESCRIPTOR, { - imageMaxSizeLabel: formatFileSize(i18n.locale, IMAGE_MAX_SIZE_BYTES), + imageMaxSizeLabel: formatFileSize(i18n.locale, GlobalLimits.getAvatarMaxSize()), }), ); return; @@ -308,13 +308,13 @@ export const AvatarUploader = observer( const uploadHint = canUploadAnimatedAvatar ? formatImageUploadRecommendedHint(i18n, { formats: ANIMATED_IMAGE_FORMATS, - maxSize: formatFileSize(i18n.locale, IMAGE_MAX_SIZE_BYTES), + maxSize: formatFileSize(i18n.locale, GlobalLimits.getAvatarMaxSize()), recommendedSize: AVATAR_RECOMMENDED_SIZE_LABEL, }) : shouldShowPremiumFeatures() ? formatImageUploadRecommendedHintWithNote(i18n, { formats: STATIC_IMAGE_FORMATS, - maxSize: formatFileSize(i18n.locale, IMAGE_MAX_SIZE_BYTES), + maxSize: formatFileSize(i18n.locale, GlobalLimits.getAvatarMaxSize()), recommendedSize: AVATAR_RECOMMENDED_SIZE_LABEL, note: i18n._(ANIMATED_AVATARS_REQUIRE_PREMIUM_NOTE_DESCRIPTOR, { animatedAvatarFormats: ANIMATED_AVATAR_FORMATS, @@ -323,7 +323,7 @@ export const AvatarUploader = observer( }) : formatImageUploadRecommendedHint(i18n, { formats: STATIC_IMAGE_FORMATS, - maxSize: formatFileSize(i18n.locale, IMAGE_MAX_SIZE_BYTES), + maxSize: formatFileSize(i18n.locale, GlobalLimits.getAvatarMaxSize()), recommendedSize: AVATAR_RECOMMENDED_SIZE_LABEL, }); openAssetSourceModal({ diff --git a/fluxer_app/src/features/user/components/modals/tabs/my_profile_tab/BannerUploader.tsx b/fluxer_app/src/features/user/components/modals/tabs/my_profile_tab/BannerUploader.tsx index 7d753ca8e..0b4fbaae3 100644 --- a/fluxer_app/src/features/user/components/modals/tabs/my_profile_tab/BannerUploader.tsx +++ b/fluxer_app/src/features/user/components/modals/tabs/my_profile_tab/BannerUploader.tsx @@ -5,8 +5,8 @@ import { ANIMATED_IMAGE_FORMATS, BANNER_ASPECT_RATIO_LABEL, BANNER_MINIMUM_SIZE_LABEL, - IMAGE_MAX_SIZE_BYTES, } from '@app/features/app/config/I18nDisplayConstants'; +import {GlobalLimits} from '@app/features/app/utils/GlobalLimits'; import {LimitResolver} from '@app/features/app/utils/LimitResolverAdapter'; import {isLimitToggleEnabled} from '@app/features/app/utils/LimitUtils'; import type {Gif} from '@app/features/expressions/commands/GifCommands'; @@ -183,11 +183,11 @@ export const BannerUploader = observer( }, [i18n]); const processBannerFile = useCallback( async (file: File) => { - if (file.size > 10 * 1024 * 1024) { + if (file.size > GlobalLimits.getAvatarMaxSize()) { showUserErrorModal( i18n._(COULDN_T_UPLOAD_BANNER_DESCRIPTOR), i18n._(BANNER_FILE_IS_TOO_LARGE_PLEASE_CHOOSE_A_DESCRIPTOR, { - imageMaxSizeLabel: formatFileSize(i18n.locale, IMAGE_MAX_SIZE_BYTES), + imageMaxSizeLabel: formatFileSize(i18n.locale, GlobalLimits.getAvatarMaxSize()), }), ); return; @@ -264,7 +264,7 @@ export const BannerUploader = observer( title: i18n._(CHANGE_BANNER_DESCRIPTOR), uploadHint: formatImageUploadMinimumHint(i18n, { formats: ANIMATED_IMAGE_FORMATS, - maxSize: formatFileSize(i18n.locale, IMAGE_MAX_SIZE_BYTES), + maxSize: formatFileSize(i18n.locale, GlobalLimits.getAvatarMaxSize()), minimumSize: BANNER_MINIMUM_SIZE_LABEL, aspectRatio: BANNER_ASPECT_RATIO_LABEL, }), diff --git a/fluxer_app/src/features/webhook/components/WebhookListItem.tsx b/fluxer_app/src/features/webhook/components/WebhookListItem.tsx index 1695a22a9..f2b1c91f7 100644 --- a/fluxer_app/src/features/webhook/components/WebhookListItem.tsx +++ b/fluxer_app/src/features/webhook/components/WebhookListItem.tsx @@ -1,7 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import {showGenericErrorModal} from '@app/features/app/components/alerts/GenericErrorModalCommands'; -import {IMAGE_MAX_SIZE_BYTES} from '@app/features/app/config/I18nDisplayConstants'; +import {GlobalLimits} from '@app/features/app/utils/GlobalLimits'; import {getAcceptString} from '@app/features/expressions/utils/AssetFormatCopy'; import {isSvgFile, readImageFileAsUploadDataUrl} from '@app/features/expressions/utils/ImageUploadFileUtils'; import { @@ -159,12 +159,12 @@ export const WebhookListItem: React.FC = observer( const handleAvatarUpload = useCallback(async () => { const [file] = await openFilePicker({accept: getAcceptString('avatar')}); if (!file) return; - if (file.size > 10 * 1024 * 1024) { + if (file.size > GlobalLimits.getAvatarMaxSize()) { showGenericErrorModal({ title: () => i18n._(SOMETHING_WENT_WRONG_DESCRIPTOR), message: () => i18n._(AVATAR_FILE_IS_TOO_LARGE_PLEASE_CHOOSE_A_DESCRIPTOR, { - imageMaxSizeLabel: formatFileSize(i18n.locale, IMAGE_MAX_SIZE_BYTES), + imageMaxSizeLabel: formatFileSize(i18n.locale, GlobalLimits.getAvatarMaxSize()), }), dataFlx: 'webhook.webhook-list-item.avatar-too-large-error-modal', }); diff --git a/fluxer_docs/src/content/docs/http-api/applications.mdx b/fluxer_docs/src/content/docs/http-api/applications.mdx index 46329c12b..db328ed3e 100644 --- a/fluxer_docs/src/content/docs/http-api/applications.mdx +++ b/fluxer_docs/src/content/docs/http-api/applications.mdx @@ -435,7 +435,7 @@ Updates the bot account an application owns and returns the resulting [bot profi 2 Accepted only when it equals the bot's current discriminator. Any other value returns the field code `BOT_DISCRIMINATOR_CANNOT_BE_CHANGED` -3 The value can include a `data:...;base64,` prefix, which Fluxer strips before it measures the payload. The remaining base64 payload is bounded at 13981016 characters for both fields, the encoded length of a full 10485760-byte image, and a longer value is rejected with the field code `BASE64_LENGTH_INVALID` +3 The value can include a `data:...;base64,` prefix, which Fluxer strips before it measures the payload. The remaining base64 payload is bounded at 139810136 characters for both fields, the encoded length of a 104857600-byte image, and a longer value is rejected with the field code `BASE64_LENGTH_INVALID`. The decoded image must fit the resolved `avatar_max_size` [limit](/http-api/instance/#limit-keys), 10485760 bytes by default, or it is rejected with `IMAGE_SIZE_EXCEEDS_LIMIT` 4 Rejected with 403 `CONTENT_BLOCKED` when content moderation or the profile substring blocklist blocks the value diff --git a/fluxer_docs/src/content/docs/http-api/channels.mdx b/fluxer_docs/src/content/docs/http-api/channels.mdx index e82791e5f..6a71a2f2f 100644 --- a/fluxer_docs/src/content/docs/http-api/channels.mdx +++ b/fluxer_docs/src/content/docs/http-api/channels.mdx @@ -531,12 +531,12 @@ Every field is optional, and an omitted field preserves its current value. | Field | Type | Description | | --- | --- | --- | | name? | ?string | The name of the group (1-100 characters after normalisation), or null to clear it | -| icon?1 | ?base64 string | The base64-encoded icon image (at most 13,981,014 encoded characters), or null to clear it | +| icon?1 | ?base64 string | The base64-encoded icon image (at most 139,810,136 encoded characters), or null to clear it | | owner_id?2 | ?snowflake | The ID of the new owner | | nicks?3 | ?map[snowflake, ?string] | The nicknames keyed by the decimal user ID (each 0-32 characters, or null) | | nsfw?4 | ?boolean | Whether the group is marked for mature content | -1 Raw base64 or a data URL, with 1 through 13,981,014 characters after the prefix +1 Raw base64 or a data URL, with 1 through 139,810,136 characters after the prefix 2 An explicit null requests no transfer. The caller must be the current owner and the target a current recipient diff --git a/fluxer_docs/src/content/docs/http-api/guild-emojis.mdx b/fluxer_docs/src/content/docs/http-api/guild-emojis.mdx index a9c299581..f31444226 100644 --- a/fluxer_docs/src/content/docs/http-api/guild-emojis.mdx +++ b/fluxer_docs/src/content/docs/http-api/guild-emojis.mdx @@ -58,9 +58,9 @@ The image and name submitted for one new emoji. 3 The decoded ceiling is the resolved `emoji_max_size` [limit key](/http-api/instance/#limit-keys) in guild scope, 524288 bytes by default -The remaining encoded part is bounded to 699052 characters, the base64 expansion of the 524288 byte default. A longer value returns the validation code `BASE64_LENGTH_INVALID` at the `image` path. A value that is not canonical base64 with correct padding returns `INVALID_BASE64_FORMAT`. An image over the resolved ceiling returns `IMAGE_SIZE_EXCEEDS_LIMIT` at the same path, and `maxSize` is the ceiling that applied. +The remaining encoded part is bounded to 139810136 characters, the base64 expansion of a 104857600 byte image. A longer value returns the validation code `BASE64_LENGTH_INVALID` at the `image` path. A value that is not canonical base64 with correct padding returns `INVALID_BASE64_FORMAT`. An image over the resolved ceiling returns `IMAGE_SIZE_EXCEEDS_LIMIT` at the same path, and `maxSize` is the ceiling that applied. -The 699052 character bound does not move with the limit key, so a resolved ceiling above 524288 bytes admits no more than 524289 decoded bytes. +The character bound does not move with the limit key, so a resolved ceiling above 104857600 bytes admits no more than 104857602 decoded bytes. Accepted upload formats are PNG, JPEG, WebP, GIF, APNG, AVIF, HEIC, HEIF, JXL, and SVG. Fluxer takes the format and the animation state from the decoded image. An animated AVIF is rejected. diff --git a/fluxer_docs/src/content/docs/http-api/guild-members.mdx b/fluxer_docs/src/content/docs/http-api/guild-members.mdx index 2bab11f5b..57a73aa03 100644 --- a/fluxer_docs/src/content/docs/http-api/guild-members.mdx +++ b/fluxer_docs/src/content/docs/http-api/guild-members.mdx @@ -127,7 +127,7 @@ The request body shared by [Modify current guild member](#modify-current-guild-m A `roles` entry that does not resolve to an existing role of the guild is dropped from the replacement, so a request naming only unknown roles clears the member's role set. -An `avatar` or `banner` base64 payload longer than 13981016 characters returns `BASE64_LENGTH_INVALID`, and a malformed one returns `INVALID_BASE64_FORMAT`. Fluxer then checks the decoded bytes against the instance-configured `avatar_max_size` [limit](/http-api/instance/#limit-keys), whose stock value is 10485760. The same limit applies to both fields. The decoded image must also be in a format the field accepts, and an animated AVIF is rejected. Pixel dimensions are not enforced. A value that is too large returns `IMAGE_SIZE_EXCEEDS_LIMIT`, and one whose format or animation is not allowed returns `INVALID_IMAGE_FORMAT`. +An `avatar` or `banner` base64 payload longer than 139810136 characters returns `BASE64_LENGTH_INVALID`, and a malformed one returns `INVALID_BASE64_FORMAT`. Fluxer then checks the decoded bytes against the instance-configured `avatar_max_size` [limit](/http-api/instance/#limit-keys), whose stock value is 10485760. The same limit applies to both fields. The decoded image must also be in a format the field accepts, and an animated AVIF is rejected. Pixel dimensions are not enforced. A value that is too large returns `IMAGE_SIZE_EXCEEDS_LIMIT`, and one whose format or animation is not allowed returns `INVALID_IMAGE_FORMAT`. Null and any `communication_disabled_until` that is not in the future both clear the timeout. A time more than 365.25 days ahead returns `TIMEOUT_CANNOT_EXCEED_365_DAYS`, and a value that passes the schema but is not a real instant returns `INVALID_TIMEOUT_VALUE`. `timeout_reason` has no effect unless `communication_disabled_until` is also supplied. diff --git a/fluxer_docs/src/content/docs/http-api/guild-stickers.mdx b/fluxer_docs/src/content/docs/http-api/guild-stickers.mdx index c9141c300..45003ac3a 100644 --- a/fluxer_docs/src/content/docs/http-api/guild-stickers.mdx +++ b/fluxer_docs/src/content/docs/http-api/guild-stickers.mdx @@ -68,7 +68,7 @@ The item shape shared by [Create guild sticker](#create-guild-sticker) and each | name1 | string | The name of the sticker (2-30 characters) | | description?2 | ?string | The sticker description (1-500 characters), or null for none | | tags? | array[string] | [Sticker tags](#sticker-tags), defaulting to an empty array | -| image3 | base64 string | The image data, with at most 524288 decoded bytes | +| image3 | base64 string | The image data (at most the resolved `sticker_max_size` limit in decoded bytes, 524288 by default) | 1 The value is trimmed before the bound applies, as it is for `description` and each tag @@ -76,9 +76,9 @@ The item shape shared by [Create guild sticker](#create-guild-sticker) and each 3 A data URL prefix is accepted and everything up to the first comma is removed before any bound applies -The remaining encoded part is bounded to 699052 characters, which encodes a full 524288 byte image, and a longer value returns the validation code `BASE64_LENGTH_INVALID` at the `image` path. A value that is not canonical base64 with correct padding returns `INVALID_BASE64_FORMAT`. +The remaining encoded part is bounded to 139810136 characters, which encodes a 104857600 byte image, and a longer value returns the validation code `BASE64_LENGTH_INVALID` at the `image` path. A value that is not canonical base64 with correct padding returns `INVALID_BASE64_FORMAT`. -Decoded bytes are at most the operator-configured `sticker_max_size` [limit key](/http-api/instance/#limit-keys) value, resolved against the guild's complete feature set and defaulting to 524288. A larger image is rejected on the `image` path with `IMAGE_SIZE_EXCEEDS_LIMIT`, whose message names the resolved ceiling. An image over 524289 bytes exceeds the 699052 character bound and returns `BASE64_LENGTH_INVALID`, so a configured value above 524289 never applies. +Decoded bytes are at most the operator-configured `sticker_max_size` [limit key](/http-api/instance/#limit-keys) value, resolved against the guild's complete feature set and defaulting to 524288. A larger image is rejected on the `image` path with `IMAGE_SIZE_EXCEEDS_LIMIT`, whose message names the resolved ceiling. The character bound does not move with the limit key, so a configured value above 104857600 admits no more than 104857602 decoded bytes. Accepted upload formats are PNG, JPEG, APNG, GIF, WebP, AVIF, and SVG. Fluxer takes the format from the decoded image. An animated AVIF is rejected. Fluxer detects animation state from the image, so this object has no animation field. diff --git a/fluxer_docs/src/content/docs/http-api/guilds.mdx b/fluxer_docs/src/content/docs/http-api/guilds.mdx index 25d92f584..c6aaaacce 100644 --- a/fluxer_docs/src/content/docs/http-api/guilds.mdx +++ b/fluxer_docs/src/content/docs/http-api/guilds.mdx @@ -522,7 +522,7 @@ Creates a guild owned by the caller. Requires a user session credential. Returns A `name` outside 1 to 100 characters after normalisation returns `STRING_LENGTH_INVALID`. A blocked name returns 403 `CONTENT_BLOCKED`. -The `icon` base64 payload is bounded to 1 to 13981016 characters and is otherwise rejected with `BASE64_LENGTH_INVALID`, and a payload that is not valid base64 with `INVALID_BASE64_FORMAT`. The decoded image must fit within the instance `avatar_max_size` limit, which defaults to the 10 MiB ceiling, and a larger image is rejected with `IMAGE_SIZE_EXCEEDS_LIMIT`. PNG, JPEG, WebP, GIF, APNG, AVIF, HEIC, HEIF, JXL, and SVG are accepted. An animated AVIF is rejected with `INVALID_IMAGE_FORMAT`. +The `icon` base64 payload is bounded to 1 to 139810136 characters and is otherwise rejected with `BASE64_LENGTH_INVALID`, and a payload that is not valid base64 with `INVALID_BASE64_FORMAT`. The decoded image must fit within the instance `avatar_max_size` limit, which defaults to the 10 MiB ceiling, and a larger image is rejected with `IMAGE_SIZE_EXCEEDS_LIMIT`. PNG, JPEG, WebP, GIF, APNG, AVIF, HEIC, HEIF, JXL, and SVG are accepted. An animated AVIF is rejected with `INVALID_IMAGE_FORMAT`. Without a template the guild is created with a `Text Channels` category holding a text channel named `general`, a `Voice Channels` category holding a voice channel named `General`, and an everyone role with the default permission set plus `CREATE_PUBLIC_THREADS`, `CREATE_PRIVATE_THREADS`, and `SEND_MESSAGES_IN_THREADS`. The `general` channel becomes the system channel. diff --git a/fluxer_docs/src/content/docs/http-api/instance.mdx b/fluxer_docs/src/content/docs/http-api/instance.mdx index 9a1e35c6b..d9546cf16 100644 --- a/fluxer_docs/src/content/docs/http-api/instance.mdx +++ b/fluxer_docs/src/content/docs/http-api/instance.mdx @@ -346,8 +346,8 @@ Each key names one limit. A key whose name begins with `feature_` is a feature g | Value | Description | | --- | --- | -| avatar_max_size | Maximum file size for avatar uploads in bytes, in user scope | -| emoji_max_size | Maximum file size for emoji uploads in bytes, in guild scope | +| avatar_max_size | Maximum file size for avatar uploads in bytes, effective up to 104857600, in user scope | +| emoji_max_size | Maximum file size for emoji uploads in bytes, effective up to 104857600, in guild scope | | feature_animated_avatar | Allow animated avatar uploads, in user scope | | feature_animated_banner | Allow animated banner uploads, in user scope | | feature_custom_discriminator | Allow custom discriminator selection, in user scope. Ignored for people on an instance with the `none` tag style | @@ -389,7 +389,7 @@ Each key names one limit. A key whose name begins with `feature_` is a feature g | max_voice_message_duration | Maximum voice message duration in seconds, in user and guild scope | | max_webhooks_per_channel | Maximum webhooks per channel, in guild scope | | max_webhooks_per_guild | Maximum webhooks per guild, in guild scope | -| sticker_max_size | Maximum file size for sticker uploads in bytes, in guild scope | +| sticker_max_size | Maximum file size for sticker uploads in bytes, effective up to 104857600, in guild scope | 1 Emoji limits are enforced as one shared total against `max_guild_emojis`, and the aliases exist only for older clients. The default of each alias equals the `max_guild_emojis` default diff --git a/fluxer_docs/src/content/docs/http-api/users/current-user.mdx b/fluxer_docs/src/content/docs/http-api/users/current-user.mdx index efe46db9c..479dde22f 100644 --- a/fluxer_docs/src/content/docs/http-api/users/current-user.mdx +++ b/fluxer_docs/src/content/docs/http-api/users/current-user.mdx @@ -98,7 +98,7 @@ Modifies the current account and returns the resulting [user](/http-api/users/#u 5 A claimed account also supplies `password`, and a body that omits it is rejected with `PASSWORD_NOT_SET`. When the instance checks for breached passwords, a breached one is rejected with [`PASSWORD_IS_TOO_COMMON`](/http-api/errors/) -6 The value may have a `data:...;base64,` prefix. The remaining payload runs 1 to 13981016 characters +6 The value may have a `data:...;base64,` prefix. The remaining payload runs 1 to 139810136 characters 7 Every non-null value requires the animated banner entitlement, which gates every profile banner, and is otherwise rejected with [`BANNERS_REQUIRE_PREMIUM`](/http-api/errors/) diff --git a/fluxer_docs/src/content/docs/http-api/webhooks.mdx b/fluxer_docs/src/content/docs/http-api/webhooks.mdx index cbd3d595e..2c2edf499 100644 --- a/fluxer_docs/src/content/docs/http-api/webhooks.mdx +++ b/fluxer_docs/src/content/docs/http-api/webhooks.mdx @@ -839,7 +839,7 @@ The guild allowance is the resolved [max_webhooks_per_guild](/http-api/instance/ | name | string | Webhook name (1-80 characters) | | avatar?1 | ?base64 string | Base64-encoded avatar image, or null | -1 A leading data URI header is stripped at the first comma before decoding, and the base64 payload is 1 through 13981016 characters +1 A leading data URI header is stripped at the first comma before decoding, and the base64 payload is 1 through 139810136 characters The decoded bytes must be at most the resolved [avatar_max_size](/http-api/instance/#limit-keys) value, which is the 10 MiB ceiling of 10485760 bytes by default. The image must be an accepted avatar upload format, and an animated AVIF is rejected. diff --git a/fluxer_docs/src/content/docs/media-proxy/routes.mdx b/fluxer_docs/src/content/docs/media-proxy/routes.mdx index 9a1486cb5..dc1e89b3d 100644 --- a/fluxer_docs/src/content/docs/media-proxy/routes.mdx +++ b/fluxer_docs/src/content/docs/media-proxy/routes.mdx @@ -23,9 +23,11 @@ Every media read route can produce these statuses in addition to the ones in its | 403 | Forbidden | The [cross-origin read policy](/media-proxy/overview/#cross-origin-reads) refuses the request `Origin` on an `mp` or `upload` endpoint | | 404 | Not found | The path is not a media route, the endpoint runs in `relay` mode, or the object does not exist | | 405 | Method not allowed | The method is neither `GET` nor `HEAD` | -| 413 | Payload too large | The stored object exceeds the 500 MiB media bound | +| 413 | Payload too large | The stored object exceeds the 500 MiB media bound and the route neither streams it unchanged nor spools it | | 502 | Bad gateway | The object store could not be read | +The 500 MiB media bound applies where a route holds an object in memory. A read that streams the stored object unchanged, such as an attachment read without a transformation parameter, has no size bound. An attachment read that asks for a frame of a video above the bound copies the video to the [spool directory](/operator/configuration/#media-proxy-settings) first, one video at a time, and returns 413 when the video exceeds the spool ceiling. Any other transformation of an object above the bound returns 413. + ## Get attachment diff --git a/fluxer_docs/src/content/docs/operator/configuration.mdx b/fluxer_docs/src/content/docs/operator/configuration.mdx index db8f6ff4e..dc3cb3ae3 100644 --- a/fluxer_docs/src/content/docs/operator/configuration.mdx +++ b/fluxer_docs/src/content/docs/operator/configuration.mdx @@ -1379,7 +1379,7 @@ Default `536870912`, 512 MiB. The memory all buffered relay uploads may hold at #### `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_DIR` -Defaults to the system temporary directory. Where relay bodies spool. Must be writable. Compose forwards it from `.env`. +Defaults to the system temporary directory. Where relay bodies spool, and where `media-proxy` copies a stored audio or video object above 500 MiB when it has to read inside the file. That happens when the API asks for the media metadata or a frame of an upload or a stored object, and when an attachment read asks for a frame of a video. Must be writable. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_CHUNK_BYTES` @@ -1387,7 +1387,11 @@ Default `1048576`. Spool chunk size. Accepts 64 KiB to 64 MiB. Compose forwards #### `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_MAX_TOTAL_BYTES` -Default `8589934592`. Total spool ceiling. Accepts 0 to 256 GiB, and must be at or above `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_MAX_BODY_BYTES`, which puts the usable floor at 500 MiB by default. Compose forwards it from `.env`. +Default `8589934592`. Total spool ceiling, shared by relay bodies and the copies of stored audio and video above 500 MiB. A relay upload that does not fit beside the copies in progress fails. An upload larger than the ceiling is stored as a plain file without media metadata, and a stored video larger than the ceiling has no poster frame. Accepts 0 to 256 GiB, and must be at or above `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_MAX_BODY_BYTES`, which puts the usable floor at 500 MiB by default. Compose forwards it from `.env`. + +Sending an audio or video attachment above 500 MiB holds up to three copies of it on local disk at once. `media-proxy` holds one in the spool directory, and `api` holds the download and the copy it rewrites without metadata in its temporary directory. Leave free space of three times the attachment size limit for each send you expect at the same moment. + +`api` waits 30 seconds for the metadata of an upload up to 500 MiB. Above that it waits 30 seconds plus one second for each 16 MiB of the upload, about 94 seconds for 1 GiB and 542 seconds for 8 GiB. In that time `media-proxy` copies the upload out of object storage, reads it, and hashes it. A deployment that cannot sustain 16 MiB per second through those steps stores the upload as a plain file. #### `FLUXER_MEDIA_PROXY_MAX_NATIVE_TRANSFORMS` @@ -1765,6 +1769,12 @@ The premium tier name and information link, the billing switch, the Stripe secre Size-based attachment lifetimes. The built-in default is on. +Turning expiry off hides every expiry date at once and stops deletion. The stored dates are kept. + +On a self-hosted instance, changes also apply to attachments that were uploaded earlier. Turning expiry on, or changing a size or lifetime rule, starts a background job that works through existing messages in chunks and recalculates the expiry of their attachments. No attachment is deleted while that job runs. An attachment whose date passed while expiry was off, or that the new settings would expire sooner, is kept for at least the renew window first. Until the job reaches a message, its attachments show the state they had before, so one whose date passed while expiry was off shows as expired until then. + +The official hosted instance does not run that job. There, a change applies to attachments uploaded from then on, and deletion of expired attachments is never paused. + #### Instance Config, Gateway Rollout Configuration Session and guild rollout percentages, RPC timeouts, and Gateway concurrency. diff --git a/fluxer_media_proxy/Cargo.toml b/fluxer_media_proxy/Cargo.toml index c7e7fea9c..882003183 100644 --- a/fluxer_media_proxy/Cargo.toml +++ b/fluxer_media_proxy/Cargo.toml @@ -33,6 +33,7 @@ http-body = "1.1.0" http-body-util = "0.1.5" hyper = {version = "1.11.1", default-features = false} libc = "0.2.189" +memmap2 = "0.9.11" moka = {version = "0.12.16", features = ["sync"]} parking_lot = "0.12.5" percent-encoding = "2.3.2" diff --git a/fluxer_media_proxy/src/media_process/av_metadata.rs b/fluxer_media_proxy/src/media_process/av_metadata.rs index 272e287c7..1ceb340ff 100644 --- a/fluxer_media_proxy/src/media_process/av_metadata.rs +++ b/fluxer_media_proxy/src/media_process/av_metadata.rs @@ -120,9 +120,6 @@ pub fn probe_av_metadata( if input.is_empty() { return Err(MediaError::MediaDecodeFailed); } - if input.len() > media_limits.max_media_proxy_bytes() { - return Err(MediaError::StreamTooLong); - } if preview_max_dimension > NSFW_PREVIEW_MAX_DIMENSION { return Err(MediaError::InvalidImageDimensions); } diff --git a/fluxer_media_proxy/src/media_process/metadata.rs b/fluxer_media_proxy/src/media_process/metadata.rs index f7ca5b105..b56c3dc16 100644 --- a/fluxer_media_proxy/src/media_process/metadata.rs +++ b/fluxer_media_proxy/src/media_process/metadata.rs @@ -137,15 +137,15 @@ fn metadata_blocking(request: MetadataBlockingRequest<'_>) -> Result constants::MAX_MEDIA_PROXY_BYTES { - return Err(MediaError::StreamTooLong); - } let sniffed = mime::sniff(input); if !mime::is_supported_media_mime(sniffed.mime) { return Err(MediaError::UnsupportedMediaType); } let initial_category = mime::category(sniffed.mime).ok_or(MediaError::UnsupportedMediaType)?; let is_image = initial_category == mime::Category::Image; + if is_image && input.len() > constants::MAX_MEDIA_PROXY_BYTES { + return Err(MediaError::StreamTooLong); + } let dims = if is_image { let dims = probe_image_dims(media_limits, input)?; validate_jxl_decode_pixels(sniffed.mime, dims)?; diff --git a/fluxer_media_proxy/src/media_process/mod.rs b/fluxer_media_proxy/src/media_process/mod.rs index 32c33cd1d..783f33009 100644 --- a/fluxer_media_proxy/src/media_process/mod.rs +++ b/fluxer_media_proxy/src/media_process/mod.rs @@ -37,7 +37,7 @@ pub use metadata::{ MetadataOptions, PreparedMetadata, finish_metadata, metadata_json_with_options, prepare_metadata, }; -pub use native_cost::{AV_NATIVE_COST_BYTES, image_transform_cost, metadata_cost}; +pub use native_cost::{AV_NATIVE_COST_BYTES, av_native_cost, image_transform_cost, metadata_cost}; pub use nsfw_processing::encode_static_image_for_nsfw; pub use transform::transform_image; pub use video_thumbnail::{ diff --git a/fluxer_media_proxy/src/media_process/native_cost.rs b/fluxer_media_proxy/src/media_process/native_cost.rs index 756fc67b0..157c25c5d 100644 --- a/fluxer_media_proxy/src/media_process/native_cost.rs +++ b/fluxer_media_proxy/src/media_process/native_cost.rs @@ -5,8 +5,8 @@ use super::image_probe::probe_image_dims; use super::loaded_image::static_output_pixels; use super::native_runtime::ensure_vips_init; use crate::{ - image_quality::ImageQuality, image_transform::ImageOptions, media_limits::MediaLimits, mime, - output_format::OutputFormat, + constants, image_quality::ImageQuality, image_transform::ImageOptions, + media_limits::MediaLimits, mime, output_format::OutputFormat, }; pub const AV_NATIVE_COST_BYTES: usize = 256 << 20; @@ -53,6 +53,12 @@ pub fn image_transform_cost( Ok(input.len().saturating_add(pixel_cost)) } +pub fn av_native_cost(input_len: usize) -> usize { + input_len + .min(constants::MAX_MEDIA_PROXY_BYTES) + .saturating_add(AV_NATIVE_COST_BYTES) +} + pub fn metadata_cost(input: &[u8], media_limits: &MediaLimits) -> Result { let sniffed_mime = mime::sniff(input).mime; match mime::category(sniffed_mime) { @@ -64,9 +70,7 @@ pub fn metadata_cost(input: &[u8], media_limits: &MediaLimits) -> Result { - Ok(input.len().saturating_add(AV_NATIVE_COST_BYTES)) - } + Some(mime::Category::Video | mime::Category::Audio) => Ok(av_native_cost(input.len())), _ => Ok(input.len()), } } diff --git a/fluxer_media_proxy/src/media_process/nsfw_processing/mod.rs b/fluxer_media_proxy/src/media_process/nsfw_processing/mod.rs index 5fc5058c1..a634d2b70 100644 --- a/fluxer_media_proxy/src/media_process/nsfw_processing/mod.rs +++ b/fluxer_media_proxy/src/media_process/nsfw_processing/mod.rs @@ -111,7 +111,6 @@ pub(super) async fn classify_nsfw_buffers( fn nsfw_video_scan_buffers(source: &NSFWScanSource<'_>) -> Result>, MediaError> { extract_video_frames_for_nsfw(VideoNSFWFramesRequest { - media_limits: source.media_limits, input: source.input, duration_seconds: source.duration_seconds, deadline_ms: source.deadline_ms, @@ -363,7 +362,6 @@ pub(super) fn nsfw_frame_seed(input: &[u8]) -> u64 { } pub(super) struct VideoNSFWFramesRequest<'a> { - pub(super) media_limits: &'a MediaLimits, pub(super) input: &'a [u8], pub(super) duration_seconds: Option, pub(super) deadline_ms: Option, @@ -374,13 +372,12 @@ pub(super) fn extract_video_frames_for_nsfw( ) -> Result>, MediaError> { use rand::SeedableRng as _; let VideoNSFWFramesRequest { - media_limits, input, duration_seconds, deadline_ms, } = request; - if input.is_empty() || input.len() > media_limits.max_media_proxy_bytes() { - return Err(MediaError::StreamTooLong); + if input.is_empty() { + return Err(MediaError::MediaDecodeFailed); } let runtime = vips_runtime()?; let seed = nsfw_frame_seed(input); diff --git a/fluxer_media_proxy/src/media_process/nsfw_processing/tests.rs b/fluxer_media_proxy/src/media_process/nsfw_processing/tests.rs index aa1b5ef5e..09f552870 100644 --- a/fluxer_media_proxy/src/media_process/nsfw_processing/tests.rs +++ b/fluxer_media_proxy/src/media_process/nsfw_processing/tests.rs @@ -56,7 +56,6 @@ fn video_nsfw_frames_are_downscaled_to_the_frame_cap() { .probe .duration_seconds; let frames = extract_video_frames_for_nsfw(VideoNSFWFramesRequest { - media_limits: &media_limits, input: &mp4, duration_seconds: duration, deadline_ms: None, @@ -215,7 +214,6 @@ fn a_sample_beyond_the_packet_budget_keeps_the_frames_that_decoded() { }; let media_limits = limits(); let frames = extract_video_frames_for_nsfw(VideoNSFWFramesRequest { - media_limits: &media_limits, input: &mp4, duration_seconds: Some(12.0), deadline_ms: None, diff --git a/fluxer_media_proxy/src/media_process/tests/metadata.rs b/fluxer_media_proxy/src/media_process/tests/metadata.rs index 1ecdebc95..3cf2ea6f5 100644 --- a/fluxer_media_proxy/src/media_process/tests/metadata.rs +++ b/fluxer_media_proxy/src/media_process/tests/metadata.rs @@ -1,22 +1,33 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +use super::super::nsfw_processing::{VideoNSFWFramesRequest, extract_video_frames_for_nsfw}; use super::super::placeholder::{encode_thumbhash, optional_thumbhash}; use super::super::{ - MediaError, MetadataOptions, NSFW_PREVIEW_MAX_DIMENSION, metadata_json_with_options, - probe_av_metadata, + AV_NATIVE_COST_BYTES, MediaError, MetadataOptions, NSFW_PREVIEW_MAX_DIMENSION, + extract_video_thumbnail, metadata_cost, metadata_json_with_options, probe_av_metadata, }; use super::fixtures::{metadata_value, test_media_limits}; use crate::{ + constants, metrics::transform::TransformMetrics, mime, nsfw::{NSFWClient, NSFWPolicy}, + output_format::OutputFormat, test_fixtures::{ fixture_audio_mp3_with_png_cover_art, fixture_audio_mp4_with_attached_picture, fixture_audio_only_mp4, fixture_h264_mp4, fixture_mkv_with_png_video_stream, fixture_mp4_with_undecodable_video, synthetic_png, synthetic_wav, }, }; -use std::sync::Arc; +use std::{io::Write as _, sync::Arc}; + +fn mapped_past_the_buffer_bound(head: &[u8]) -> memmap2::Mmap { + let mut file = tempfile::tempfile().expect("temp file"); + file.write_all(head).expect("written head"); + file.set_len(constants::MAX_MEDIA_PROXY_BYTES as u64 + 1) + .expect("sparse tail"); + unsafe { memmap2::Mmap::map(&file).expect("mapped file") } +} #[test] fn metadata_json_includes_dimensions_and_placeholder() { @@ -258,3 +269,54 @@ fn metadata_refuses_jxl_above_the_decode_pixel_cap() { }); assert_eq!(MediaError::InvalidImageDimensions, err); } + +#[test] +fn metadata_refuses_an_image_past_the_buffer_bound() { + let png = mapped_past_the_buffer_bound(&synthetic_png(16, 16)); + assert_eq!("image/png", mime::sniff(&png).mime); + let err = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() + .block_on(async { + metadata_json_with_options( + &png, + "huge.png", + MetadataOptions::default(), + &test_media_limits(), + &NSFWClient::disabled(), + &TransformMetrics::new(), + ) + .await + .unwrap_err() + }); + assert_eq!(MediaError::StreamTooLong, err); +} + +#[test] +fn native_cost_of_a_video_past_the_buffer_bound_leaves_out_the_mapped_tail() { + let mp4 = mapped_past_the_buffer_bound(&fixture_h264_mp4()); + assert_eq!( + constants::MAX_MEDIA_PROXY_BYTES + AV_NATIVE_COST_BYTES, + metadata_cost(&mp4, &test_media_limits()).expect("video cost") + ); +} + +#[test] +fn video_reads_accept_an_input_past_the_buffer_bound() { + let mp4 = mapped_past_the_buffer_bound(&fixture_h264_mp4()); + let limits = test_media_limits(); + + let probe = probe_av_metadata(&mp4, NSFW_PREVIEW_MAX_DIMENSION, &limits, None) + .expect("probed video") + .probe; + assert!(probe.has_video); + let poster = extract_video_thumbnail(&mp4, OutputFormat::WebP, &limits).expect("poster frame"); + assert_eq!("image/webp", mime::sniff(&poster.bytes).mime); + let frames = extract_video_frames_for_nsfw(VideoNSFWFramesRequest { + input: &mp4, + duration_seconds: probe.duration_seconds, + deadline_ms: None, + }) + .expect("sampled frames"); + assert!(!frames.is_empty()); +} diff --git a/fluxer_media_proxy/src/media_process/tests/nsfw.rs b/fluxer_media_proxy/src/media_process/tests/nsfw.rs index e74947c7c..4c22e2c27 100644 --- a/fluxer_media_proxy/src/media_process/tests/nsfw.rs +++ b/fluxer_media_proxy/src/media_process/tests/nsfw.rs @@ -4,7 +4,7 @@ use super::super::nsfw_processing::{ VideoNSFWFramesRequest, compute_frame_sample_timestamps, extract_video_frames_for_nsfw, nsfw_frame_seed, }; -use super::super::probe_av_metadata; +use super::super::{MediaError, probe_av_metadata}; use super::fixtures::test_media_limits; #[test] @@ -58,7 +58,6 @@ fn extract_video_frames_for_nsfw_returns_multiple_frames() { .probe .duration_seconds; let frames = extract_video_frames_for_nsfw(VideoNSFWFramesRequest { - media_limits: &limits, input: &bytes, duration_seconds: duration, deadline_ms: None, @@ -74,3 +73,17 @@ fn extract_video_frames_for_nsfw_returns_multiple_frames() { assert_eq!(&f[..2], &[0xFF, 0xD8], "not a JPEG"); } } + +#[test] +fn an_empty_input_fails_to_decode_for_the_probe_and_the_frame_sampler() { + let frames = extract_video_frames_for_nsfw(VideoNSFWFramesRequest { + input: &[], + duration_seconds: None, + deadline_ms: None, + }); + assert_eq!(Some(MediaError::MediaDecodeFailed), frames.err()); + assert_eq!( + Some(MediaError::MediaDecodeFailed), + probe_av_metadata(&[], 0, &test_media_limits(), None).err() + ); +} diff --git a/fluxer_media_proxy/src/media_process/video_thumbnail.rs b/fluxer_media_proxy/src/media_process/video_thumbnail.rs index 3adb04b60..351f742b3 100644 --- a/fluxer_media_proxy/src/media_process/video_thumbnail.rs +++ b/fluxer_media_proxy/src/media_process/video_thumbnail.rs @@ -45,9 +45,6 @@ pub fn extract_video_thumbnail_with_options( options: VideoThumbnailOptions, media_limits: &MediaLimits, ) -> Result { - if input.len() > media_limits.max_media_proxy_bytes() { - return Err(MediaError::StreamTooLong); - } let VideoThumbnailOptions { format, width, diff --git a/fluxer_media_proxy/src/server/media_operations/input.rs b/fluxer_media_proxy/src/server/media_operations/input.rs index d921e2550..73e2b43c1 100644 --- a/fluxer_media_proxy/src/server/media_operations/input.rs +++ b/fluxer_media_proxy/src/server/media_operations/input.rs @@ -4,18 +4,24 @@ use super::failure::MediaFailure; use crate::{ byte_budget::BudgetedBytes, constants::{self, AssetExtension}, + mime, server::{ external::fetch_external, params::url_filename, self_origin::{self, SelfOrigin}, state::AppState, }, - storage::StorageError, + spool::{SpoolError, spool_to_temp}, + storage::{self, StorageError, StreamObject}, }; +use axum::body::Body; use base64::{Engine as _, engine::general_purpose}; use bytes::Bytes; +use futures_util::StreamExt as _; use tracing::warn; +pub(in crate::server) const STORED_SNIFF_PREFIX_BYTES: usize = 8192; + pub(in crate::server) enum MediaInput { Base64 { data: String, @@ -89,17 +95,20 @@ pub(in crate::server) async fn load_media_input( upload_filename, filename, } => { - let object = app - .store - .read_object(&app.cfg.storage.bucket_uploads, &upload_filename) - .await - .map_err(|err| { - let failure = MediaFailure::MetadataUploadRead; - warn!(reason = failure.code(), key = upload_filename.as_str(), %err); - failure - })?; + let data = read_stored_input( + app, + &app.cfg.storage.bucket_uploads, + &upload_filename, + limit.decoded_bytes, + ) + .await + .map_err(|err| { + let failure = MediaFailure::MetadataUploadRead; + warn!(reason = failure.code(), key = upload_filename.as_str(), %err); + failure + })?; Ok(LoadedMediaInput { - data: object.data, + data, filename: filename.unwrap_or(upload_filename), }) } @@ -108,13 +117,15 @@ pub(in crate::server) async fn load_media_input( key, filename, } => { - let object = app.store.read_object(&bucket, &key).await.map_err(|err| { - let failure = MediaFailure::MetadataS3Read; - warn!(reason = failure.code(), bucket = bucket.as_str(), key = key.as_str(), %err); - failure - })?; + let data = read_stored_input(app, &bucket, &key, limit.decoded_bytes) + .await + .map_err(|err| { + let failure = MediaFailure::MetadataS3Read; + warn!(reason = failure.code(), bucket = bucket.as_str(), key = key.as_str(), %err); + failure + })?; Ok(LoadedMediaInput { - data: object.data, + data, filename: filename.unwrap_or(key), }) } @@ -160,6 +171,89 @@ async fn load_external_url( }) } +async fn read_stored_input( + app: &AppState, + bucket: &str, + key: &str, + buffer_limit: usize, +) -> Result { + let budget = storage::unversioned_read_budget(buffer_limit); + match app + .store + .read_object_limited(bucket, key, buffer_limit, &budget) + .await + { + Err(StorageError::StreamTooLong) => Ok(spool_stored_media( + app, + bucket, + key, + &[mime::Category::Video, mime::Category::Audio], + ) + .await? + .data), + other => Ok(other?.data.as_bytes().clone()), + } +} + +pub(in crate::server) async fn spool_stored_media( + app: &AppState, + bucket: &str, + key: &str, + categories: &[mime::Category], +) -> Result { + let range = format!("bytes=0-{}", STORED_SNIFF_PREFIX_BYTES - 1); + let head = app.store.stream_object(bucket, key, Some(&range)).await?; + let prefix = read_sniff_prefix(head.body) + .await + .map_err(|err| StorageError::ObjectStorage(err.into()))?; + let sniffed = mime::sniff(&prefix).mime; + if !mime::is_supported_media_mime(sniffed) + || !mime::category(sniffed).is_some_and(|category| categories.contains(&category)) + { + return Err(StorageError::StreamTooLong); + } + let object = app.store.stream_object(bucket, key, None).await?; + Ok(storage::Object { + data: spool_stream(app, object).await?, + content_type: sniffed.to_owned(), + }) +} + +async fn spool_stream(app: &AppState, object: StreamObject) -> Result { + let Some(content_length) = object.content_length else { + return Err(StorageError::StreamTooLong); + }; + let relay = &app.cfg.upload_relay; + let spooled = spool_to_temp( + object.body, + Some(content_length), + relay.spool_max_total_bytes, + &relay.spool_dir, + relay.spool_chunk_bytes, + relay.spool_max_total_bytes, + ) + .await + .map_err(|err| match err { + SpoolError::PayloadTooLarge => StorageError::StreamTooLong, + other => StorageError::ObjectStorage(other.into()), + })?; + Ok(Bytes::from_owner(spooled.into_mapped()?)) +} + +pub(in crate::server) async fn read_sniff_prefix(body: Body) -> Result, axum::Error> { + let mut prefix = Vec::with_capacity(STORED_SNIFF_PREFIX_BYTES); + let mut chunks = body.into_data_stream(); + while prefix.len() < STORED_SNIFF_PREFIX_BYTES { + let Some(chunk) = chunks.next().await else { + break; + }; + let chunk = chunk?; + let wanted = chunk.len().min(STORED_SNIFF_PREFIX_BYTES - prefix.len()); + prefix.extend_from_slice(&chunk[..wanted]); + } + Ok(prefix) +} + async fn read_own_object( app: &AppState, bucket: &str, @@ -182,3 +276,220 @@ async fn read_own_object( pub(super) fn retained_input_bytes(data: BudgetedBytes) -> Bytes { Bytes::from_owner(data) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + config::Config, + storage::tests::{FakeObject, FakeS3, fake_s3}, + }; + use axum::http::{Method, StatusCode, header}; + + const BUFFER_LIMIT: MediaInputLimit = MediaInputLimit { + encoded_bytes: 1024, + decoded_bytes: 1024, + }; + + fn local_storage_app(root: &std::path::Path) -> AppState { + let spool_dir = root.join("spool"); + std::fs::create_dir(&spool_dir).unwrap(); + let cfg = Config::load_from_iter([ + ("FLUXER_MEDIA_PROXY_SECRET_KEY", "secret".to_owned()), + ("FLUXER_MEDIA_PROXY_STORAGE_BACKEND", "local".to_owned()), + ( + "FLUXER_MEDIA_PROXY_STORAGE_ROOT", + root.display().to_string(), + ), + ( + "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_DIR", + spool_dir.display().to_string(), + ), + ]) + .expect("test config"); + AppState::for_tests(cfg) + } + + #[tokio::test] + async fn a_stored_input_larger_than_the_buffer_limit_is_read_from_a_spooled_copy() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().canonicalize().unwrap(); + let app = local_storage_app(&root); + let uploads = app.cfg.storage.bucket_uploads.clone(); + let cdn = app.cfg.storage.bucket_cdn.clone(); + let mp4 = crate::test_fixtures::fixture_h264_mp4(); + assert!(mp4.len() > BUFFER_LIMIT.decoded_bytes); + for bucket in [&uploads, &cdn] { + app.store + .write_object(bucket, "big.mp4", &mp4, "video/mp4") + .await + .unwrap(); + } + + let upload = load_media_input( + &app, + MediaInput::Upload { + upload_filename: "big.mp4".to_owned(), + filename: None, + }, + BUFFER_LIMIT, + ) + .await + .unwrap_or_else(|failure| panic!("upload input failed: {}", failure.code())); + let stored = load_media_input( + &app, + MediaInput::Storage { + bucket: cdn, + key: "big.mp4".to_owned(), + filename: Some("named.mp4".to_owned()), + }, + BUFFER_LIMIT, + ) + .await + .unwrap_or_else(|failure| panic!("stored input failed: {}", failure.code())); + + assert_eq!(&mp4[..], &upload.data[..]); + assert_eq!("big.mp4", upload.filename); + assert_eq!(&mp4[..], &stored.data[..]); + assert_eq!("named.mp4", stored.filename); + let metadata = + crate::media_process::probe_av_metadata(&upload.data, 0, &app.media.limits(), None) + .unwrap(); + assert!(metadata.probe.has_video); + assert!( + std::fs::read_dir(root.join("spool")) + .unwrap() + .next() + .is_none(), + "a spooled copy is never visible in the spool directory" + ); + } + + fn upload_input(key: &str) -> MediaInput { + MediaInput::Upload { + upload_filename: key.to_owned(), + filename: None, + } + } + + fn body_reads(fake: &FakeS3) -> Vec> { + fake.requests() + .into_iter() + .filter(|(method, _, _, _)| *method == Method::GET) + .map(|(_, _, headers, _)| { + headers + .get(header::RANGE) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + }) + .collect() + } + + #[tokio::test] + async fn object_storage_spools_a_video_larger_than_the_buffer_limit() { + let fake = fake_s3().await; + let mp4 = crate::test_fixtures::fixture_h264_mp4(); + fake.put_object( + "uploads/big.mp4", + FakeObject { + body: mp4.clone(), + content_type: Some("application/octet-stream".to_owned()), + ..FakeObject::default() + }, + ); + let tmp = tempfile::tempdir().unwrap(); + let app = AppState::for_tests(fake.config(tmp.path())); + + let loaded = load_media_input(&app, upload_input("big.mp4"), BUFFER_LIMIT) + .await + .unwrap_or_else(|failure| panic!("upload input failed: {}", failure.code())); + + assert_eq!(&mp4[..], &loaded.data[..]); + assert_eq!( + vec![None, Some("bytes=0-8191".to_owned()), None], + body_reads(&fake) + ); + } + + #[tokio::test] + async fn an_oversized_input_that_is_not_audio_or_video_is_refused_from_its_prefix() { + let fake = fake_s3().await; + let mut png = crate::test_fixtures::synthetic_png(8, 8); + png.resize(BUFFER_LIMIT.decoded_bytes + 1, 0); + for (key, body) in [ + ("uploads/huge.png", png), + ( + "uploads/notes.mp4", + vec![b'a'; BUFFER_LIMIT.decoded_bytes + 1], + ), + ] { + fake.put_object( + key, + FakeObject { + body, + ..FakeObject::default() + }, + ); + } + let tmp = tempfile::tempdir().unwrap(); + let app = AppState::for_tests(fake.config(tmp.path())); + + for key in ["huge.png", "notes.mp4"] { + let failure = load_media_input(&app, upload_input(key), BUFFER_LIMIT) + .await + .err() + .expect("only audio and video are read past the buffer limit"); + assert_eq!("metadata_upload_read", failure.code(), "{key}"); + } + + assert_eq!( + vec![ + None, + Some("bytes=0-8191".to_owned()), + None, + Some("bytes=0-8191".to_owned()) + ], + body_reads(&fake) + ); + } + + #[tokio::test] + async fn a_stream_without_a_declared_length_is_not_spooled() { + let tmp = tempfile::tempdir().unwrap(); + let app = local_storage_app(&tmp.path().canonicalize().unwrap()); + let object = StreamObject { + body: Body::from_stream(futures_util::stream::iter([Ok::<_, std::io::Error>( + Bytes::from_static(b"payload"), + )])), + status: StatusCode::OK, + content_length: None, + content_type: "video/mp4".to_owned(), + byte_range: None, + total_length: None, + }; + + let refused = spool_stream(&app, object).await; + + assert!(matches!(refused, Err(StorageError::StreamTooLong))); + } + + #[tokio::test] + async fn a_missing_stored_input_is_reported_as_a_read_failure() { + let tmp = tempfile::tempdir().unwrap(); + let app = local_storage_app(&tmp.path().canonicalize().unwrap()); + + let failure = load_media_input( + &app, + MediaInput::Upload { + upload_filename: "missing.mp4".to_owned(), + filename: None, + }, + BUFFER_LIMIT, + ) + .await + .err() + .expect("a missing upload cannot load"); + + assert_eq!("metadata_upload_read", failure.code()); + } +} diff --git a/fluxer_media_proxy/src/server/media_operations/mod.rs b/fluxer_media_proxy/src/server/media_operations/mod.rs index 322602a58..b6cb746cd 100644 --- a/fluxer_media_proxy/src/server/media_operations/mod.rs +++ b/fluxer_media_proxy/src/server/media_operations/mod.rs @@ -4,7 +4,10 @@ mod failure; mod input; pub(in crate::server) use failure::MediaFailure; -pub(in crate::server) use input::{MediaInput, MediaInputLimit, load_media_input}; +pub(in crate::server) use input::{ + MediaInput, MediaInputLimit, STORED_SNIFF_PREFIX_BYTES, load_media_input, read_sniff_prefix, + spool_stored_media, +}; use crate::{ constants::{self, AssetExtension}, @@ -34,6 +37,9 @@ pub(in crate::server) async fn resolve_metadata( include_data: bool, ) -> Result { let mut input = load_media_input(app, input, MediaInputLimit::INTERNAL_REQUEST).await?; + if include_data && input.data.len() > constants::MAX_MEDIA_PROXY_BYTES { + return Err(MediaFailure::MetadataDecodedTooLarge); + } if include_data && metadata_input_is_svg(&input) { input = rasterize_metadata_svg(app, input).await?; } diff --git a/fluxer_media_proxy/src/server/routes/internal.rs b/fluxer_media_proxy/src/server/routes/internal.rs index be49df6e2..0f4b5105e 100644 --- a/fluxer_media_proxy/src/server/routes/internal.rs +++ b/fluxer_media_proxy/src/server/routes/internal.rs @@ -10,8 +10,8 @@ use crate::{ server::{ format_policy::is_svg_content_type, media_operations::{ - MediaFailure, MediaInput, MediaInputLimit, MetadataOutput, load_media_input, - resolve_metadata, + MediaFailure, MediaInput, MediaInputLimit, MetadataOutput, STORED_SNIFF_PREFIX_BYTES, + load_media_input, read_sniff_prefix, resolve_metadata, }, response::{ MediaResponse, @@ -34,12 +34,9 @@ use axum::{ }; use base64::{Engine as _, engine::general_purpose}; use bytes::Bytes; -use futures_util::StreamExt as _; use serde::Deserialize; use std::sync::Arc; -const UPLOAD_SNIFF_PREFIX_BYTES: usize = 8192; - #[derive(Debug, Deserialize)] struct MetadataRequest { version: Option, @@ -304,7 +301,7 @@ pub(in crate::server) async fn sniff_handler( Ok(req) => req, Err(_) => return text(StatusCode::BAD_REQUEST, "Bad Request"), }; - let range = format!("bytes=0-{}", UPLOAD_SNIFF_PREFIX_BYTES - 1); + let range = format!("bytes=0-{}", STORED_SNIFF_PREFIX_BYTES - 1); let object = match app .store .stream_object( @@ -337,20 +334,6 @@ pub(in crate::server) async fn sniff_handler( ) } -async fn read_sniff_prefix(body: Body) -> Result, axum::Error> { - let mut prefix = Vec::with_capacity(UPLOAD_SNIFF_PREFIX_BYTES); - let mut chunks = body.into_data_stream(); - while prefix.len() < UPLOAD_SNIFF_PREFIX_BYTES { - let Some(chunk) = chunks.next().await else { - break; - }; - let chunk = chunk?; - let wanted = chunk.len().min(UPLOAD_SNIFF_PREFIX_BYTES - prefix.len()); - prefix.extend_from_slice(&chunk[..wanted]); - } - Ok(prefix) -} - fn check_internal_auth(headers: &HeaderMap, secret: &str) -> bool { let Some(auth) = headers .get(header::AUTHORIZATION) @@ -885,7 +868,7 @@ mod tests { let stream_source = upload.clone(); let body = Body::from_stream(futures_util::stream::poll_fn(move |_| { assert!( - yielded < UPLOAD_SNIFF_PREFIX_BYTES, + yielded < STORED_SNIFF_PREFIX_BYTES, "the body was polled again after {yielded} bytes were yielded" ); let end = (yielded + 1000).min(stream_source.len()); @@ -896,7 +879,7 @@ mod tests { let prefix = read_sniff_prefix(body).await.expect("prefix read"); - assert_eq!(&upload[..UPLOAD_SNIFF_PREFIX_BYTES], prefix.as_slice()); + assert_eq!(&upload[..STORED_SNIFF_PREFIX_BYTES], prefix.as_slice()); } #[tokio::test] diff --git a/fluxer_media_proxy/src/server/stored/mod.rs b/fluxer_media_proxy/src/server/stored/mod.rs index 32e071ad3..3b7ace73d 100644 --- a/fluxer_media_proxy/src/server/stored/mod.rs +++ b/fluxer_media_proxy/src/server/stored/mod.rs @@ -20,6 +20,7 @@ use crate::{ effective_animated_image_output_format, extension_from_mime, is_svg_content_type, same_format_loaded_image_request_can_use_original_with_sniff, }, + media_operations::spool_stored_media, params::{ animated_param, asset_manual_format_override, asset_wants_animated, bool_param, filename_from_storage_key, @@ -31,12 +32,13 @@ use crate::{ }, state::AppState, transform::{ - ServeBytesRequest, + CachedVideoTransformRequest, ServeBytesRequest, cache::{ CachedTransformRequest, cached_transform, cached_transform_hit, coalescer_failure_response, }, cache_key::{TransformCacheKeyInput, transform_cache_key}, + cached_video_transform_response, execution::{coalesced_work_result, deadline_instant, run_transform}, original_image_content_type, parameters::TransformRoute, @@ -336,6 +338,66 @@ async fn read_attachment( .store .read_object(&app.cfg.storage.bucket_cdn, key) .await + { + Ok(object) => object, + Err(StorageError::StreamTooLong) => { + return serve_spooled_attachment_video(app, method, key, params, headers).await; + } + Err(err) => return storage_error_response(key, err), + }; + serve_bytes_or_transform( + app.media.transforms(), + ServeBytesRequest { + method, + data: object.data, + content_type: object.content_type, + cache_identity: key, + filename, + route: TransformRoute::Attachment, + params, + headers, + }, + ) + .await +} + +static SPOOLED_ATTACHMENT_READS: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +async fn serve_spooled_attachment_video( + app: &Arc, + method: Method, + key: &str, + params: &HashMap, + headers: &HeaderMap, +) -> Response { + let filename = filename_from_storage_key(key); + let cached = || { + cached_video_transform_response( + app.media.transforms(), + CachedVideoTransformRequest { + method: method.clone(), + cache_identity: key, + filename, + route: TransformRoute::Attachment, + params, + headers, + }, + ) + }; + if let Some(response) = cached() { + return response; + } + let _serial = SPOOLED_ATTACHMENT_READS.lock().await; + if let Some(response) = cached() { + return response; + } + let object = match spool_stored_media( + app, + &app.cfg.storage.bucket_cdn, + key, + &[mime::Category::Video], + ) + .await { Ok(object) => object, Err(err) => return storage_error_response(key, err), diff --git a/fluxer_media_proxy/src/server/stored/passthrough.rs b/fluxer_media_proxy/src/server/stored/passthrough.rs index eebdd355d..87a21df4c 100644 --- a/fluxer_media_proxy/src/server/stored/passthrough.rs +++ b/fluxer_media_proxy/src/server/stored/passthrough.rs @@ -111,9 +111,6 @@ pub(in crate::server) async fn serve_stored_passthrough_stream( Ok(value) => value, Err(err) => return storage_error_response(key, err), }; - if total_len > constants::MAX_MEDIA_PROXY_BYTES { - return storage_error_response(key, StorageError::StreamTooLong); - } streaming_media_response(StreamingMediaResponse { method, object, @@ -135,9 +132,6 @@ async fn serve_stored_passthrough_head( Ok(head) => head, Err(err) => return storage_error_response(key, err), }; - if head.content_length > constants::MAX_MEDIA_PROXY_BYTES as u64 { - return storage_error_response(key, StorageError::StreamTooLong); - } let content_type = passthrough_content_type(&head.content_type, key); if app.cfg.mode == DeploymentMode::Mp && (is_svg_content_type(&content_type) diff --git a/fluxer_media_proxy/src/server/stored/tests.rs b/fluxer_media_proxy/src/server/stored/tests.rs index a790dd6c2..2dfd95f09 100644 --- a/fluxer_media_proxy/src/server/stored/tests.rs +++ b/fluxer_media_proxy/src/server/stored/tests.rs @@ -285,6 +285,116 @@ async fn a_stored_passthrough_stream_declares_the_length_it_serves() { assert_eq!(&b"hello"[..], &response_body(response).await[..]); } +#[tokio::test] +async fn a_stored_passthrough_serves_objects_larger_than_the_buffer_cap() { + let tmp = tempfile::tempdir().unwrap(); + let storage_root = tmp.path().canonicalize().unwrap(); + let app = test_app_state(asset_test_config(&storage_root)); + let bucket = app.cfg.storage.bucket_cdn.clone(); + app.store + .write_object(&bucket, "a/big.mp4", b"head", "video/mp4") + .await + .unwrap(); + let total_len = constants::MAX_MEDIA_PROXY_BYTES as u64 + 1; + std::fs::OpenOptions::new() + .write(true) + .open(storage_root.join(&bucket).join("a/big.mp4")) + .unwrap() + .set_len(total_len) + .unwrap(); + let mut range = HeaderMap::new(); + range.insert(header::RANGE, "bytes=0-3".parse().unwrap()); + + let head = + passthrough::serve_stored_raw(&app, Method::HEAD, &bucket, "a/big.mp4", &HeaderMap::new()) + .await; + assert_eq!(StatusCode::OK, head.status()); + assert_eq!( + Some(total_len.to_string().as_str()), + head.headers() + .get(header::CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + ); + + let get = passthrough::serve_stored_raw(&app, Method::GET, &bucket, "a/big.mp4", &range).await; + assert_eq!(StatusCode::PARTIAL_CONTENT, get.status()); + assert_eq!( + Some(format!("bytes 0-3/{total_len}").as_str()), + get.headers() + .get(header::CONTENT_RANGE) + .and_then(|value| value.to_str().ok()) + ); + assert_eq!(&b"head"[..], &response_body(get).await[..]); +} + +fn poster_params() -> HashMap { + HashMap::from([ + ("format".to_owned(), "webp".to_owned()), + ("width".to_owned(), "64".to_owned()), + ]) +} + +#[tokio::test] +async fn a_spooled_attachment_video_serves_its_poster_and_then_answers_from_the_cache() { + let tmp = tempfile::tempdir().unwrap(); + let storage_root = tmp.path().canonicalize().unwrap(); + let app = test_app_state(asset_test_config(&storage_root)); + let bucket = app.cfg.storage.bucket_cdn.clone(); + let key = "attachments/1/2/clip.mp4"; + app.store + .write_object( + &bucket, + key, + &crate::test_fixtures::fixture_h264_mp4(), + "application/octet-stream", + ) + .await + .unwrap(); + let params = poster_params(); + let headers = HeaderMap::new(); + + let poster = serve_spooled_attachment_video(&app, Method::GET, key, ¶ms, &headers).await; + assert_eq!(StatusCode::OK, poster.status()); + let poster = response_body(poster).await; + assert_eq!("image/webp", mime::sniff(&poster).mime); + + std::fs::remove_file(storage_root.join(&bucket).join(key)).unwrap(); + let cached = serve_spooled_attachment_video(&app, Method::GET, key, ¶ms, &headers).await; + assert_eq!(StatusCode::OK, cached.status()); + assert_eq!(&poster[..], &response_body(cached).await[..]); +} + +#[tokio::test] +async fn a_spooled_attachment_read_refuses_everything_but_video() { + let tmp = tempfile::tempdir().unwrap(); + let storage_root = tmp.path().canonicalize().unwrap(); + let app = test_app_state(asset_test_config(&storage_root)); + let bucket = app.cfg.storage.bucket_cdn.clone(); + let params = poster_params(); + for (key, data) in [ + ( + "attachments/1/2/huge.png", + crate::test_fixtures::synthetic_png(8, 8), + ), + ( + "attachments/1/2/song.wav", + crate::test_fixtures::synthetic_wav(), + ), + ("attachments/1/2/disk.iso", vec![0u8; 4096]), + ] { + app.store + .write_object(&bucket, key, &data, "application/octet-stream") + .await + .unwrap(); + + let refused = + serve_spooled_attachment_video(&app, Method::GET, key, ¶ms, &HeaderMap::new()) + .await; + + assert_eq!(StatusCode::PAYLOAD_TOO_LARGE, refused.status(), "{key}"); + } +} + fn fake_asset_object(read_status: Option) -> crate::storage::tests::FakeObject { crate::storage::tests::FakeObject { body: crate::test_fixtures::synthetic_png(512, 512), diff --git a/fluxer_media_proxy/src/server/transform/execution.rs b/fluxer_media_proxy/src/server/transform/execution.rs index f4a0e178f..b4951eced 100644 --- a/fluxer_media_proxy/src/server/transform/execution.rs +++ b/fluxer_media_proxy/src/server/transform/execution.rs @@ -82,9 +82,7 @@ pub(in crate::server) async fn run_video_transform( let deadline = deadline_instant(deadline_ms); let media_limits = runtime.limits(); let transform_metrics = runtime.metrics(); - let cost_bytes = data - .len() - .saturating_add(media_process::AV_NATIVE_COST_BYTES); + let cost_bytes = media_process::av_native_cost(data.len()); let timed = runtime .tasks() .run_native( diff --git a/fluxer_media_proxy/src/server/transform/mod.rs b/fluxer_media_proxy/src/server/transform/mod.rs index e21bf0ee5..19842aae1 100644 --- a/fluxer_media_proxy/src/server/transform/mod.rs +++ b/fluxer_media_proxy/src/server/transform/mod.rs @@ -310,6 +310,73 @@ fn original_bytes_response(request: OriginalBytes<'_>) -> Response { }) } +struct VideoTransformPlan { + format: OutputFormat, + quality: ImageQuality, + cache_key: String, +} + +fn video_transform_plan( + route: TransformRoute, + cache_identity: &str, + transform: &ValidatedTransformParameters, + requested_format: AssetExtension, +) -> VideoTransformPlan { + let format = OutputFormat::coerce_from_extension(requested_format); + let quality = transform.quality.unwrap_or(ImageQuality::High); + VideoTransformPlan { + format, + quality, + cache_key: transform_cache_key(TransformCacheKeyInput { + route, + asset_kind: None, + cache_identity, + width: transform.width, + height: transform.height, + format, + quality: Some(quality), + animated: transform.animated, + effort: None, + resize_mode: None, + }), + } +} + +pub(in crate::server) struct CachedVideoTransformRequest<'a> { + pub(in crate::server) method: Method, + pub(in crate::server) cache_identity: &'a str, + pub(in crate::server) filename: &'a str, + pub(in crate::server) route: TransformRoute, + pub(in crate::server) params: &'a HashMap, + pub(in crate::server) headers: &'a HeaderMap, +} + +pub(in crate::server) fn cached_video_transform_response( + runtime: &TransformRuntime, + request: CachedVideoTransformRequest<'_>, +) -> Option { + let transform = + validate_transform_parameters(request.params, &runtime.limits(), request.route).ok()?; + let plan = video_transform_plan( + request.route, + request.cache_identity, + &transform, + transform.explicit_format?, + ); + let hit = cached_transform_hit(runtime, &plan.cache_key)?; + Some(transformed_response(TransformedBytes { + method: request.method, + data: hit.data.as_bytes().clone(), + content_type: hit.format.mime(), + filename: request.filename, + range_header: request + .headers + .get(header::RANGE) + .and_then(|v| v.to_str().ok()), + requested_download: transform.requested_download, + })) +} + struct VideoTransformRequest<'a> { method: Method, data: Bytes, @@ -352,20 +419,11 @@ async fn serve_video_transform( requested_download: transform.requested_download, }); }; - let format = OutputFormat::coerce_from_extension(requested_format); - let quality = transform.quality.unwrap_or(ImageQuality::High); - let cache_key = transform_cache_key(TransformCacheKeyInput { - route, - asset_kind: None, - cache_identity, - width: transform.width, - height: transform.height, + let VideoTransformPlan { format, - quality: Some(quality), - animated: transform.animated, - effort: None, - resize_mode: None, - }); + quality, + cache_key, + } = video_transform_plan(route, cache_identity, &transform, requested_format); if let Some(hit) = cached_transform_hit(runtime, &cache_key) { return transformed_response(TransformedBytes { method, diff --git a/fluxer_media_proxy/src/spool.rs b/fluxer_media_proxy/src/spool.rs index 576f99f19..5c86af139 100644 --- a/fluxer_media_proxy/src/spool.rs +++ b/fluxer_media_proxy/src/spool.rs @@ -2,6 +2,7 @@ use axum::body::Body; use http_body_util::BodyExt; +use memmap2::Mmap; use std::{io::SeekFrom, path::Path}; use thiserror::Error; use tokio::{ @@ -43,6 +44,12 @@ impl SpooledBody { (file, length) } + pub fn into_mapped(self) -> std::io::Result { + let file = self.file.as_ref().expect("SpooledBody already consumed"); + let map = unsafe { Mmap::map(file)? }; + Ok(MappedSpool { map, _spool: self }) + } + pub fn len(&self) -> u64 { self.length } @@ -52,6 +59,17 @@ impl SpooledBody { } } +pub struct MappedSpool { + map: Mmap, + _spool: SpooledBody, +} + +impl AsRef<[u8]> for MappedSpool { + fn as_ref(&self) -> &[u8] { + &self.map + } +} + impl Drop for SpooledBody { fn drop(&mut self) { if self.budget_reservation > 0 { diff --git a/fluxer_media_proxy/src/storage/local.rs b/fluxer_media_proxy/src/storage/local.rs index 74811109f..b4e8157e0 100644 --- a/fluxer_media_proxy/src/storage/local.rs +++ b/fluxer_media_proxy/src/storage/local.rs @@ -62,7 +62,6 @@ impl Store { &self, bucket: &str, key: &str, - max_bytes: usize, ) -> Result { let path = self.local_path(bucket, key).await?; let metadata = tokio::fs::symlink_metadata(&path) @@ -71,9 +70,6 @@ impl Store { if !metadata.is_file() { return Err(StorageError::NotFound); } - if metadata.len() > max_bytes as u64 { - return Err(StorageError::StreamTooLong); - } Ok(HeadResult { content_length: metadata.len(), content_type: local_content_type(key), @@ -86,9 +82,6 @@ impl Store { ) -> Result { let path = self.local_path(plan.bucket, plan.key).await?; let (mut file, metadata) = open_local_read_file(&path).await?; - if metadata.len() > plan.max_bytes as u64 { - return Err(StorageError::StreamTooLong); - } let total_len = usize::try_from(metadata.len()).map_err(|_| StorageError::StreamTooLong)?; let byte_range = match plan.range { StreamRange::Full => None, diff --git a/fluxer_media_proxy/src/storage/mod.rs b/fluxer_media_proxy/src/storage/mod.rs index 89b547605..e0e9c5281 100644 --- a/fluxer_media_proxy/src/storage/mod.rs +++ b/fluxer_media_proxy/src/storage/mod.rs @@ -80,7 +80,6 @@ enum ContentDigestRequest { struct ObjectStreamPlan<'a> { bucket: &'a str, key: &'a str, - max_bytes: usize, range: StreamRange<'a>, } diff --git a/fluxer_media_proxy/src/storage/object_stream.rs b/fluxer_media_proxy/src/storage/object_stream.rs index 1ce4ea805..1de0f81d8 100644 --- a/fluxer_media_proxy/src/storage/object_stream.rs +++ b/fluxer_media_proxy/src/storage/object_stream.rs @@ -4,23 +4,13 @@ use super::{ HeadResult, ObjectStreamPlan, StorageError, Store, StreamObject, StreamRange, record_storage_outcome, }; -use crate::{config::StorageBackend, constants}; +use crate::config::StorageBackend; impl Store { pub async fn head_object(&self, bucket: &str, key: &str) -> Result { - self.head_object_limited(bucket, key, constants::MAX_MEDIA_PROXY_BYTES) - .await - } - - pub async fn head_object_limited( - &self, - bucket: &str, - key: &str, - max_bytes: usize, - ) -> Result { match self.cfg.storage.backend { - StorageBackend::Local => self.head_local(bucket, key, max_bytes).await, - StorageBackend::S3 => self.head_s3(bucket, key, max_bytes).await, + StorageBackend::Local => self.head_local(bucket, key).await, + StorageBackend::S3 => self.head_s3(bucket, key).await, } } @@ -33,7 +23,6 @@ impl Store { self.stream_object_inner(ObjectStreamPlan { bucket, key, - max_bytes: constants::MAX_MEDIA_PROXY_BYTES, range: match range_header { Some(header) => StreamRange::Header(header), None => StreamRange::Full, diff --git a/fluxer_media_proxy/src/storage/s3.rs b/fluxer_media_proxy/src/storage/s3.rs index 1f36cd44b..08b35938e 100644 --- a/fluxer_media_proxy/src/storage/s3.rs +++ b/fluxer_media_proxy/src/storage/s3.rs @@ -82,7 +82,6 @@ impl Store { &self, bucket: &str, key: &str, - max_bytes: usize, ) -> Result { // A HEAD is a read, so it follows the same endpoint and signing policy as a body read. // Sending it to the write origin instead would bypass the configured read endpoint and @@ -98,9 +97,6 @@ impl Store { return Err(StorageError::S3(s3_error_summary(response).await)); } let content_length = http_headers::parse_content_length(response.headers()).unwrap_or(0); - if content_length > max_bytes as u64 { - return Err(StorageError::StreamTooLong); - } let content_type = response_content_type(response.headers()); Ok(HeadResult { content_length, @@ -155,12 +151,6 @@ impl Store { return Err(StorageError::S3(s3_error_summary(response).await)); } let content_length = http_headers::parse_content_length(response.headers()); - if let Some(content_length) = content_length - && matches!(plan.range, StreamRange::Full) - && content_length > plan.max_bytes as u64 - { - return Err(StorageError::StreamTooLong); - } let content_type = response_content_type(response.headers()); let content_range = header_str(response.headers(), header::CONTENT_RANGE) .and_then(|value| range::parse_content_range(Some(&value))); diff --git a/fluxer_messages/src/router_impl.rs b/fluxer_messages/src/router_impl.rs index 2d0acef86..aab2ada9d 100644 --- a/fluxer_messages/src/router_impl.rs +++ b/fluxer_messages/src/router_impl.rs @@ -51,13 +51,14 @@ impl RouterService for MessagesRouter { can_read_message_history, media_endpoint, include_reactions, + attachment_decay, nonce, tts, include_hidden, threads_mask, .. } => Some(format!( - "api-get:{channel_id}:{message_id}:{viewer_user_id}:{source_guild_id:?}:{message_history_cutoff_ms:?}:{can_read_message_history}:{media_endpoint}:{include_reactions:?}:{nonce:?}:{tts:?}:{include_hidden}:{threads_mask}" + "api-get:{channel_id}:{message_id}:{viewer_user_id}:{source_guild_id:?}:{message_history_cutoff_ms:?}:{can_read_message_history}:{media_endpoint}:{include_reactions:?}:{attachment_decay:?}:{nonce:?}:{tts:?}:{include_hidden}:{threads_mask}" )), MessageRequest::BuildResponse { .. } => None, MessageRequest::BuildResponses { .. } => None, @@ -73,12 +74,13 @@ impl RouterService for MessagesRouter { can_read_message_history, media_endpoint, include_reactions, + attachment_decay, include_hidden, threads_mask, exclude_types, .. } => Some(format!( - "api-list:{channel_id}:{viewer_user_id}:{limit}:{before_id:?}:{after_id:?}:{around_id:?}:{source_guild_id:?}:{message_history_cutoff_ms:?}:{can_read_message_history}:{media_endpoint}:{include_reactions:?}:{include_hidden}:{threads_mask}:{exclude_types:?}" + "api-list:{channel_id}:{viewer_user_id}:{limit}:{before_id:?}:{after_id:?}:{around_id:?}:{source_guild_id:?}:{message_history_cutoff_ms:?}:{can_read_message_history}:{media_endpoint}:{include_reactions:?}:{attachment_decay:?}:{include_hidden}:{threads_mask}:{exclude_types:?}" )), MessageRequest::ExtractMentions { .. } => None, } @@ -104,6 +106,7 @@ mod tests { media_proxy_secret_key: "secret".to_owned(), attachment_url_secret_base64: None, include_reactions: None, + attachment_decay: None, include_hidden: false, threads_mask, exclude_types, @@ -122,6 +125,7 @@ mod tests { media_proxy_secret_key: "secret".to_owned(), attachment_url_secret_base64: None, include_reactions: None, + attachment_decay: None, nonce: None, tts: None, include_hidden: false, @@ -142,4 +146,30 @@ mod tests { MessagesRouter::coalesce_key(&get_request(true)) ); } + + #[test] + fn coalesce_key_separates_attachment_decay_states() { + let mut list = list_request(false, Vec::new()); + if let MessageRequest::ListResponses { + attachment_decay, .. + } = &mut list + { + *attachment_decay = Some(false); + } + let mut get = get_request(false); + if let MessageRequest::GetResponseById { + attachment_decay, .. + } = &mut get + { + *attachment_decay = Some(false); + } + assert_ne!( + MessagesRouter::coalesce_key(&list_request(false, Vec::new())), + MessagesRouter::coalesce_key(&list) + ); + assert_ne!( + MessagesRouter::coalesce_key(&get_request(false)), + MessagesRouter::coalesce_key(&get) + ); + } } diff --git a/fluxer_messages/src/shard_impl.rs b/fluxer_messages/src/shard_impl.rs index 15b670152..1ec52d1e9 100644 --- a/fluxer_messages/src/shard_impl.rs +++ b/fluxer_messages/src/shard_impl.rs @@ -144,6 +144,8 @@ enum MessagesStorage { Deletions(DeletedMessageKeys), #[cfg(test)] Memory(std::sync::Arc>), + #[cfg(test)] + Expiries(std::sync::Arc>>), } #[cfg(test)] @@ -398,9 +400,18 @@ struct ResponseBuildOptions { include_hidden: bool, threads_mask: bool, excluded_types: Vec, + attachment_decay: bool, } impl ResponseBuildOptions { + fn expiring_attachment_ids(&self, messages: &[&Message]) -> HashSet { + if self.attachment_decay { + collect_attachment_ids(messages) + } else { + HashSet::new() + } + } + fn hides(&self, message: &Message) -> bool { self.threads_mask && message.message_type == MESSAGE_TYPE_THREAD_CREATED } @@ -1138,7 +1149,7 @@ impl MessagesShard { .chain(referenced_messages.values()) .collect::>(); let mention_context = build_message_mention_context(&all_messages); - let attachment_ids = collect_attachment_ids(&all_messages); + let attachment_ids = options.expiring_attachment_ids(&all_messages); let channel_ids = collect_channel_mention_ids(&all_messages, &mention_context); let user_ids = collect_user_ids(&all_messages, &mention_context); let reactions_future = self.fetch_reactions_for_messages(messages, options); @@ -1863,7 +1874,7 @@ impl MessagesStorage { storage.get_by_id(channel_id, bucket, message_id).await } #[cfg(test)] - MessagesStorage::Deletions(_) => Ok(None), + MessagesStorage::Deletions(_) | MessagesStorage::Expiries(_) => Ok(None), #[cfg(test)] MessagesStorage::Memory(messages) => Ok(messages .iter() @@ -1894,7 +1905,7 @@ impl MessagesStorage { .await } #[cfg(test)] - MessagesStorage::Deletions(_) => Ok(Vec::new()), + MessagesStorage::Deletions(_) | MessagesStorage::Expiries(_) => Ok(Vec::new()), #[cfg(test)] MessagesStorage::Memory(messages) => Ok(memory_buckets( messages, channel_id, min_bucket, max_bucket, limit, true, @@ -1922,7 +1933,7 @@ impl MessagesStorage { .await } #[cfg(test)] - MessagesStorage::Deletions(_) => Ok(Vec::new()), + MessagesStorage::Deletions(_) | MessagesStorage::Expiries(_) => Ok(Vec::new()), #[cfg(test)] MessagesStorage::Memory(messages) => Ok(memory_buckets( messages, channel_id, min_bucket, max_bucket, limit, false, @@ -1947,7 +1958,7 @@ impl MessagesStorage { storage.fetch_latest_bucket(channel_id, bucket, limit).await } #[cfg(test)] - MessagesStorage::Deletions(_) => Ok(Vec::new()), + MessagesStorage::Deletions(_) | MessagesStorage::Expiries(_) => Ok(Vec::new()), #[cfg(test)] MessagesStorage::Memory(messages) => { Ok(memory_bucket(messages, channel_id, bucket, None, limit)) @@ -1981,7 +1992,7 @@ impl MessagesStorage { .await } #[cfg(test)] - MessagesStorage::Deletions(_) => Ok(Vec::new()), + MessagesStorage::Deletions(_) | MessagesStorage::Expiries(_) => Ok(Vec::new()), #[cfg(test)] MessagesStorage::Memory(messages) => Ok(memory_bucket( messages, @@ -2019,7 +2030,7 @@ impl MessagesStorage { .await } #[cfg(test)] - MessagesStorage::Deletions(_) => Ok(Vec::new()), + MessagesStorage::Deletions(_) | MessagesStorage::Expiries(_) => Ok(Vec::new()), #[cfg(test)] MessagesStorage::Memory(messages) => Ok(memory_bucket( messages, @@ -2054,7 +2065,7 @@ impl MessagesStorage { Ok(()) } #[cfg(test)] - MessagesStorage::Memory(_) => Ok(()), + MessagesStorage::Memory(_) | MessagesStorage::Expiries(_) => Ok(()), } } @@ -2088,7 +2099,9 @@ impl MessagesStorage { .await } #[cfg(test)] - MessagesStorage::Deletions(_) | MessagesStorage::Memory(_) => HashMap::new(), + MessagesStorage::Deletions(_) + | MessagesStorage::Memory(_) + | MessagesStorage::Expiries(_) => HashMap::new(), } } @@ -2106,6 +2119,11 @@ impl MessagesStorage { } #[cfg(test)] MessagesStorage::Deletions(_) | MessagesStorage::Memory(_) => Ok(HashMap::new()), + #[cfg(test)] + MessagesStorage::Expiries(expiries) => Ok(attachment_ids + .into_iter() + .filter_map(|id| expiries.get(&id).map(|expires_at| (id, *expires_at))) + .collect()), } } } @@ -2549,6 +2567,7 @@ impl ShardService for MessagesShard { media_proxy_secret_key, attachment_url_secret_base64, include_reactions, + attachment_decay, nonce, tts, include_hidden, @@ -2576,6 +2595,7 @@ impl ShardService for MessagesShard { attachment_url_secret_base64.as_deref(), ), include_reactions: include_reactions.unwrap_or(true), + attachment_decay: attachment_decay.unwrap_or(true), nonce, tts: tts.unwrap_or(false), include_hidden, @@ -2599,6 +2619,7 @@ impl ShardService for MessagesShard { media_proxy_secret_key, attachment_url_secret_base64, include_reactions, + attachment_decay, nonce, tts, include_hidden, @@ -2623,6 +2644,7 @@ impl ShardService for MessagesShard { attachment_url_secret_base64.as_deref(), ), include_reactions: include_reactions.unwrap_or(true), + attachment_decay: attachment_decay.unwrap_or(true), nonce, tts: tts.unwrap_or(false), include_hidden, @@ -2646,6 +2668,7 @@ impl ShardService for MessagesShard { media_proxy_secret_key, attachment_url_secret_base64, include_reactions, + attachment_decay, include_hidden, threads_mask, } => { @@ -2668,6 +2691,7 @@ impl ShardService for MessagesShard { attachment_url_secret_base64.as_deref(), ), include_reactions: include_reactions.unwrap_or(true), + attachment_decay: attachment_decay.unwrap_or(true), nonce: None, tts: false, include_hidden, @@ -2692,6 +2716,7 @@ impl ShardService for MessagesShard { media_proxy_secret_key, attachment_url_secret_base64, include_reactions, + attachment_decay, include_hidden, threads_mask, mut exclude_types, @@ -2735,6 +2760,7 @@ impl ShardService for MessagesShard { attachment_url_secret_base64.as_deref(), ), include_reactions: include_reactions.unwrap_or(true), + attachment_decay: attachment_decay.unwrap_or(true), nonce: None, tts: false, include_hidden, @@ -4215,6 +4241,7 @@ mod tests { include_hidden: false, threads_mask: false, excluded_types: Vec::new(), + attachment_decay: true, } } @@ -4689,6 +4716,67 @@ mod tests { assert_eq!(Some(true), mapped.expired); } + #[test] + fn attachment_expiry_is_only_looked_up_while_decay_is_on() { + let mut message = orphaned_message(1); + message.attachments = Some(vec![signed_attachment()]); + let off = ResponseBuildOptions { + attachment_decay: false, + ..build_options() + }; + + assert_eq!( + HashSet::from([SIGNED_ATTACHMENT_ID]), + build_options().expiring_attachment_ids(&[&message]) + ); + assert!(off.expiring_attachment_ids(&[&message]).is_empty()); + } + + #[tokio::test] + async fn a_passed_expiry_is_ignored_in_responses_while_decay_is_off() { + let shard = MessagesShard { + storage: MessagesStorage::Expiries(std::sync::Arc::new(HashMap::from([( + SIGNED_ATTACHMENT_ID, + DateTime::::from_timestamp_millis(now_epoch_millis() - 1_000).unwrap(), + )]))), + transport: InMemoryTransport::new(), + }; + let users = stub_user_service(&shard.transport).await; + let mut message = authored_message(1_509_197_195_776_110_592); + message.attachments = Some(vec![signed_attachment()]); + let off = ResponseBuildOptions { + attachment_decay: false, + ..signing_options() + }; + + let while_on = shard + .build_api_response_from_message(message.clone(), signing_options()) + .await + .unwrap() + .expect("the message is visible"); + let while_off = shard + .build_api_response_from_message(message.clone(), off.clone()) + .await + .unwrap() + .expect("the message is visible"); + let listed_while_off = shard + .build_api_responses_from_messages(vec![message], off) + .await + .unwrap(); + users.abort(); + + let expired = &while_on.attachments[0]; + assert_eq!(Some(true), expired.expired); + assert_eq!(None, expired.url); + assert!(expired.expires_at.is_some()); + for response in [&while_off, &listed_while_off[0]] { + let kept = &response.attachments[0]; + assert_eq!(None, kept.expired); + assert_eq!(None, kept.expires_at); + assert!(kept.url.is_some()); + } + } + #[test] fn an_attachment_backed_embed_signs_every_own_url_field() { let deleted = DeletedMessageKeys::default(); diff --git a/fluxer_messages/src/types.rs b/fluxer_messages/src/types.rs index 8a00aa1e7..deada4529 100644 --- a/fluxer_messages/src/types.rs +++ b/fluxer_messages/src/types.rs @@ -83,6 +83,7 @@ pub enum MessageRequest { #[serde(default)] attachment_url_secret_base64: Option, include_reactions: Option, + attachment_decay: Option, nonce: Option, tts: Option, #[serde(default)] @@ -101,6 +102,7 @@ pub enum MessageRequest { #[serde(default)] attachment_url_secret_base64: Option, include_reactions: Option, + attachment_decay: Option, nonce: Option, tts: Option, #[serde(default)] @@ -119,6 +121,7 @@ pub enum MessageRequest { #[serde(default)] attachment_url_secret_base64: Option, include_reactions: Option, + attachment_decay: Option, #[serde(default)] include_hidden: bool, #[serde(default)] @@ -139,6 +142,7 @@ pub enum MessageRequest { #[serde(default)] attachment_url_secret_base64: Option, include_reactions: Option, + attachment_decay: Option, #[serde(default)] include_hidden: bool, #[serde(default)] diff --git a/knip.json b/knip.json index eb07cc578..c90bd4c34 100644 --- a/knip.json +++ b/knip.json @@ -44,7 +44,8 @@ "pino", "undici", "validator" - ] + ], + "ignoreBinaries": ["ffmpeg", "ffprobe"] }, "fluxer_app": { "entry": [ diff --git a/packages/constants/src/LimitConfigMetadata.ts b/packages/constants/src/LimitConfigMetadata.ts index 8b8819e49..fbcc264b9 100644 --- a/packages/constants/src/LimitConfigMetadata.ts +++ b/packages/constants/src/LimitConfigMetadata.ts @@ -1,6 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import {MAX_BIO_LENGTH} from '@fluxer/constants/src/LimitConstants'; +import {BASE64_IMAGE_MAX_SIZE, MAX_BIO_LENGTH} from '@fluxer/constants/src/LimitConstants'; export const LIMIT_KEYS = [ 'avatar_max_size', @@ -320,6 +320,8 @@ export const LIMIT_KEY_METADATA: Record = { scope: 'guild', isToggle: false, unit: 'bytes', + min: 1, + max: BASE64_IMAGE_MAX_SIZE, }, sticker_max_size: { key: 'sticker_max_size', @@ -329,6 +331,8 @@ export const LIMIT_KEY_METADATA: Record = { scope: 'guild', isToggle: false, unit: 'bytes', + min: 1, + max: BASE64_IMAGE_MAX_SIZE, }, max_attachment_file_size: { key: 'max_attachment_file_size', @@ -347,6 +351,8 @@ export const LIMIT_KEY_METADATA: Record = { scope: 'user', isToggle: false, unit: 'bytes', + min: 1, + max: BASE64_IMAGE_MAX_SIZE, }, max_custom_backgrounds: { key: 'max_custom_backgrounds', diff --git a/packages/constants/src/LimitConstants.ts b/packages/constants/src/LimitConstants.ts index 543ffe481..2dd6a66f6 100644 --- a/packages/constants/src/LimitConstants.ts +++ b/packages/constants/src/LimitConstants.ts @@ -78,6 +78,7 @@ export const STICKER_MAX_SIZE = ASSET_FORMAT_POLICY.sticker.maxBytes; export const ATTACHMENT_MAX_SIZE_PREMIUM = 500 * 1024 * 1024; export const ATTACHMENT_MAX_SIZE_NON_PREMIUM = 25 * 1024 * 1024; export const ATTACHMENT_MAX_SIZE_BOT = 50 * 1024 * 1024; +export const BASE64_IMAGE_MAX_SIZE = 100 * 1024 * 1024; export const ATTACHMENT_UPLOAD_CHUNK_THRESHOLD = 10 * 1024 * 1024; const ATTACHMENT_UPLOAD_MIN_CHUNK_SIZE = 10 * 1024 * 1024; const ATTACHMENT_UPLOAD_CHUNK_SIZE_STEP = 1 * 1024 * 1024; diff --git a/packages/schema/src/domains/channel/ChannelRequestSchemas.ts b/packages/schema/src/domains/channel/ChannelRequestSchemas.ts index 73cee0ad9..1d601c192 100644 --- a/packages/schema/src/domains/channel/ChannelRequestSchemas.ts +++ b/packages/schema/src/domains/channel/ChannelRequestSchemas.ts @@ -3,7 +3,6 @@ import {ChannelTypes} from '@fluxer/constants/src/ChannelConstants'; import {CONTENT_WARNING_TEXT_MAX_LENGTH} from '@fluxer/constants/src/GuildConstants'; import { - AVATAR_MAX_SIZE, CHANNEL_RATE_LIMIT_PER_USER_MAX, CHANNEL_RATE_LIMIT_PER_USER_MIN, CHANNEL_TOPIC_MAX_LENGTH, @@ -33,7 +32,7 @@ import { import {ThreadAutoArchiveDurationSchema} from '@fluxer/schema/src/domains/channel/ThreadRequestSchemas'; import {ReadStateResponse} from '@fluxer/schema/src/domains/gateway/GatewaySchemas'; import {ChannelOverwriteTypeSchema, GeneralChannelNameType} from '@fluxer/schema/src/primitives/ChannelValidators'; -import {base64LengthForBytes, createBase64StringType} from '@fluxer/schema/src/primitives/FileValidators'; +import {createBase64ImageType} from '@fluxer/schema/src/primitives/FileValidators'; import {ContentWarningLevelSchema} from '@fluxer/schema/src/primitives/GuildValidators'; import {QueryBooleanType} from '@fluxer/schema/src/primitives/QueryValidators'; import { @@ -134,9 +133,7 @@ const ChannelUpdateCommon = ChannelCommonBase.extend({ 'Legacy: setting true maps to nsfw_override=true; setting false maps to nsfw_override=null (inherit). Prefer nsfw_override.', ), ...ChannelContentWarningFields, - icon: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE)) - .nullish() - .describe('Base64-encoded icon image for group DM channels'), + icon: createBase64ImageType().nullish().describe('Base64-encoded icon image for group DM channels'), owner_id: SnowflakeType.nullish().describe('ID of the new owner for group DM channels'), nicks: ChannelNicknameOverrides.optional().describe('Custom nicknames for users in this channel'), rtc_region: createStringType(RTC_REGION_ID_MIN_LENGTH, RTC_REGION_ID_MAX_LENGTH) @@ -257,9 +254,7 @@ const ChannelUpdateLinkRequest = ChannelUpdateCommon.extend({ const ChannelUpdateGroupDmRequest = z.object({ type: createNamedLiteral(ChannelTypes.GROUP_DM, 'GROUP_DM', 'Channel type (group DM)'), name: GeneralChannelNameType.nullish().describe('The name of the group DM'), - icon: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE)) - .nullish() - .describe('Base64-encoded icon image for the group DM'), + icon: createBase64ImageType().nullish().describe('Base64-encoded icon image for the group DM'), owner_id: SnowflakeType.nullish().describe('ID of the new owner of the group DM'), nicks: ChannelNicknameOverrides.nullish().describe('Custom nicknames for users in this group DM'), nsfw: z.boolean().nullish().describe('Whether the group DM is marked for mature content (owner only)'), diff --git a/packages/schema/src/domains/guild/GuildRequestSchemas.ts b/packages/schema/src/domains/guild/GuildRequestSchemas.ts index c2052858f..20611f91a 100644 --- a/packages/schema/src/domains/guild/GuildRequestSchemas.ts +++ b/packages/schema/src/domains/guild/GuildRequestSchemas.ts @@ -8,19 +8,16 @@ import { SystemChannelFlagsDescriptions, } from '@fluxer/constants/src/GuildConstants'; import { - AVATAR_MAX_SIZE, - EMOJI_MAX_SIZE, MAX_GUILD_ROLES, MAX_GUILD_STICKER_TAGS, MAX_TEMP_BAN_DURATION_SECONDS, MIN_TEMP_BAN_DURATION_SECONDS, - STICKER_MAX_SIZE, } from '@fluxer/constants/src/LimitConstants'; import {SudoVerificationSchema} from '@fluxer/schema/src/domains/auth/AuthSchemas'; import {GuildFeatureSchema} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas'; import {TemplateSerializedGuild} from '@fluxer/schema/src/domains/guild/GuildTemplateSchemas'; import {VanityURLCodeType} from '@fluxer/schema/src/primitives/ChannelValidators'; -import {base64LengthForBytes, createBase64StringType} from '@fluxer/schema/src/primitives/FileValidators'; +import {createBase64ImageType} from '@fluxer/schema/src/primitives/FileValidators'; import { ContentWarningLevelSchema, DefaultMessageNotificationsSchema, @@ -52,9 +49,7 @@ function coerceBlankStringToNull(value: unknown): unknown { export const GuildCreateRequest = z.object({ name: createStringType(1, 100).describe('The name of the guild (1-100 characters)'), - icon: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE)) - .nullish() - .describe('Base64-encoded image data for the guild icon'), + icon: createBase64ImageType().nullish().describe('Base64-encoded image data for the guild icon'), empty_features: z.boolean().optional().describe('Whether to create the guild without default features'), template: TemplateSerializedGuild.optional().describe('Serialised template data to use for guild structure'), }); @@ -64,9 +59,7 @@ export type GuildCreateRequest = z.infer; export const GuildUpdateRequest = z .object({ name: createStringType(1, 100).describe('The name of the guild (1-100 characters)'), - icon: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE)) - .nullish() - .describe('Base64-encoded image data for the guild icon'), + icon: createBase64ImageType().nullish().describe('Base64-encoded image data for the guild icon'), system_channel_id: SnowflakeType.nullish().describe('The ID of the channel where system messages are sent'), system_channel_flags: createBitflagInt32Type( SystemChannelFlags, @@ -108,13 +101,9 @@ export const GuildUpdateRequest = z GuildExplicitContentFilterSchema, 'Level of content filtering for explicit media', ), - banner: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE)) - .nullish() - .describe('Base64-encoded image data for the guild banner'), - splash: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE)) - .nullish() - .describe('Base64-encoded image data for the guild splash screen'), - embed_splash: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE)) + banner: createBase64ImageType().nullish().describe('Base64-encoded image data for the guild banner'), + splash: createBase64ImageType().nullish().describe('Base64-encoded image data for the guild splash screen'), + embed_splash: createBase64ImageType() .nullish() .describe('Base64-encoded image data for the embedded invite splash'), splash_card_alignment: SplashCardAlignmentSchema.optional().describe( @@ -148,12 +137,8 @@ export const GuildMemberUpdateRequest = z.object({ .optional() .transform((ids) => (ids ? new Set(ids) : undefined)) .describe(`Array of role IDs to assign to the member (max ${MAX_GUILD_ROLES})`), - avatar: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE)) - .nullish() - .describe('Base64-encoded image data for the member guild avatar'), - banner: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE)) - .nullish() - .describe('Base64-encoded image data for the member guild banner'), + avatar: createBase64ImageType().nullish().describe('Base64-encoded image data for the member guild avatar'), + banner: createBase64ImageType().nullish().describe('Base64-encoded image data for the member guild banner'), bio: createStringType(1, 320).nullish().describe('The member guild profile bio (1-320 characters)'), pronouns: createStringType(1, 40).nullish().describe('The member guild profile pronouns (1-40 characters)'), accent_color: ColorType.nullish().describe('The accent color for the member guild profile as an integer'), @@ -208,9 +193,7 @@ export const GuildEmojiCreateRequest = z.object({ name: createStringType(2, 32) .refine((value) => /^[a-zA-Z0-9_]+$/.test(value), 'Emoji name can only contain letters, numbers, and underscores') .describe('The name of the emoji (2-32 characters, alphanumeric and underscores only)'), - image: createBase64StringType(1, base64LengthForBytes(EMOJI_MAX_SIZE)).describe( - 'Base64-encoded image data for the emoji', - ), + image: createBase64ImageType().describe('Base64-encoded image data for the emoji'), }); export type GuildEmojiCreateRequest = z.infer; @@ -247,9 +230,7 @@ export const GuildStickerCreateRequest = z.object({ .optional() .default([]) .describe(`Array of autocomplete/suggestion tags (max ${MAX_GUILD_STICKER_TAGS} tags, each 1-30 characters)`), - image: createBase64StringType(1, base64LengthForBytes(STICKER_MAX_SIZE)).describe( - 'Base64-encoded image data for the sticker', - ), + image: createBase64ImageType().describe('Base64-encoded image data for the sticker'), }); export type GuildStickerCreateRequest = z.infer; diff --git a/packages/schema/src/domains/oauth/OAuthSchemas.ts b/packages/schema/src/domains/oauth/OAuthSchemas.ts index 34582274b..cbb25b380 100644 --- a/packages/schema/src/domains/oauth/OAuthSchemas.ts +++ b/packages/schema/src/domains/oauth/OAuthSchemas.ts @@ -1,10 +1,10 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import {ApplicationFlags, BotFlags, BotFlagsDescriptions} from '@fluxer/constants/src/BotConstants'; -import {AVATAR_MAX_SIZE, MAX_APPLICATION_REDIRECT_URIS} from '@fluxer/constants/src/LimitConstants'; +import {MAX_APPLICATION_REDIRECT_URIS} from '@fluxer/constants/src/LimitConstants'; import {PublicUserFlags, PublicUserFlagsDescriptions} from '@fluxer/constants/src/UserConstants'; import {UserPartialResponse} from '@fluxer/schema/src/domains/user/UserResponseSchemas'; -import {base64LengthForBytes, createBase64StringType} from '@fluxer/schema/src/primitives/FileValidators'; +import {createBase64ImageType} from '@fluxer/schema/src/primitives/FileValidators'; import { createBitflagInt32Type, createNamedObject, @@ -414,12 +414,8 @@ export type ApplicationUpdateRequest = z.infer; export const BotProfileUpdateRequest = z.object({ username: UsernameType.optional().describe('The username of the bot'), discriminator: DiscriminatorType.optional().describe('The discriminator of the bot'), - avatar: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE)) - .nullish() - .describe('The avatar image as base64'), - banner: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE)) - .nullish() - .describe('The banner image as base64'), + avatar: createBase64ImageType().nullish().describe('The avatar image as base64'), + banner: createBase64ImageType().nullish().describe('The banner image as base64'), bio: createStringType(0, 1024).nullish().describe('The bio or description of the bot'), bot_flags: createBitflagInt32Type(BotFlags, BotFlagsDescriptions, 'The bot user flags', 'BotFlags').optional(), }); diff --git a/packages/schema/src/domains/user/UserRequestSchemas.ts b/packages/schema/src/domains/user/UserRequestSchemas.ts index dacdbbb1d..f49b66125 100644 --- a/packages/schema/src/domains/user/UserRequestSchemas.ts +++ b/packages/schema/src/domains/user/UserRequestSchemas.ts @@ -1,6 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import {AVATAR_MAX_SIZE, MAX_GROUP_DM_OTHER_RECIPIENTS, MAX_GUILDS_PREMIUM} from '@fluxer/constants/src/LimitConstants'; +import {MAX_GROUP_DM_OTHER_RECIPIENTS, MAX_GUILDS_PREMIUM} from '@fluxer/constants/src/LimitConstants'; import {StatusTypes} from '@fluxer/constants/src/StatusConstants'; import { DEFAULT_GUILD_FOLDER_ICON, @@ -23,7 +23,7 @@ import { SYNCED_PREFERENCES_MAX_ENCODED_LENGTH, } from '@fluxer/schema/src/domains/user/SyncedPreferencesCodec'; import {isValidSingleUnicodeEmoji} from '@fluxer/schema/src/primitives/EmojiValidators'; -import {base64LengthForBytes, createBase64StringType} from '@fluxer/schema/src/primitives/FileValidators'; +import {createBase64ImageType} from '@fluxer/schema/src/primitives/FileValidators'; import {LocaleSchema} from '@fluxer/schema/src/primitives/LocaleSchema'; import {createQueryIntegerType, DateTimeType, QueryBooleanType} from '@fluxer/schema/src/primitives/QueryValidators'; import { @@ -67,12 +67,8 @@ export const UserUpdateRequest = z email: EmailType.describe('The email address for the account'), new_password: PasswordType.describe('The new password to set'), password: PasswordType.describe('The current password for verification'), - avatar: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE)) - .nullish() - .describe('Base64-encoded avatar image'), - banner: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE)) - .nullish() - .describe('Base64-encoded profile banner image'), + avatar: createBase64ImageType().nullish().describe('Base64-encoded avatar image'), + banner: createBase64ImageType().nullish().describe('Base64-encoded profile banner image'), bio: createStringType(1, 320).nullish().describe('User biography text (max 320 characters)'), pronouns: createStringType(1, 40).nullish().describe('User pronouns (max 40 characters)'), accent_color: ColorType.nullish().describe('Profile accent color as integer'), diff --git a/packages/schema/src/domains/webhook/WebhookRequestSchemas.ts b/packages/schema/src/domains/webhook/WebhookRequestSchemas.ts index d2241c968..192c2b34c 100644 --- a/packages/schema/src/domains/webhook/WebhookRequestSchemas.ts +++ b/packages/schema/src/domains/webhook/WebhookRequestSchemas.ts @@ -6,7 +6,7 @@ import { MessageFlags, MessageFlagsDescriptions, } from '@fluxer/constants/src/ChannelConstants'; -import {AVATAR_MAX_SIZE, MAX_MESSAGE_LENGTH_PREMIUM} from '@fluxer/constants/src/LimitConstants'; +import {MAX_MESSAGE_LENGTH_PREMIUM} from '@fluxer/constants/src/LimitConstants'; import { MAX_APPLIED_TAGS_PER_THREAD, THREAD_NAME_MAX_LENGTH, @@ -19,7 +19,7 @@ import { RichEmbedRequest, } from '@fluxer/schema/src/domains/message/MessageRequestSchemas'; import {AllowedMentionsRequest, MessageReferenceRequest} from '@fluxer/schema/src/domains/message/SharedMessageSchemas'; -import {base64LengthForBytes, createBase64StringType} from '@fluxer/schema/src/primitives/FileValidators'; +import {createBase64ImageType} from '@fluxer/schema/src/primitives/FileValidators'; import {QueryBooleanType} from '@fluxer/schema/src/primitives/QueryValidators'; import { coerceNumberFromString, @@ -37,9 +37,7 @@ import {z} from 'zod'; export const WebhookCreateRequest = z.object({ name: WebhookNameType.describe('The name of the webhook'), - avatar: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE)) - .nullish() - .describe('The avatar image as a base64-encoded data URI'), + avatar: createBase64ImageType().nullish().describe('The avatar image as a base64-encoded data URI'), }); export type WebhookCreateRequest = z.infer; @@ -47,9 +45,7 @@ export type WebhookCreateRequest = z.infer; export const WebhookUpdateRequest = z .object({ name: WebhookNameType.describe('The new name of the webhook'), - avatar: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE)) - .nullish() - .describe('The new avatar image as a base64-encoded data URI'), + avatar: createBase64ImageType().nullish().describe('The new avatar image as a base64-encoded data URI'), channel_id: SnowflakeType.describe('The ID of the channel to move the webhook to'), }) .partial(); @@ -59,9 +55,7 @@ export type WebhookUpdateRequest = z.infer; export const WebhookTokenUpdateRequest = z .strictObject({ name: WebhookNameType.describe('The new name of the webhook'), - avatar: createBase64StringType(1, base64LengthForBytes(AVATAR_MAX_SIZE)) - .nullish() - .describe('The new avatar image as a base64-encoded data URI'), + avatar: createBase64ImageType().nullish().describe('The new avatar image as a base64-encoded data URI'), }) .partial(); diff --git a/packages/schema/src/primitives/FileValidators.ts b/packages/schema/src/primitives/FileValidators.ts index 7a48a49ff..a4df6b592 100644 --- a/packages/schema/src/primitives/FileValidators.ts +++ b/packages/schema/src/primitives/FileValidators.ts @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +import {BASE64_IMAGE_MAX_SIZE} from '@fluxer/constants/src/LimitConstants'; import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes'; import { normalizeString, @@ -106,3 +107,7 @@ export function createBase64StringType(minLength = 1, maxLength = 256) { 'Base64ImageType', ); } + +export function createBase64ImageType() { + return createBase64StringType(1, base64LengthForBytes(BASE64_IMAGE_MAX_SIZE)); +} diff --git a/packages/schema/src/primitives/tests/FileValidators.test.ts b/packages/schema/src/primitives/tests/FileValidators.test.ts index eca349da2..cc036a86f 100644 --- a/packages/schema/src/primitives/tests/FileValidators.test.ts +++ b/packages/schema/src/primitives/tests/FileValidators.test.ts @@ -1,9 +1,15 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import {AVATAR_MAX_SIZE, EMOJI_MAX_SIZE, STICKER_MAX_SIZE} from '@fluxer/constants/src/LimitConstants'; +import { + AVATAR_MAX_SIZE, + BASE64_IMAGE_MAX_SIZE, + EMOJI_MAX_SIZE, + STICKER_MAX_SIZE, +} from '@fluxer/constants/src/LimitConstants'; import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes'; import { base64LengthForBytes, + createBase64ImageType, createBase64StringType, FilenameType, isValidBase64, @@ -120,6 +126,7 @@ describe('base64LengthForBytes', () => { [4, 8], [EMOJI_MAX_SIZE, 699052], [AVATAR_MAX_SIZE, 13981016], + [BASE64_IMAGE_MAX_SIZE, 139810136], ])('encodes a ceiling of %i bytes in %i characters', (bytes, expected) => { expect(base64LengthForBytes(bytes)).toBe(expected); }); @@ -138,3 +145,13 @@ describe('base64LengthForBytes', () => { }, ); }); + +describe('createBase64ImageType', () => { + it.each([EMOJI_MAX_SIZE + 1, STICKER_MAX_SIZE * 2, AVATAR_MAX_SIZE + 1])( + 'accepts %i decoded bytes so configured size limits above the stock values can apply', + (bytes) => { + const encoded = Buffer.alloc(bytes, 1).toString('base64'); + expect(createBase64ImageType().parse(encoded)).toBe(encoded); + }, + ); +});