From 522cf08e6152bfd2dee8015a1f8288786633efa1 Mon Sep 17 00:00:00 2001 From: Hampus Date: Fri, 18 Sep 2026 15:57:32 +0200 Subject: [PATCH] feat(media-proxy): sign attachment URLs and gate origins (#2830) --- Cargo.lock | 2 + config/env/development.env | 1 + deploy/self-hosting/.env.example | 30 + deploy/self-hosting/docker-compose.yml | 4 + .../pkgs/media_proxy_utils/package.json | 4 + .../src/AttachmentUrlSignature.ts | 211 ++ fluxer_api/src/api/Config.ts | 3 + fluxer_api/src/api/app/ControllerRegistry.ts | 2 + .../api/attachment/AttachmentController.ts | 38 + .../src/api/attachment/AttachmentUrls.ts | 76 + .../message/AttachmentProcessingService.ts | 71 +- .../message/MessageResponseDataService.ts | 4 + .../services/message/UploadSegmentSignal.ts | 209 ++ fluxer_api/src/api/config/APIConfig.ts | 3 + .../api/favorite_gif/FavoriteGifResolver.ts | 14 +- .../api/favorite_meme/FavoriteMemeModel.ts | 4 +- .../api/favorite_meme/FavoriteMemeService.ts | 38 +- .../src/api/infrastructure/EmbedService.ts | 49 +- .../src/api/infrastructure/IMediaService.ts | 6 + .../src/api/infrastructure/MediaService.ts | 48 +- fluxer_api/src/api/openapi/openapi.json | 96 +- .../ChannelRateLimitConfig.ts | 4 + fluxer_api/src/api/test/Setup.ts | 1 + fluxer_api/src/api/test/TestMediaService.ts | 5 + ...ositoryBackedMessageResponseDataService.ts | 31 +- fluxer_api/src/api/utils/UnfurlerUtils.ts | 8 +- fluxer_api/src/api/webhook/WebhookService.ts | 6 +- .../src/api/worker/tasks/HarvestGuildData.ts | 40 +- .../src/api/worker/tasks/HarvestUserData.ts | 4 +- .../src/features/app/constants/Endpoints.ts | 1 + .../components/AutocompleteMemePreview.tsx | 8 +- .../components/MessageAttachmentStateUtils.ts | 17 +- .../channel/components/MobileMemesPicker.tsx | 6 +- .../components/embeds/ChannelEmbed.tsx | 26 +- .../attachments/TextualAttachmentPreview.tsx | 62 +- .../TextualAttachmentPreviewFetch.ts | 84 + .../components/embeds/media/EmbedGifv.tsx | 3 + .../embeds/media/VoiceMessagePlayer.tsx | 3 + .../pickers/gif/FavoriteGifTypes.ts | 27 +- .../components/pickers/gif/GifPickerGrid.tsx | 3 + .../pickers/gif/GifPickerGridData.ts | 24 +- .../components/pickers/memes/MemeGridItem.tsx | 11 +- .../features/expressions/state/FavoriteGif.ts | 18 +- .../gateway/transport/GatewayConnection.ts | 2 + .../markdown/renderers/LinkRenderer.tsx | 7 +- .../components/modals/MediaViewerModal.tsx | 15 +- .../hooks/useAttachmentRefreshOnError.ts | 14 + .../messaging/hooks/useMediaLoading.ts | 6 + .../messaging/hooks/useNearViewport.ts | 23 +- .../messaging/state/AttachmentUrlRefresher.ts | 242 +++ .../messaging/utils/AttachmentCdnUrl.ts | 198 ++ .../messaging/utils/FileDownloadUtils.ts | 46 +- .../features/messaging/utils/SpoilerUtils.ts | 3 +- .../ui/action_menu/items/MediaMenuData.tsx | 16 +- .../src/features/ui/utils/NativeUtils.ts | 10 +- .../media_player/components/AudioPlayer.tsx | 3 + .../components/InlineAudioPlayer.tsx | 4 + fluxer_common/Cargo.toml | 1 + fluxer_common/src/attachment_url_signature.rs | 1127 ++++++++++ fluxer_common/src/lib.rs | 1 + .../attachment_url_signature_vectors.json | 793 +++++++ fluxer_docs/scripts/VerifyDocsCoverage.ts | 20 + .../src/content/docs/http-api/memes.mdx | 2 +- .../src/content/docs/http-api/messages.mdx | 118 +- .../docs/http-api/users/data-harvest.mdx | 2 +- .../src/content/docs/media-proxy/overview.md | 113 +- .../docs/media-proxy/responses-and-limits.md | 30 +- .../src/content/docs/media-proxy/routes.mdx | 26 +- .../content/docs/media-proxy/upload-relay.mdx | 2 +- .../content/docs/operator/configuration.mdx | 36 +- .../content/docs/operator/reverse-proxy.mdx | 76 + fluxer_media_proxy/src/cli.rs | 85 +- fluxer_media_proxy/src/config/mod.rs | 110 +- fluxer_media_proxy/src/config/parse.rs | 111 +- .../src/config/tests/attachment_signature.rs | 233 ++ fluxer_media_proxy/src/config/tests/cors.rs | 258 +++ fluxer_media_proxy/src/config/tests/mod.rs | 63 +- fluxer_media_proxy/src/http_headers.rs | 103 +- .../src/metrics/attachment_signature.rs | 40 + fluxer_media_proxy/src/metrics/mod.rs | 8 + fluxer_media_proxy/src/metrics/rendering.rs | 15 +- fluxer_media_proxy/src/metrics/request.rs | 5 +- fluxer_media_proxy/src/metrics/tests.rs | 98 +- fluxer_media_proxy/src/mime/mod.rs | 5 +- fluxer_media_proxy/src/mime/registry.rs | 35 + fluxer_media_proxy/src/mime/tests.rs | 131 +- fluxer_media_proxy/src/request_log/mod.rs | 187 +- .../src/server/attachment_signature.rs | 432 ++++ fluxer_media_proxy/src/server/cors.rs | 337 +++ fluxer_media_proxy/src/server/middleware.rs | 384 +++- fluxer_media_proxy/src/server/mod.rs | 2 + fluxer_media_proxy/src/server/relay/tests.rs | 95 +- .../src/server/routes/dispatch.rs | 1880 ++++++++++++++++- .../src/server/routes/internal.rs | 428 +++- fluxer_media_proxy/src/server/routes/ops.rs | 7 +- fluxer_media_proxy/src/server/routes/relay.rs | 3 +- fluxer_media_proxy/src/server/runtime.rs | 329 ++- fluxer_media_proxy/src/server/self_origin.rs | 58 +- fluxer_media_proxy/src/server/state.rs | 4 + fluxer_media_proxy/src/server/stored/mod.rs | 18 + fluxer_media_proxy/src/storage/tests/mod.rs | 12 +- fluxer_messages/Cargo.toml | 1 + fluxer_messages/src/shard_impl.rs | 803 ++++++- fluxer_messages/src/types.rs | 8 + packages/config/src/ConfigLoader.ts | 26 + packages/config/src/MasterConfig.ts | 3 + .../src/config_loader/EnvironmentOverrides.ts | 4 + .../src/domains/message/AttachmentSchemas.ts | 27 + 108 files changed, 10148 insertions(+), 441 deletions(-) create mode 100644 fluxer_api/pkgs/media_proxy_utils/src/AttachmentUrlSignature.ts create mode 100644 fluxer_api/src/api/attachment/AttachmentController.ts create mode 100644 fluxer_api/src/api/attachment/AttachmentUrls.ts create mode 100644 fluxer_api/src/api/channel/services/message/UploadSegmentSignal.ts create mode 100644 fluxer_app/src/features/channel/components/embeds/attachments/TextualAttachmentPreviewFetch.ts create mode 100644 fluxer_app/src/features/messaging/hooks/useAttachmentRefreshOnError.ts create mode 100644 fluxer_app/src/features/messaging/state/AttachmentUrlRefresher.ts create mode 100644 fluxer_app/src/features/messaging/utils/AttachmentCdnUrl.ts create mode 100644 fluxer_common/src/attachment_url_signature.rs create mode 100644 fluxer_common/src/testdata/attachment_url_signature_vectors.json create mode 100644 fluxer_media_proxy/src/config/tests/attachment_signature.rs create mode 100644 fluxer_media_proxy/src/config/tests/cors.rs create mode 100644 fluxer_media_proxy/src/metrics/attachment_signature.rs create mode 100644 fluxer_media_proxy/src/server/attachment_signature.rs create mode 100644 fluxer_media_proxy/src/server/cors.rs diff --git a/Cargo.lock b/Cargo.lock index a4716d8d2..61e9cfd04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1870,6 +1870,7 @@ name = "fluxer-messages" version = "0.1.0" dependencies = [ "anyhow", + "base64 0.23.1", "chrono", "criterion", "fluxer-svc", @@ -2038,6 +2039,7 @@ dependencies = [ "aws-sigv4", "axum", "base64 0.23.1", + "hex", "hmac 0.13.0", "maxminddb", "moka", diff --git a/config/env/development.env b/config/env/development.env index a06df9451..c67d3b22e 100644 --- a/config/env/development.env +++ b/config/env/development.env @@ -130,6 +130,7 @@ PUBLIC_RELEASE_CHANNEL=canary PUBLIC_BOOTSTRAP_API_ENDPOINT=/api PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT=http://localhost:8088/api FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64=Zmx1eGVyLWRldi11cGxvYWQtcmVsYXktc2VjcmV0LTAwMDA= +FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64=Zmx1eGVyLWRldi1hdHRhY2htZW50LXVybC1zZWNyZXQ= AWS_EC2_METADATA_DISABLED=true AWS_ACCESS_KEY_ID=fluxer AWS_SECRET_ACCESS_KEY=fluxer-secret diff --git a/deploy/self-hosting/.env.example b/deploy/self-hosting/.env.example index 62a868726..a87b89529 100644 --- a/deploy/self-hosting/.env.example +++ b/deploy/self-hosting/.env.example @@ -198,6 +198,36 @@ FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME #FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS=https://chat.example.com #FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS=http://chat.example.com:19080 +# Two optional media policies, both off unless you turn them on. Nothing below is +# needed for a working instance, and an upgrade never adds any of it. +# +# The first limits which web origins may read media through CORS. A request with +# no Origin header is always served, so direct links, image tags and native +# clients keep working. The allowlist defaults to the public origin above, which +# is where this instance serves its web app. The hosted web client and the +# desktop app run on https://web.fluxer.app, so add that origin, as in the second +# allowlist line, if people use them with this instance. +# +# The second makes an attachment read need a signed URL, which stops a copied +# link working forever elsewhere. Turning it on takes two settings: a secret, and +# the mode. Generate the secret with openssl rand -base64 32. It is a comma +# separated list, the first entry signs and every entry verifies. To rotate, add +# the new secret second and run docker compose up -d, then move it first and run +# again. Remove the old secret no sooner than a day after that, because an +# ordinary URL it signed stays valid for up to a day. A data package URL inside a +# harvest export never expires, so removing a secret ends every data package URL +# it signed and those exports have to be rebuilt. +# +# Each mode is off, report or enforce on its own. Set a mode to report first to +# log what enforce would refuse while refusing nothing. media-proxy reads these +# at container start, so apply a change with docker compose up -d media-proxy. +# docker compose restart media-proxy keeps the old environment. +#FLUXER_MEDIA_PROXY_CORS_MODE=enforce +#FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS=https://chat.example.com +#FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS=https://chat.example.com,https://web.fluxer.app +#FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64= +#FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE=enforce + # Extra Content-Security-Policy sources, appended to the built-in ones. Set these # only when a browser must reach an origin the defaults do not cover, such as a # voice server hosted on a domain other than FLUXER_DOMAIN. Separate several diff --git a/deploy/self-hosting/docker-compose.yml b/deploy/self-hosting/docker-compose.yml index 4ae273908..b52046b31 100644 --- a/deploy/self-hosting/docker-compose.yml +++ b/deploy/self-hosting/docker-compose.yml @@ -97,6 +97,7 @@ x-fluxer-env: &fluxer-env FLUXER_GATEWAY_RPC_AUTH_TOKEN: ${FLUXER_GATEWAY_RPC_AUTH_TOKEN:?set FLUXER_GATEWAY_RPC_AUTH_TOKEN in .env} FLUXER_MEDIA_PROXY_SECRET_KEY: ${FLUXER_MEDIA_PROXY_SECRET_KEY:?set FLUXER_MEDIA_PROXY_SECRET_KEY in .env} FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64:?set FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 in .env} + FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64: ${FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64:-} FLUXER_ADMIN_SECRET_KEY_BASE: ${FLUXER_ADMIN_SECRET_KEY_BASE:?set FLUXER_ADMIN_SECRET_KEY_BASE in .env} FLUXER_ADMIN_OAUTH_CLIENT_SECRET: ${FLUXER_ADMIN_OAUTH_CLIENT_SECRET:?set FLUXER_ADMIN_OAUTH_CLIENT_SECRET in .env} @@ -473,6 +474,9 @@ services: FLUXER_MEDIA_PROXY_MODE: upload FLUXER_MEDIA_PROXY_STORAGE_BACKEND: s3 FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media + FLUXER_MEDIA_PROXY_CORS_MODE: ${FLUXER_MEDIA_PROXY_CORS_MODE:-off} + FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS: ${FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}:${FLUXER_PUBLIC_PORT:-443}}} + FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE: ${FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE:-off} FLUXER_S3_READ_SIGNED: "true" depends_on: seaweedfs-init: {condition: service_completed_successfully} diff --git a/fluxer_api/pkgs/media_proxy_utils/package.json b/fluxer_api/pkgs/media_proxy_utils/package.json index df03a0d56..022329509 100644 --- a/fluxer_api/pkgs/media_proxy_utils/package.json +++ b/fluxer_api/pkgs/media_proxy_utils/package.json @@ -24,6 +24,10 @@ "import": "./src/MediaProxySigner.ts", "types": "./src/MediaProxySigner.ts" }, + "./src/AttachmentUrlSignature": { + "import": "./src/AttachmentUrlSignature.ts", + "types": "./src/AttachmentUrlSignature.ts" + }, "./*": "./*" }, "main": "./src/MediaProxyUtils.ts", diff --git a/fluxer_api/pkgs/media_proxy_utils/src/AttachmentUrlSignature.ts b/fluxer_api/pkgs/media_proxy_utils/src/AttachmentUrlSignature.ts new file mode 100644 index 000000000..3c3c7e74d --- /dev/null +++ b/fluxer_api/pkgs/media_proxy_utils/src/AttachmentUrlSignature.ts @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import crypto from 'node:crypto'; + +export const ATTACHMENT_URL_TTL_SECS = 86_400; +export const ATTACHMENT_URL_BUCKET_SECS = 43_200; + +export const ORDINARY_USAGE = ''; +export const DATA_PACKAGE_USAGE = 'dp'; + +export type AttachmentUrlUsage = typeof ORDINARY_USAGE | typeof DATA_PACKAGE_USAGE; +export type SignatureParameterName = 'ex' | 'is' | 'hm' | 'uc'; + +const SIGNATURE_DOMAIN = 'fluxer-attachment-url-v1'; +const ATTACHMENT_PATH_PREFIX = '/attachments/'; +const SIGNATURE_PARAMETER_NAMES: ReadonlyArray = ['ex', 'is', 'hm', 'uc']; +const DATA_PACKAGE_EXPIRES = '0'; +const WINDOW_HEX_LENGTH = 8; +const MAX_WINDOW_SECS = 0xff_ff_ff_ff; +const LEADING_SLASHES_REGEX = /^\/+/u; +const TRAILING_SLASHES_REGEX = /\/+$/u; + +const textEncoder = new TextEncoder(); +const strictTextDecoder = new TextDecoder('utf-8', {fatal: true, ignoreBOM: true}); + +export interface AttachmentUrlWindow { + issued: number; + expires: number; +} + +export interface SignAttachmentUrlOptions { + mediaEndpoint: string; + secret: Uint8Array; + nowSecs: number; + anchorSecs: number; +} + +function hexNibble(byte: number): number { + if (byte >= 0x30 && byte <= 0x39) return byte - 0x30; + if (byte >= 0x41 && byte <= 0x46) return byte - 0x41 + 10; + if (byte >= 0x61 && byte <= 0x66) return byte - 0x61 + 10; + return -1; +} + +function percentDecodeBytes(value: string, plusAsSpace: boolean): Uint8Array { + const bytes = textEncoder.encode(value); + const decoded = new Uint8Array(bytes.length); + let length = 0; + let index = 0; + while (index < bytes.length) { + const byte = bytes[index] as number; + if (byte === 0x25 && index + 2 < bytes.length) { + const high = hexNibble(bytes[index + 1] as number); + const low = hexNibble(bytes[index + 2] as number); + if (high >= 0 && low >= 0) { + decoded[length] = (high << 4) | low; + length += 1; + index += 3; + continue; + } + } + decoded[length] = plusAsSpace && byte === 0x2b ? 0x20 : byte; + length += 1; + index += 1; + } + return decoded.subarray(0, length); +} + +export function percentDecodeStorageKey(path: string): string | null { + try { + return strictTextDecoder.decode(percentDecodeBytes(path.replace(LEADING_SLASHES_REGEX, ''), false)); + } catch { + return null; + } +} + +export function signatureParameterName(name: string): SignatureParameterName | null { + const decoded = percentDecodeBytes(name, true); + if (decoded.length !== 2) return null; + const candidate = String.fromCharCode(decoded[0] as number, decoded[1] as number); + return SIGNATURE_PARAMETER_NAMES.find((entry) => entry === candidate) ?? null; +} + +export function isSignatureParameterName(name: string): boolean { + return signatureParameterName(name) !== null; +} + +function isSafeStorageKey(key: string): boolean { + if (key.length === 0 || key.startsWith('/')) return false; + return key + .split('/') + .every((component) => component.length > 0 && component !== '.' && component !== '..' && !component.includes('\0')); +} + +function firstIndexOf(value: string, characters: ReadonlyArray): number { + let found = -1; + for (const character of characters) { + const index = value.indexOf(character); + if (index >= 0 && (found < 0 || index < found)) { + found = index; + } + } + return found; +} + +function rawPathFromUrl(url: string): string | null { + const schemeIndex = url.indexOf('://'); + if (schemeIndex < 0) return null; + const afterAuthority = url.slice(schemeIndex + 3); + const boundary = firstIndexOf(afterAuthority, ['/', '?', '#']); + if (boundary < 0 || afterAuthority[boundary] !== '/') return ''; + const path = afterAuthority.slice(boundary); + const queryIndex = firstIndexOf(path, ['?', '#']); + return queryIndex < 0 ? path : path.slice(0, queryIndex); +} + +function parseWebUrl(value: string): URL | null { + try { + const parsed = new URL(value); + return parsed.protocol === 'http:' || parsed.protocol === 'https:' ? parsed : null; + } catch { + return null; + } +} + +export function attachmentStorageKeyFromUrl(url: string, mediaEndpoint: string): string | null { + const target = parseWebUrl(url); + const endpoint = parseWebUrl(mediaEndpoint); + if (!target || !endpoint || target.origin !== endpoint.origin) return null; + const path = rawPathFromUrl(url); + if (path === null) return null; + const endpointPath = (rawPathFromUrl(mediaEndpoint) ?? '').replace(TRAILING_SLASHES_REGEX, ''); + if (!path.startsWith(`${endpointPath}${ATTACHMENT_PATH_PREFIX}`)) return null; + const storageKey = percentDecodeStorageKey(path.slice(endpointPath.length)); + if (storageKey === null || !isSafeStorageKey(storageKey)) return null; + return storageKey; +} + +interface SplitUrl { + base: string; + query: string; + fragment: string; +} + +function splitUrl(url: string): SplitUrl { + const fragmentIndex = url.indexOf('#'); + const head = fragmentIndex < 0 ? url : url.slice(0, fragmentIndex); + const fragment = fragmentIndex < 0 ? '' : url.slice(fragmentIndex); + const queryIndex = head.indexOf('?'); + if (queryIndex < 0) return {base: head, query: '', fragment}; + return {base: head.slice(0, queryIndex), query: head.slice(queryIndex + 1), fragment}; +} + +function preservedFields(query: string): Array { + if (query.length === 0) return []; + return query.split('&').filter((field) => { + if (field.length === 0 || field === '=') return false; + const separator = field.indexOf('='); + return !isSignatureParameterName(separator < 0 ? field : field.slice(0, separator)); + }); +} + +export function stripAttachmentSignature(url: string): string { + const {base, query, fragment} = splitUrl(url); + const preserved = preservedFields(query); + if (preserved.length === 0) return `${base}${fragment}`; + return `${base}?${preserved.join('&')}${fragment}`; +} + +export function issueWindow(anchorSecs: number, nowSecs: number): AttachmentUrlWindow { + const elapsed = Math.max(0, nowSecs - anchorSecs); + const issued = anchorSecs + Math.floor(elapsed / ATTACHMENT_URL_BUCKET_SECS) * ATTACHMENT_URL_BUCKET_SECS; + return {issued, expires: issued + ATTACHMENT_URL_TTL_SECS}; +} + +export function canonicalInput(storageKey: string, exHex: string, isHex: string, usage: AttachmentUrlUsage): string { + return `${SIGNATURE_DOMAIN}\n${exHex}\n${isHex}\n${usage}\n${storageKey}`; +} + +function windowHex(value: number): string { + return value.toString(16).padStart(WINDOW_HEX_LENGTH, '0'); +} + +function signUsage(url: string, options: SignAttachmentUrlOptions, usage: AttachmentUrlUsage): string { + const storageKey = attachmentStorageKeyFromUrl(url, options.mediaEndpoint); + if (storageKey === null) return url; + const {issued, expires} = issueWindow(options.anchorSecs, options.nowSecs); + if (!Number.isSafeInteger(issued) || issued < 0 || expires > MAX_WINDOW_SECS) return url; + const isDataPackage = usage === DATA_PACKAGE_USAGE; + const exHex = windowHex(isDataPackage ? 0 : expires); + const isHex = windowHex(issued); + const signature = crypto + .createHmac('sha256', options.secret) + .update(canonicalInput(storageKey, exHex, isHex, usage)) + .digest('hex'); + const signatureFields = isDataPackage + ? `ex=${DATA_PACKAGE_EXPIRES}&is=${isHex}&hm=${signature}&uc=${DATA_PACKAGE_USAGE}` + : `ex=${exHex}&is=${isHex}&hm=${signature}`; + const {base, query, fragment} = splitUrl(url); + const preserved = preservedFields(query); + const fields = preserved.length === 0 ? signatureFields : `${signatureFields}&${preserved.join('&')}`; + return `${base}?${fields}${fragment}`; +} + +export function signAttachmentUrl(url: string, options: SignAttachmentUrlOptions): string { + return signUsage(url, options, ORDINARY_USAGE); +} + +export function signDataPackageAttachmentUrl(url: string, options: SignAttachmentUrlOptions): string { + return signUsage(url, options, DATA_PACKAGE_USAGE); +} diff --git a/fluxer_api/src/api/Config.ts b/fluxer_api/src/api/Config.ts index d252f5aa7..2694cc1d2 100644 --- a/fluxer_api/src/api/Config.ts +++ b/fluxer_api/src/api/Config.ts @@ -263,6 +263,9 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig { tokenTtlSecs: uploadRelayConfig.token_ttl_secs, keepDirectCountries: uploadRelayConfig.keep_direct_countries, }, + attachmentUrls: { + secretsBase64: master.services.media_proxy.attachment_urls.secrets_base64, + }, }, geoip: geoipSourceConfig, proxy: { diff --git a/fluxer_api/src/api/app/ControllerRegistry.ts b/fluxer_api/src/api/app/ControllerRegistry.ts index 868a841b3..51d466750 100644 --- a/fluxer_api/src/api/app/ControllerRegistry.ts +++ b/fluxer_api/src/api/app/ControllerRegistry.ts @@ -1,6 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import {registerAdminControllers} from '@app/api/admin/controllers/index'; +import {AttachmentController} from '@app/api/attachment/AttachmentController'; import {AuthController} from '@app/api/auth/AuthController'; import {BlueskyOAuthController} from '@app/api/bluesky/BlueskyOAuthController'; import {Config} from '@app/api/Config'; @@ -45,6 +46,7 @@ export function registerControllers(routes: HonoApp, config: APIConfig): void { GeolocationController(routes); registerAdminControllers(routes); AuthController(routes); + AttachmentController(routes); ChannelController(routes); ConnectionController(routes); BlueskyOAuthController(routes); diff --git a/fluxer_api/src/api/attachment/AttachmentController.ts b/fluxer_api/src/api/attachment/AttachmentController.ts new file mode 100644 index 000000000..b51374207 --- /dev/null +++ b/fluxer_api/src/api/attachment/AttachmentController.ts @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {signAttachmentUrl} from '@app/api/attachment/AttachmentUrls'; +import {LoginRequired} from '@app/api/middleware/AuthMiddleware'; +import {RateLimitMiddleware} from '@app/api/middleware/RateLimitMiddleware'; +import {OpenAPI} from '@app/api/middleware/ResponseTypeMiddleware'; +import {RateLimitConfigs} from '@app/api/RateLimitConfig'; +import type {HonoApp} from '@app/api/types/HonoEnv'; +import {Validator} from '@app/api/Validator'; +import { + RefreshAttachmentUrlsRequest, + RefreshAttachmentUrlsResponse, +} from '@fluxer/schema/src/domains/message/AttachmentSchemas'; + +export function AttachmentController(app: HonoApp) { + app.post( + '/attachments/refresh-urls', + RateLimitMiddleware(RateLimitConfigs.ATTACHMENT_URLS_REFRESH), + LoginRequired, + Validator('json', RefreshAttachmentUrlsRequest), + OpenAPI({ + operationId: 'refresh_attachment_urls', + summary: 'Refresh attachment URLs', + responseSchema: RefreshAttachmentUrlsResponse, + statusCode: 200, + security: ['botToken', 'sessionToken'], + tags: ['Messages'], + description: + 'Reissues the expiring signature on attachment URLs. Returns one entry per requested URL, in the order they were requested, each pairing the URL exactly as it was sent with a freshly signed copy. A URL that is not an attachment URL of this instance is returned unchanged. No membership or existence check is performed.', + }), + async (ctx) => { + const urls = ctx.req.valid('json').attachment_urls; + return ctx.json({ + refreshed_urls: urls.map((original) => ({original, refreshed: signAttachmentUrl(original)})), + }); + }, + ); +} diff --git a/fluxer_api/src/api/attachment/AttachmentUrls.ts b/fluxer_api/src/api/attachment/AttachmentUrls.ts new file mode 100644 index 000000000..ae0c5ec65 --- /dev/null +++ b/fluxer_api/src/api/attachment/AttachmentUrls.ts @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {AttachmentID, ChannelID} from '@app/api/BrandedTypes'; +import {Config} from '@app/api/Config'; +import {makeAttachmentCdnUrl} from '@app/api/channel/services/message/MessageHelpers'; +import {extractTimestampBigInt} from '@fluxer/snowflake/src/SnowflakeUtils'; +import { + attachmentStorageKeyFromUrl, + type SignAttachmentUrlOptions, + signDataPackageAttachmentUrl as signDataPackageWithSecret, + signAttachmentUrl as signWithSecret, + stripAttachmentSignature as stripSignature, +} from '@pkgs/media_proxy_utils/src/AttachmentUrlSignature'; + +const SNOWFLAKE_SEGMENT_REGEX = /^[0-9]{1,20}$/u; +const STORAGE_KEY_MIN_SEGMENTS = 4; + +function signingSecret(): Buffer | null { + const configured = Config.mediaProxy.attachmentUrls.secretsBase64[0]; + if (!configured) return null; + const secret = Buffer.from(configured, 'base64'); + return secret.length === 0 ? null : secret; +} + +function anchorSecsFromStorageKey(storageKey: string): number | null { + const segments = storageKey.split('/'); + if (segments.length < STORAGE_KEY_MIN_SEGMENTS || segments[0] !== 'attachments') return null; + const attachmentId = segments[2] as string; + if (!SNOWFLAKE_SEGMENT_REGEX.test(segments[1] as string) || !SNOWFLAKE_SEGMENT_REGEX.test(attachmentId)) return null; + return Math.floor(extractTimestampBigInt(BigInt(attachmentId)) / 1000); +} + +function signingOptions(url: string, nowSecs?: number): SignAttachmentUrlOptions | null { + const secret = signingSecret(); + if (secret === null) return null; + const mediaEndpoint = Config.endpoints.media; + const storageKey = attachmentStorageKeyFromUrl(url, mediaEndpoint); + if (storageKey === null) return null; + const anchorSecs = anchorSecsFromStorageKey(storageKey); + if (anchorSecs === null) return null; + return {mediaEndpoint, secret, nowSecs: nowSecs ?? Math.floor(Date.now() / 1000), anchorSecs}; +} + +export function signAttachmentUrl(url: string, nowSecs?: number): string { + const options = signingOptions(url, nowSecs); + return options === null ? url : signWithSecret(url, options); +} + +export function signDataPackageAttachmentUrl(url: string, nowSecs?: number): string { + const options = signingOptions(url, nowSecs); + return options === null ? url : signDataPackageWithSecret(url, options); +} + +export function stripAttachmentSignature(url: string): string { + return stripSignature(url); +} + +export function stripOwnAttachmentSignature(url: string): string { + return attachmentStorageKeyFromUrl(url, Config.endpoints.media) === null ? url : stripSignature(url); +} + +export function makeSignedAttachmentCdnUrl( + channelId: ChannelID, + attachmentId: AttachmentID | bigint, + filename: string, +): string { + return signAttachmentUrl(makeAttachmentCdnUrl(channelId, attachmentId, filename)); +} + +export function makeDataPackageAttachmentCdnUrl( + channelId: ChannelID, + attachmentId: AttachmentID | bigint, + filename: string, +): string { + return signDataPackageAttachmentUrl(makeAttachmentCdnUrl(channelId, attachmentId, filename)); +} diff --git a/fluxer_api/src/api/channel/services/message/AttachmentProcessingService.ts b/fluxer_api/src/api/channel/services/message/AttachmentProcessingService.ts index 420664544..88b5bcfbe 100644 --- a/fluxer_api/src/api/channel/services/message/AttachmentProcessingService.ts +++ b/fluxer_api/src/api/channel/services/message/AttachmentProcessingService.ts @@ -1,7 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import fs from 'node:fs'; -import {createAttachmentID, type UserID} from '@app/api/BrandedTypes'; +import {createAttachmentID, createGuildID, createUserID, type UserID} from '@app/api/BrandedTypes'; import {Config} from '@app/api/Config'; import type {AttachmentToProcess} from '@app/api/channel/AttachmentDTOs'; import type {AttachmentUploadTraceRepository} from '@app/api/channel/repositories/message/AttachmentUploadTraceRepository'; @@ -11,6 +11,7 @@ import { makeAttachmentCdnKey, validateAttachmentIds, } from '@app/api/channel/services/message/MessageHelpers'; +import {scheduleUploadSegmentSignal} from '@app/api/channel/services/message/UploadSegmentSignal'; import type {MessageAttachment} from '@app/api/database/types/MessageTypes'; import {contentModerationService, type ModerationContext} from '@app/api/infrastructure/ContentModerationService'; import type { @@ -65,6 +66,7 @@ interface ProcessedAttachment { hasVirusDetected: boolean; applyFinalObjectMetadata: boolean; sourceLocalPath: string | null; + sniffedContentType: string | null; } export class AttachmentProcessingService { @@ -172,6 +174,24 @@ export class AttachmentProcessingService { } return result.attachment; }); + scheduleUploadSegmentSignal({ + userId: params.uploadUserId, + guildId: params.guild ? createGuildID(BigInt(params.guild.id)) : null, + guildOwnerId: params.guild ? createUserID(BigInt(params.guild.owner_id)) : null, + channelId: params.message.channelId, + messageId: params.message.id, + attachments: processedAttachments.map((attachment, index) => ({ + attachmentId: attachment.attachment_id, + uploadKey: results[index].copyOperation.sourceKey, + filename: attachment.filename, + contentType: attachment.content_type, + size: attachment.size, + duration: attachment.duration ?? null, + waveform: attachment.waveform ?? null, + sniffedContentType: results[index].sniffedContentType, + requestIp: bindingResults[index].bound?.request_ip ?? null, + })), + }); return {attachments: processedAttachments, hasVirusDetected: false}; } @@ -213,7 +233,31 @@ export class AttachmentProcessingService { let applyFinalObjectMetadata = false; const clientDuration: number | null = attachment.duration ?? null; const waveform: string | null = attachment.waveform ?? null; - const isMedia = isMediaFile(contentType); + const sniffedContentType = isMediaFile(contentType) + ? null + : await this.sniffAttachmentMediaType({ + index, + uploadFilename: attachment.upload_filename, + filename: attachment.filename, + }); + if (sniffedContentType !== null) { + Logger.warn( + { + surface: 'message_attachment', + userId: params.uploadUserId.toString(), + guildId: params.guild?.id ?? null, + channelId: message.channelId.toString(), + messageId: message.id.toString(), + attachmentId: attachmentId.toString(), + uploadKey: attachment.upload_filename, + filename: attachment.filename, + filenameContentType: contentType, + sniffedContentType, + }, + 'content_moderation.attachment_type_mismatch', + ); + } + const isMedia = isMediaFile(contentType) || sniffedContentType !== null; let metadata: MediaProxyMetadataResponse | null = null; if (isMedia) { metadata = await this.getAttachmentMediaMetadata({ @@ -303,6 +347,7 @@ export class AttachmentProcessingService { hasVirusDetected, applyFinalObjectMetadata, sourceLocalPath: null, + sniffedContentType, }; } const isAudio = contentType.startsWith('audio/'); @@ -341,6 +386,7 @@ export class AttachmentProcessingService { hasVirusDetected, applyFinalObjectMetadata, sourceLocalPath: retainedLocalPath, + sniffedContentType, }; } catch (error) { if (sourceLocalPath) { @@ -350,6 +396,27 @@ export class AttachmentProcessingService { } } + private async sniffAttachmentMediaType(params: { + index: number; + uploadFilename: string; + filename: string; + }): Promise { + const sniff = await this.mediaService.sniffUpload(params.uploadFilename); + if (sniff) { + return sniff.content_type; + } + Logger.warn( + { + context: METADATA_PROBE_DEGRADED_CONTEXT, + attachmentIndex: params.index, + uploadFilename: params.uploadFilename, + filename: params.filename, + }, + 'Attachment content sniff unavailable, storing attachment with its filename type', + ); + return null; + } + private async getAttachmentMediaMetadata(params: { index: number; uploadFilename: string; diff --git a/fluxer_api/src/api/channel/services/message/MessageResponseDataService.ts b/fluxer_api/src/api/channel/services/message/MessageResponseDataService.ts index e0fd644d9..21bad3d5e 100644 --- a/fluxer_api/src/api/channel/services/message/MessageResponseDataService.ts +++ b/fluxer_api/src/api/channel/services/message/MessageResponseDataService.ts @@ -107,6 +107,7 @@ export class MessageResponseDataService { can_read_message_history: params.access.canReadMessageHistory, media_endpoint: Config.endpoints.media, media_proxy_secret_key: Config.mediaProxy.secretKey, + attachment_url_secret_base64: Config.mediaProxy.attachmentUrls.secretsBase64[0], include_reactions: true, }); if (typeof response === 'object' && 'FoundApiMany' in response) { @@ -147,6 +148,7 @@ export class MessageResponseDataService { can_read_message_history: params.access.canReadMessageHistory, media_endpoint: Config.endpoints.media, media_proxy_secret_key: Config.mediaProxy.secretKey, + attachment_url_secret_base64: Config.mediaProxy.attachmentUrls.secretsBase64[0], include_reactions: true, nonce: params.nonce, tts: params.tts, @@ -177,6 +179,7 @@ export class MessageResponseDataService { can_read_message_history: params.access.canReadMessageHistory, media_endpoint: Config.endpoints.media, media_proxy_secret_key: Config.mediaProxy.secretKey, + attachment_url_secret_base64: Config.mediaProxy.attachmentUrls.secretsBase64[0], include_reactions: params.includeReactions ?? true, nonce: params.nonce, tts: params.tts, @@ -244,6 +247,7 @@ export class MessageResponseDataService { can_read_message_history: params.access.canReadMessageHistory, media_endpoint: Config.endpoints.media, media_proxy_secret_key: Config.mediaProxy.secretKey, + attachment_url_secret_base64: Config.mediaProxy.attachmentUrls.secretsBase64[0], include_reactions: params.includeReactions ?? true, }); if (typeof response !== 'object' || !('FoundApiMany' in response)) { diff --git a/fluxer_api/src/api/channel/services/message/UploadSegmentSignal.ts b/fluxer_api/src/api/channel/services/message/UploadSegmentSignal.ts new file mode 100644 index 000000000..b5b3b0bb4 --- /dev/null +++ b/fluxer_api/src/api/channel/services/message/UploadSegmentSignal.ts @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {AttachmentID, ChannelID, GuildID, MessageID, UserID} from '@app/api/BrandedTypes'; +import {Logger} from '@app/api/Logger'; +import {getKVClient} from '@app/api/middleware/ServiceRegistry'; +import {snowflakeToDate} from '@fluxer/snowflake/src/Snowflake'; + +const WINDOW_MS = 600000; +const USER_THRESHOLD = 60; +const GUILD_THRESHOLD = 120; +const FRESH_GUILD_THRESHOLD = 20; +const FRESH_GUILD_MAX_AGE_MS = 604800000; +const SEGMENT_MAX_BYTES = 16 * 1024 * 1024; +const SEGMENT_MAX_DURATION_SECONDS = 30; + +const KEY_PREFIX = 'abuse:upload_segment:'; +const WINDOW_TTL_SECONDS = WINDOW_MS / 1000; +const MAX_IN_FLIGHT_SIGNALS = 32; +const SURFACE = 'message_attachment'; +const SIGNAL_MESSAGE = 'content_moderation.upload_segment_pattern'; +const FAILURE_MESSAGE = 'content_moderation.upload_segment_signal_failed'; +const DROPPED_MESSAGE = 'content_moderation.upload_segment_signal_dropped'; +const PLAYLIST_CONTENT_TYPES = new Set(['application/vnd.apple.mpegurl', 'application/x-mpegurl']); + +export type UploadSegmentScope = 'user' | 'guild' | 'fresh_guild'; + +export interface UploadSegmentAttachment { + attachmentId: AttachmentID; + uploadKey: string; + filename: string; + contentType: string; + size: bigint; + duration: number | null; + waveform: string | null; + sniffedContentType: string | null; + requestIp: string | null; +} + +export interface UploadSegmentSignalInput { + userId: UserID; + guildId: GuildID | null; + guildOwnerId: UserID | null; + channelId: ChannelID; + messageId: MessageID; + attachments: ReadonlyArray; +} + +interface CountedScope { + scope: UploadSegmentScope; + key: string; + threshold: number; +} + +function baseContentType(contentType: string): string { + const [base] = contentType.split(';'); + return (base ?? '').trim().toLowerCase(); +} + +export function isSegmentShapedAttachment(attachment: UploadSegmentAttachment): boolean { + if (attachment.waveform !== null) { + return false; + } + if (attachment.sniffedContentType !== null) { + return true; + } + const contentType = baseContentType(attachment.contentType); + const isSegmentType = + contentType.startsWith('video/') || contentType.startsWith('audio/') || PLAYLIST_CONTENT_TYPES.has(contentType); + if (!isSegmentType) { + return false; + } + if (attachment.size > BigInt(SEGMENT_MAX_BYTES)) { + return false; + } + return attachment.duration === null || attachment.duration <= SEGMENT_MAX_DURATION_SECONDS; +} + +export function isRungValue(count: number, threshold: number): boolean { + if (count < threshold || count % threshold !== 0) { + return false; + } + const multiple = count / threshold; + return (multiple & (multiple - 1)) === 0; +} + +function uploaderOwnsGuild(input: UploadSegmentSignalInput): boolean { + return input.guildOwnerId !== null && input.guildOwnerId === input.userId; +} + +function resolveScopes(input: UploadSegmentSignalInput, windowIndex: number, nowMs: number): Array { + const scopes: Array = [ + { + scope: 'user', + key: `${KEY_PREFIX}user:${input.userId.toString()}:${windowIndex}`, + threshold: USER_THRESHOLD, + }, + ]; + if (input.guildId === null) { + return scopes; + } + const guildAgeMs = nowMs - snowflakeToDate(input.guildId).getTime(); + const isFreshGuild = uploaderOwnsGuild(input) && guildAgeMs < FRESH_GUILD_MAX_AGE_MS; + scopes.push({ + scope: isFreshGuild ? 'fresh_guild' : 'guild', + key: `${KEY_PREFIX}guild:${input.guildId.toString()}:${windowIndex}`, + threshold: isFreshGuild ? FRESH_GUILD_THRESHOLD : GUILD_THRESHOLD, + }); + return scopes; +} + +function buildSignalFields(params: { + input: UploadSegmentSignalInput; + segments: ReadonlyArray; + windowIndex: number; + scope: CountedScope; + count: number; +}): Record { + const {input, segments, windowIndex, scope, count} = params; + const requestIps = new Set(); + for (const segment of segments) { + if (segment.requestIp !== null) { + requestIps.add(segment.requestIp); + } + } + return { + surface: SURFACE, + scope: scope.scope, + count, + threshold: scope.threshold, + windowMs: WINDOW_MS, + windowStartedAt: new Date(windowIndex * WINDOW_MS).toISOString(), + userId: input.userId.toString(), + userCreatedAt: snowflakeToDate(input.userId).toISOString(), + guildId: input.guildId === null ? null : input.guildId.toString(), + guildCreatedAt: input.guildId === null ? null : snowflakeToDate(input.guildId).toISOString(), + guildOwnerId: input.guildOwnerId === null ? null : input.guildOwnerId.toString(), + uploaderOwnsGuild: uploaderOwnsGuild(input), + channelId: input.channelId.toString(), + messageId: input.messageId.toString(), + attachmentIds: segments.map((segment) => segment.attachmentId.toString()), + uploadKeys: segments.map((segment) => segment.uploadKey), + requestIps: Array.from(requestIps), + filenames: segments.map((segment) => segment.filename), + contentTypes: segments.map((segment) => segment.contentType), + disguisedCount: segments.filter((segment) => segment.sniffedContentType !== null).length, + }; +} + +function scopeFields(input: UploadSegmentSignalInput): Record { + return { + surface: SURFACE, + userId: input.userId.toString(), + guildId: input.guildId === null ? null : input.guildId.toString(), + channelId: input.channelId.toString(), + messageId: input.messageId.toString(), + }; +} + +const inFlightSignals = new Set>(); + +export function scheduleUploadSegmentSignal(input: UploadSegmentSignalInput): void { + if (!input.attachments.some(isSegmentShapedAttachment)) { + return; + } + if (inFlightSignals.size >= MAX_IN_FLIGHT_SIGNALS) { + Logger.warn(scopeFields(input), DROPPED_MESSAGE); + return; + } + const pending = recordUploadSegmentSignal(input).finally(() => { + inFlightSignals.delete(pending); + }); + inFlightSignals.add(pending); +} + +export async function flushUploadSegmentSignals(): Promise { + while (inFlightSignals.size > 0) { + await Promise.all([...inFlightSignals]); + } +} + +export async function recordUploadSegmentSignal(input: UploadSegmentSignalInput): Promise { + try { + const segments = input.attachments.filter(isSegmentShapedAttachment); + if (segments.length === 0) { + return; + } + const nowMs = Date.now(); + const windowIndex = Math.floor(nowMs / WINDOW_MS); + const scopes = resolveScopes(input, windowIndex, nowMs); + const kv = getKVClient(); + const counts: Array = []; + for (const scope of scopes) { + const count = await kv.incr(scope.key); + counts.push(count); + if (count === 1) { + await kv.expire(scope.key, WINDOW_TTL_SECONDS); + } + } + for (const [index, scope] of scopes.entries()) { + const count = counts[index]; + if (!isRungValue(count, scope.threshold)) { + continue; + } + Logger.warn(buildSignalFields({input, segments, windowIndex, scope, count}), SIGNAL_MESSAGE); + } + } catch (error) { + Logger.warn({error, ...scopeFields(input)}, FAILURE_MESSAGE); + } +} diff --git a/fluxer_api/src/api/config/APIConfig.ts b/fluxer_api/src/api/config/APIConfig.ts index 0f4019278..26b68a04e 100644 --- a/fluxer_api/src/api/config/APIConfig.ts +++ b/fluxer_api/src/api/config/APIConfig.ts @@ -118,6 +118,9 @@ export interface APIConfig { tokenTtlSecs: number; keepDirectCountries: Array; }; + attachmentUrls: { + secretsBase64: Array; + }; }; geoip: APIGeoipConfig; proxy: { diff --git a/fluxer_api/src/api/favorite_gif/FavoriteGifResolver.ts b/fluxer_api/src/api/favorite_gif/FavoriteGifResolver.ts index e74c06611..d08aa3233 100644 --- a/fluxer_api/src/api/favorite_gif/FavoriteGifResolver.ts +++ b/fluxer_api/src/api/favorite_gif/FavoriteGifResolver.ts @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +import {signAttachmentUrl} from '@app/api/attachment/AttachmentUrls'; import {tryExtractGifProviderSlug} from '@app/api/gif/GifProviderUtils'; import type {GifService} from '@app/api/gif/GifService'; import type {IGifProvider} from '@app/api/gif/IGifProvider'; @@ -108,10 +109,11 @@ function favoriteGifEntryFromExternalMedia({ mediaService: IMediaService; metadata: MediaProxyMetadataResponse | null; }): ResolvedGifEntrySchema { - const proxyUrl = mediaService.getExternalMediaProxyURL(url); - const media = directMediaFormatFromMetadata({url, proxyUrl, metadata}); + const signedUrl = signAttachmentUrl(url); + const proxyUrl = signAttachmentUrl(mediaService.getExternalMediaProxyURL(url)); + const media = directMediaFormatFromMetadata({url: signedUrl, proxyUrl, metadata}); return { - url, + url: signedUrl, proxy_url: proxyUrl, width: metadata?.width ?? 0, height: metadata?.height ?? 0, @@ -153,16 +155,16 @@ function favoriteGifEntryFromEmbedMedia({ mediaService: IMediaService; media: EmbedMediaResponse; }): ResolvedGifEntrySchema { - const proxyUrl = media.proxy_url ?? mediaService.getExternalMediaProxyURL(media.url); + const proxyUrl = signAttachmentUrl(media.proxy_url ?? mediaService.getExternalMediaProxyURL(media.url)); const width = media.width ?? 0; const height = media.height ?? 0; const contentType = media.content_type ?? ''; return { - url, + url: signAttachmentUrl(url), proxy_url: proxyUrl, width, height, - media: directMediaFormatFromDetails({url: media.url, proxyUrl, contentType, width, height}), + media: directMediaFormatFromDetails({url: signAttachmentUrl(media.url), proxyUrl, contentType, width, height}), content_type: contentType, placeholder: media.placeholder ?? null, }; diff --git a/fluxer_api/src/api/favorite_meme/FavoriteMemeModel.ts b/fluxer_api/src/api/favorite_meme/FavoriteMemeModel.ts index 306efdf4f..237c9485f 100644 --- a/fluxer_api/src/api/favorite_meme/FavoriteMemeModel.ts +++ b/fluxer_api/src/api/favorite_meme/FavoriteMemeModel.ts @@ -1,13 +1,13 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +import {makeSignedAttachmentCdnUrl} from '@app/api/attachment/AttachmentUrls'; import {userIdToChannelId} from '@app/api/BrandedTypes'; -import {makeAttachmentCdnUrl} from '@app/api/channel/services/message/MessageHelpers'; import type {FavoriteMeme} from '@app/api/models/FavoriteMeme'; import {assertSafeByteSize} from '@app/api/utils/ByteSizeUtils'; import type {FavoriteMemeResponse} from '@fluxer/schema/src/domains/meme/MemeSchemas'; export function mapFavoriteMemeToResponse(meme: FavoriteMeme): FavoriteMemeResponse { - const url = makeAttachmentCdnUrl(userIdToChannelId(meme.userId), meme.attachmentId, meme.filename); + const url = makeSignedAttachmentCdnUrl(userIdToChannelId(meme.userId), meme.attachmentId, meme.filename); return { id: meme.id.toString(), user_id: meme.userId.toString(), diff --git a/fluxer_api/src/api/favorite_meme/FavoriteMemeService.ts b/fluxer_api/src/api/favorite_meme/FavoriteMemeService.ts index ce646688a..0c989f454 100644 --- a/fluxer_api/src/api/favorite_meme/FavoriteMemeService.ts +++ b/fluxer_api/src/api/favorite_meme/FavoriteMemeService.ts @@ -1,11 +1,12 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import type {ApiContext} from '@app/api/ApiContext'; +import {makeSignedAttachmentCdnUrl} from '@app/api/attachment/AttachmentUrls'; import type {ChannelID, MemeID, MessageID, UserID} from '@app/api/BrandedTypes'; import {createAttachmentID, createMemeID, userIdToChannelId} from '@app/api/BrandedTypes'; import {Config} from '@app/api/Config'; import type {ChannelService} from '@app/api/channel/services/ChannelService'; -import {makeAttachmentCdnKey, makeAttachmentCdnUrl} from '@app/api/channel/services/message/MessageHelpers'; +import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers'; import {mapFavoriteMemeToResponse} from '@app/api/favorite_meme/FavoriteMemeModel'; import type {IFavoriteMemeRepository} from '@app/api/favorite_meme/IFavoriteMemeRepository'; import { @@ -35,6 +36,7 @@ import {MediaMetadataError} from '@fluxer/errors/src/domains/core/MediaMetadataE import {UnknownFavoriteMemeError} from '@fluxer/errors/src/domains/core/UnknownFavoriteMemeError'; import type {GifMediaFormat} from '@fluxer/schema/src/domains/gif/GifSchemas'; import {normalizeFilename} from '@fluxer/schema/src/primitives/FileValidators'; +import {attachmentStorageKeyFromUrl} from '@pkgs/media_proxy_utils/src/AttachmentUrlSignature'; import mime from 'mime'; type MessageAttachmentCandidate = Message['attachments'][number]; @@ -78,6 +80,11 @@ function isAnimatedEmbedMedia(contentType: string | null | undefined, flags: num return ((flags ?? 0) & EmbedMediaFlags.IS_ANIMATED) !== 0; } +function isAttachmentKeyInChannel(storageKey: string, channelId: ChannelID): boolean { + const segments = storageKey.split('/'); + return segments.length === 4 && segments[0] === 'attachments' && segments[1] === channelId.toString(); +} + function resolveFavoriteMemeAnimationFlag( media: Pick, metadata: Pick | null | undefined, @@ -315,7 +322,7 @@ export class FavoriteMemeService { this.ensureFavoriteMemeTagLimit(user, urlTags); const metadata = await this.apiContext.services.media.getMetadata({ type: 'external', - url, + url: url, with_base64: true, nsfw: 'allow', }); @@ -544,7 +551,7 @@ export class FavoriteMemeService { embedCount: embeds.length, }); } - return this.mediaFromEmbed(embeds[preferredEmbedIndex], `embed_${preferredEmbedIndex}`); + return this.mediaFromEmbed(embeds[preferredEmbedIndex], `embed_${preferredEmbedIndex}`, message.channelId); } if (attachments.length > 0) { let attachment: MessageAttachmentCandidate | undefined; @@ -566,7 +573,7 @@ export class FavoriteMemeService { } } for (const embed of embeds) { - const media = await this.mediaFromEmbed(embed, 'media'); + const media = await this.mediaFromEmbed(embed, 'media', message.channelId); if (media) return media; } return null; @@ -587,7 +594,7 @@ export class FavoriteMemeService { const isGifv = isAnimatedAttachment(attachment.contentType, attachment.flags); return { isExternal: false, - url: makeAttachmentCdnUrl(message.channelId, attachment.id, attachment.filename), + url: makeSignedAttachmentCdnUrl(message.channelId, attachment.id, attachment.filename), sourceKey: makeAttachmentCdnKey(message.channelId, attachment.id, attachment.filename), filename: attachment.filename, contentType: attachment.contentType, @@ -607,6 +614,7 @@ export class FavoriteMemeService { private async mediaFromEmbed( embed: MessageEmbedCandidate, fallbackFilename: string, + channelId: ChannelID, ): Promise { const media = embed.image || embed.video || embed.thumbnail; if (!media?.url) { @@ -617,13 +625,14 @@ export class FavoriteMemeService { if (!this.isValidMediaType(contentType)) { return null; } - const isExternal = !this.isInternalCDNUrl(media.url); + const candidateKey = attachmentStorageKeyFromUrl(media.url, Config.endpoints.media); + const sourceKey = candidateKey !== null && isAttachmentKeyInChannel(candidateKey, channelId) ? candidateKey : null; const isGifv = embed.type === 'gifv' || isAnimatedEmbedMedia(media.contentType, media.flags); const detectedGif = embed.type === 'gifv' ? await this.detectGifFromUrl(media.url) : null; return { - isExternal, + isExternal: sourceKey === null, url: media.url, - sourceKey: isExternal ? '' : this.extractStorageKeyFromUrl(media.url) || '', + sourceKey: sourceKey ?? '', filename, contentType, size: BigInt(0), @@ -639,10 +648,6 @@ export class FavoriteMemeService { }; } - private isInternalCDNUrl(url: string): boolean { - return url.startsWith(`${Config.endpoints.media}/`); - } - private isValidMediaType(contentType: string): boolean { return contentType.startsWith('image/') || contentType.startsWith('video/') || contentType.startsWith('audio/'); } @@ -661,15 +666,6 @@ export class FavoriteMemeService { } } - private extractStorageKeyFromUrl(url: string): string | null { - try { - const urlObj = new URL(url); - return urlObj.pathname.substring(1); - } catch { - return null; - } - } - private ensureFavoriteMemeTagLimit(user: User, tags?: Array): void { const limit = this.resolveUserLimit(user, 'max_favorite_meme_tags', MAX_FAVORITE_MEME_TAGS); if ((tags?.length ?? 0) > limit) { diff --git a/fluxer_api/src/api/infrastructure/EmbedService.ts b/fluxer_api/src/api/infrastructure/EmbedService.ts index 5c26c3dae..f02f9f205 100644 --- a/fluxer_api/src/api/infrastructure/EmbedService.ts +++ b/fluxer_api/src/api/infrastructure/EmbedService.ts @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +import {stripOwnAttachmentSignature} from '@app/api/attachment/AttachmentUrls'; import type {ChannelID, MessageID} from '@app/api/BrandedTypes'; import type {RichEmbedMediaWithMetadata} from '@app/api/channel/EmbedTypes'; import type {IChannelRepository} from '@app/api/channel/IChannelRepository'; @@ -53,6 +54,15 @@ interface InitialUrlEmbedResult { hasUncachedUrls: boolean; } +type RichEmbedRequestWithMedia = RichEmbedRequest & { + image?: RichEmbedMediaWithMetadata | null; + thumbnail?: RichEmbedMediaWithMetadata | null; +}; + +function canonicalUrl(url: string | null | undefined): string | null { + return url == null ? null : stripOwnAttachmentSignature(url); +} + export class EmbedService { private readonly MAX_EMBED_CHARACTERS = 6000; private readonly MAX_EMBED_CHARACTERS_BUG_HUNTER = 12000; @@ -228,20 +238,20 @@ export class EmbedService { type: embed.type ?? null, title: embed.title ?? null, description: embed.description ?? null, - url: embed.url ?? null, + url: canonicalUrl(embed.url), timestamp: embed.timestamp ? new Date(embed.timestamp) : null, color: embed.color ?? null, author: embed.author ? { name: embed.author.name ?? null, - url: embed.author.url ?? null, - icon_url: embed.author.icon_url ?? null, + url: canonicalUrl(embed.author.url), + icon_url: canonicalUrl(embed.author.icon_url), } : null, provider: embed.provider ? { name: embed.provider.name ?? null, - url: embed.provider.url ?? null, + url: canonicalUrl(embed.provider.url), } : null, thumbnail: this.mapResponseMedia(embed.thumbnail), @@ -251,7 +261,7 @@ export class EmbedService { footer: embed.footer ? { text: embed.footer.text ?? null, - icon_url: embed.footer.icon_url ?? null, + icon_url: canonicalUrl(embed.footer.icon_url), } : null, fields: @@ -272,7 +282,7 @@ export class EmbedService { private mapResponseMedia(media?: MessageEmbedResponse['image']): MessageEmbed['image'] { if (!media) return null; return { - url: media.url, + url: stripOwnAttachmentSignature(media.url), content_type: media.content_type ?? null, content_hash: media.content_hash ?? null, width: media.width ?? null, @@ -303,13 +313,7 @@ export class EmbedService { } } - private async createEmbed( - embed: RichEmbedRequest & { - image?: RichEmbedMediaWithMetadata | null; - thumbnail?: RichEmbedMediaWithMetadata | null; - }, - nsfwMode: MediaProxyNsfwMode, - ): Promise { + private async createEmbed(embed: RichEmbedRequestWithMedia, nsfwMode: MediaProxyNsfwMode): Promise { const [author, footer, imageResult, thumbnailResult] = await Promise.all([ this.processAuthor(embed.author ?? undefined, nsfwMode), this.processFooter(embed.footer ?? undefined, nsfwMode), @@ -326,7 +330,7 @@ export class EmbedService { type: 'rich', title: embed.title ?? null, description: embed.description ?? null, - url: embed.url ?? null, + url: canonicalUrl(embed.url), timestamp: embed.timestamp ?? null, color: embed.color ?? 0, footer: footer?.toMessageEmbedFooter() ?? null, @@ -363,7 +367,7 @@ export class EmbedService { if (attachmentMetadata) { return { media: new EmbedMedia({ - url: request.url, + url: stripOwnAttachmentSignature(request.url), width: attachmentMetadata.width, height: attachmentMetadata.height, description: request.description ?? null, @@ -376,7 +380,7 @@ export class EmbedService { nsfw: attachmentMetadata.nsfw ?? false, }; } - const {url, metadata} = await this.resolveExternalMedia(request.url, nsfwMode); + const {url, metadata} = await this.resolveExternalMedia(stripOwnAttachmentSignature(request.url), nsfwMode); if (!metadata) { return { media: new EmbedMedia({ @@ -418,13 +422,14 @@ export class EmbedService { url: string; metadata: MediaProxyMetadataResponse | null; }> { + const stored = stripOwnAttachmentSignature(url); const directMetadata = await this.mediaService.getMetadata({ type: 'external', url, ...mediaProxyMetadataPolicy(nsfwMode), }); if (this.isRenderableMediaType(directMetadata?.content_type)) { - return {url, metadata: directMetadata}; + return {url: stored, metadata: directMetadata}; } const unfurled = await this.unfurlerService.unfurl(url, nsfwMode); for (const embed of unfurled) { @@ -437,11 +442,11 @@ export class EmbedService { ...mediaProxyMetadataPolicy(nsfwMode), }); if (this.isRenderableMediaType(candidateMetadata?.content_type)) { - return {url: candidate, metadata: candidateMetadata}; + return {url: stripOwnAttachmentSignature(candidate), metadata: candidateMetadata}; } } } - return {url, metadata: directMetadata}; + return {url: stored, metadata: directMetadata}; } private isRenderableMediaType(contentType: string | null | undefined): boolean { @@ -461,11 +466,11 @@ export class EmbedService { url: author.icon_url, ...mediaProxyMetadataPolicy(nsfwMode), }); - if (metadata) iconUrl = author.icon_url; + if (metadata) iconUrl = stripOwnAttachmentSignature(author.icon_url); } return new EmbedAuthor({ name: author.name, - url: author.url ?? null, + url: canonicalUrl(author.url), icon_url: iconUrl, }); } @@ -482,7 +487,7 @@ export class EmbedService { url: footer.icon_url, ...mediaProxyMetadataPolicy(nsfwMode), }); - if (metadata) iconUrl = footer.icon_url; + if (metadata) iconUrl = stripOwnAttachmentSignature(footer.icon_url); } return new EmbedFooter({ text: footer.text, diff --git a/fluxer_api/src/api/infrastructure/IMediaService.ts b/fluxer_api/src/api/infrastructure/IMediaService.ts index f611158ed..cf40d66fb 100644 --- a/fluxer_api/src/api/infrastructure/IMediaService.ts +++ b/fluxer_api/src/api/infrastructure/IMediaService.ts @@ -57,6 +57,10 @@ export interface MediaProxyMetadataResponse { nsfw_probability?: number; } +export interface MediaProxySniffResponse { + content_type: string | null; +} + export type MediaProxyFrameRequest = | { type: 'upload'; @@ -85,5 +89,7 @@ export abstract class IMediaService { abstract getThumbnail(uploadFilename: string): Promise; + abstract sniffUpload(uploadFilename: string): Promise; + abstract extractFrames(request: MediaProxyFrameRequest): Promise; } diff --git a/fluxer_api/src/api/infrastructure/MediaService.ts b/fluxer_api/src/api/infrastructure/MediaService.ts index 1a779a113..51682a312 100644 --- a/fluxer_api/src/api/infrastructure/MediaService.ts +++ b/fluxer_api/src/api/infrastructure/MediaService.ts @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +import {signAttachmentUrl} from '@app/api/attachment/AttachmentUrls'; import {Config} from '@app/api/Config'; import { IMediaService, @@ -9,11 +10,13 @@ import { type MediaProxyMetadataRequest, type MediaProxyMetadataResponse, type MediaProxyNsfwMode, + type MediaProxySniffResponse, } from '@app/api/infrastructure/IMediaService'; import {Logger} from '@app/api/Logger'; import * as FetchUtils from '@app/api/utils/FetchUtils'; import {isJsonRecord, parseJsonWithGuard} from '@app/api/utils/JsonBoundaryUtils'; import {ExplicitContentCannotBeSentError} from '@fluxer/errors/src/domains/moderation/ExplicitContentCannotBeSentError'; +import {attachmentStorageKeyFromUrl} from '@pkgs/media_proxy_utils/src/AttachmentUrlSignature'; import * as MediaProxyUtils from '@pkgs/media_proxy_utils/src/MediaProxyUtils'; import {ms} from 'itty-time'; @@ -30,7 +33,13 @@ const MEDIA_PROXY_METADATA_WITH_BASE64_MAX_BYTES = 64 * 1024 * 1024; const MEDIA_PROXY_ERROR_MAX_BYTES = 16 * 1024; const MEDIA_PROXY_THUMBNAIL_MAX_BYTES = 8 * 1024 * 1024; const MEDIA_PROXY_FRAMES_MAX_BYTES = 512 * 1024; +const MEDIA_PROXY_SNIFF_MAX_BYTES = 1024; const MEDIA_PROXY_REQUEST_TIMEOUT_MS = ms('30 seconds'); +const MEDIA_PROXY_SNIFF_TIMEOUT_MS = ms('2 seconds'); + +function isMediaProxySniffResponse(value: unknown): value is MediaProxySniffResponse { + return isJsonRecord(value) && (value.content_type === null || typeof value.content_type === 'string'); +} function isMediaProxyMetadataResponse(value: unknown): value is MediaProxyMetadataResponse { if (!isJsonRecord(value)) return false; @@ -135,6 +144,9 @@ export class MediaService extends IMediaService { } getExternalMediaProxyURL(url: string): string { + if (attachmentStorageKeyFromUrl(url, Config.endpoints.media) !== null) { + return signAttachmentUrl(url); + } let urlObj: URL; try { urlObj = new URL(url); @@ -166,6 +178,34 @@ export class MediaService extends IMediaService { } } + async sniffUpload(uploadFilename: string): Promise { + const response = await this.makeRequest( + '/_sniff', + { + type: 'upload', + upload_filename: uploadFilename, + }, + MEDIA_PROXY_SNIFF_TIMEOUT_MS, + ); + if (!response) return null; + try { + const responseText = await FetchUtils.streamToStringWithLimit(response.body, { + maxBytes: MEDIA_PROXY_SNIFF_MAX_BYTES, + headers: response.headers, + description: 'Media proxy sniff response', + }); + const sniff = parseJsonWithGuard(responseText, isMediaProxySniffResponse); + if (!sniff) { + Logger.error({uploadFilename}, 'Media proxy returned invalid sniff response'); + return null; + } + return {content_type: sniff.content_type}; + } catch (error) { + Logger.error({error, uploadFilename}, 'Failed to read media proxy sniff response'); + return null; + } + } + async extractFrames(request: MediaProxyFrameRequest): Promise { const response = await this.makeRequest('/_frames', request); if (!response) { @@ -183,7 +223,11 @@ export class MediaService extends IMediaService { return data; } - private async makeRequest(endpoint: string, body: MediaProxyRequestBody): Promise { + private async makeRequest( + endpoint: string, + body: MediaProxyRequestBody, + timeoutMs: number = MEDIA_PROXY_REQUEST_TIMEOUT_MS, + ): Promise { try { const url = `http://${Config.mediaProxy.host}:${Config.mediaProxy.port}${endpoint}`; const response = await fetch(url, { @@ -193,7 +237,7 @@ export class MediaService extends IMediaService { 'Content-Type': 'application/json', Authorization: `Bearer ${Config.mediaProxy.secretKey}`, }, - signal: AbortSignal.timeout(MEDIA_PROXY_REQUEST_TIMEOUT_MS), + signal: AbortSignal.timeout(timeoutMs), }); if (!response.ok) { const errorText = await FetchUtils.streamToStringWithLimit(response.body, { diff --git a/fluxer_api/src/api/openapi/openapi.json b/fluxer_api/src/api/openapi/openapi.json index 0aa798912..8301ac723 100644 --- a/fluxer_api/src/api/openapi/openapi.json +++ b/fluxer_api/src/api/openapi/openapi.json @@ -107,6 +107,63 @@ "security": [{"botToken": []}] } }, + "/attachments/refresh-urls": { + "post": { + "operationId": "refresh_attachment_urls", + "summary": "Refresh attachment URLs", + "tags": ["Messages"], + "responses": { + "200": { + "description": "Success", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/RefreshAttachmentUrlsResponse"}}} + }, + "400": { + "description": "Bad Request - The request was malformed or contained invalid data", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + }, + "401": { + "description": "Unauthorized - Authentication is required or the token is invalid", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + }, + "403": { + "description": "Forbidden - You do not have permission to perform this action", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + }, + "429": { + "description": "Too Many Requests - You are being rate limited", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ThrottledError"}}}, + "headers": { + "Retry-After": { + "description": "Number of seconds to wait before retrying (only on 429)", + "schema": {"type": "integer"} + }, + "X-RateLimit-Limit": { + "description": "The number of requests that can be made in the current window", + "schema": {"type": "integer"} + }, + "X-RateLimit-Remaining": { + "description": "The number of remaining requests that can be made", + "schema": {"type": "integer"} + }, + "X-RateLimit-Reset": { + "description": "Unix timestamp when the rate limit resets", + "schema": {"type": "integer"} + } + } + }, + "500": { + "description": "Internal Server Error - An unexpected error occurred", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}} + } + }, + "description": "Reissues the expiring signature on attachment URLs. Returns one entry per requested URL, in the order they were requested, each pairing the URL exactly as it was sent with a freshly signed copy. A URL that is not an attachment URL of this instance is returned unchanged. No membership or existence check is performed.", + "security": [{"botToken": []}, {"sessionToken": []}], + "requestBody": { + "required": true, + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/RefreshAttachmentUrlsRequest"}}} + } + } + }, "/auth/authorize-ip": { "post": { "operationId": "authorize_ip_address", @@ -28845,6 +28902,31 @@ "properties": {"token": {"description": "The IP authorization token from email", "type": "string"}}, "required": ["token"] }, + "RefreshAttachmentUrlsRequest": { + "type": "object", + "properties": { + "attachment_urls": { + "minItems": 1, + "maxItems": 50, + "type": "array", + "items": {"type": "string", "maxLength": 2048}, + "description": "Attachment URLs to refresh (1-50 entries, each at most 2048 characters)" + } + }, + "required": ["attachment_urls"] + }, + "RefreshAttachmentUrlsResponse": { + "type": "object", + "properties": { + "refreshed_urls": { + "type": "array", + "items": {"$ref": "#/components/schemas/RefreshedAttachmentUrl"}, + "description": "One entry per requested URL, in the order they were requested" + } + }, + "required": ["refreshed_urls"], + "additionalProperties": false + }, "ApplicationsMeResponse": { "type": "object", "properties": { @@ -29351,6 +29433,18 @@ {"name": "SPAMMER", "value": "64", "description": "User is flagged as a spammer"} ] }, + "RefreshedAttachmentUrl": { + "type": "object", + "properties": { + "original": {"type": "string", "description": "The requested URL, echoed back unchanged"}, + "refreshed": { + "type": "string", + "description": "The same URL carrying a fresh signature, or the original when it is not an attachment URL of ours" + } + }, + "required": ["original", "refreshed"], + "additionalProperties": false + }, "PasswordType": {"type": "string"}, "EmailType": {"type": "string"}, "AuthSessionLocation": { @@ -34217,8 +34311,8 @@ {"name": "Billing", "description": "Subscription and payment management via Stripe"}, {"name": "Premium", "description": "Premium subscription features and benefits"}, {"name": "Gifts", "description": "Gift codes and redemption"}, - {"name": "Connections"}, {"name": "Messages"}, + {"name": "Connections"}, {"name": "Donations"}, {"name": "Experiments"}, {"name": "Geolocation"}, diff --git a/fluxer_api/src/api/rate_limit_configs/ChannelRateLimitConfig.ts b/fluxer_api/src/api/rate_limit_configs/ChannelRateLimitConfig.ts index 0304a40a2..b9824196b 100644 --- a/fluxer_api/src/api/rate_limit_configs/ChannelRateLimitConfig.ts +++ b/fluxer_api/src/api/rate_limit_configs/ChannelRateLimitConfig.ts @@ -68,6 +68,10 @@ export const ChannelRateLimitConfigs = { bucket: 'attachment:delete', config: {limit: 40, windowMs: ms('10 seconds')}, } as RouteRateLimitConfig, + ATTACHMENT_URLS_REFRESH: { + bucket: 'attachment:refresh_urls', + config: {limit: 20, windowMs: ms('10 seconds')}, + } as RouteRateLimitConfig, CHANNEL_TYPING: { bucket: 'channel:typing::channel_id', config: {limit: 20, windowMs: ms('10 seconds')}, diff --git a/fluxer_api/src/api/test/Setup.ts b/fluxer_api/src/api/test/Setup.ts index ee0ea4a5d..1b80b99ee 100644 --- a/fluxer_api/src/api/test/Setup.ts +++ b/fluxer_api/src/api/test/Setup.ts @@ -62,6 +62,7 @@ function setDefaultTestEnv(): void { FLUXER_S3_SECRET_ACCESS_KEY: 'test', FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED: 'false', FLUXER_MEDIA_PROXY_SECRET_KEY: 'test-media-secret', + FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64: 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=', FLUXER_ADMIN_SECRET_KEY_BASE: 'test-admin-secret', FLUXER_ADMIN_OAUTH_CLIENT_SECRET: 'test-admin-oauth-secret', FLUXER_APP_PROXY_PORT: '8773', diff --git a/fluxer_api/src/api/test/TestMediaService.ts b/fluxer_api/src/api/test/TestMediaService.ts index 1bf91b95b..af05aebd7 100644 --- a/fluxer_api/src/api/test/TestMediaService.ts +++ b/fluxer_api/src/api/test/TestMediaService.ts @@ -8,6 +8,7 @@ import { type MediaProxyFrameResponse, type MediaProxyMetadataRequest, type MediaProxyMetadataResponse, + type MediaProxySniffResponse, } from '@app/api/infrastructure/IMediaService'; import type {IStorageService} from '@app/api/infrastructure/IStorageService'; @@ -87,6 +88,10 @@ export class TestMediaService extends IMediaService { return Buffer.alloc(1024); } + async sniffUpload(_uploadFilename: string): Promise { + return {content_type: null}; + } + async extractFrames(_request: MediaProxyFrameRequest): Promise { return { frames: [ diff --git a/fluxer_api/src/api/test/mocks/RepositoryBackedMessageResponseDataService.ts b/fluxer_api/src/api/test/mocks/RepositoryBackedMessageResponseDataService.ts index f8e023974..690386173 100644 --- a/fluxer_api/src/api/test/mocks/RepositoryBackedMessageResponseDataService.ts +++ b/fluxer_api/src/api/test/mocks/RepositoryBackedMessageResponseDataService.ts @@ -2,9 +2,9 @@ import {AttachmentDecayRepository} from '@app/api/attachment/AttachmentDecayRepository'; import {AttachmentDecayService} from '@app/api/attachment/AttachmentDecayService'; +import {makeSignedAttachmentCdnUrl, signAttachmentUrl} from '@app/api/attachment/AttachmentUrls'; import type {ChannelID, GuildID, MessageID, UserID} from '@app/api/BrandedTypes'; import {createUserID} from '@app/api/BrandedTypes'; -import {Config} from '@app/api/Config'; import { type MessageResponseAccessContext, MessageResponseDataService, @@ -58,6 +58,14 @@ class NoopNatsConnectionManager implements INatsConnectionManager { } } +function signOwnUrl(url: string | null | undefined): string | null { + return url == null ? null : signAttachmentUrl(url); +} + +function mediaProxyUrl(url: string | null | undefined): string | null { + return url?.startsWith('http') ? signAttachmentUrl(url) : null; +} + export class RepositoryBackedMessageResponseDataService extends MessageResponseDataService { private readonly attachmentDecayRepository = new AttachmentDecayRepository(); private readonly attachmentDecayService = new AttachmentDecayService(this.attachmentDecayRepository); @@ -343,8 +351,7 @@ export class RepositoryBackedMessageResponseDataService extends MessageResponseD } private mapAttachmentUrl(message: Message, attachment: Attachment): string { - const filename = encodeURIComponent(attachment.filename); - return `${Config.endpoints.media}/attachments/${message.channelId.toString()}/${message.id.toString()}/${attachment.id.toString()}/${filename}`; + return makeSignedAttachmentCdnUrl(message.channelId, attachment.id, attachment.filename); } private async mapAttachments( @@ -399,7 +406,7 @@ export class RepositoryBackedMessageResponseDataService extends MessageResponseD private mapEmbed(embed: Embed, message: Message): MessageEmbedResponse { return { type: embed.type ?? 'rich', - url: embed.url, + url: signOwnUrl(embed.url), title: embed.title, color: embed.color, timestamp: embed.timestamp?.toISOString() ?? null, @@ -425,9 +432,9 @@ export class RepositoryBackedMessageResponseDataService extends MessageResponseD const iconUrl = 'iconUrl' in author ? author.iconUrl : null; return { name: author.name, - url: author.url, - icon_url: iconUrl, - proxy_icon_url: iconUrl, + url: signOwnUrl(author.url), + icon_url: signOwnUrl(iconUrl), + proxy_icon_url: mediaProxyUrl(iconUrl), }; } @@ -435,8 +442,8 @@ export class RepositoryBackedMessageResponseDataService extends MessageResponseD if (!footer?.text) return null; return { text: footer.text, - icon_url: footer.iconUrl, - proxy_icon_url: footer.iconUrl, + icon_url: signOwnUrl(footer.iconUrl), + proxy_icon_url: mediaProxyUrl(footer.iconUrl), }; } @@ -450,10 +457,10 @@ export class RepositoryBackedMessageResponseDataService extends MessageResponseD private mapEmbedMedia(media: EmbedMedia | null, message: Message) { if (!media?.url) return null; - const url = this.resolveAttachmentUrl(media.url, message); + const resolved = this.resolveAttachmentUrl(media.url, message); return { - url, - proxy_url: url.startsWith('http') ? url : null, + url: signAttachmentUrl(resolved), + proxy_url: mediaProxyUrl(resolved), content_type: media.contentType, content_hash: media.contentHash, width: media.width, diff --git a/fluxer_api/src/api/utils/UnfurlerUtils.ts b/fluxer_api/src/api/utils/UnfurlerUtils.ts index 89d865aea..357e1f245 100644 --- a/fluxer_api/src/api/utils/UnfurlerUtils.ts +++ b/fluxer_api/src/api/utils/UnfurlerUtils.ts @@ -1,6 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import {createHash} from 'node:crypto'; +import {stripOwnAttachmentSignature} from '@app/api/attachment/AttachmentUrls'; import {Config} from '@app/api/Config'; import {Logger} from '@app/api/Logger'; import * as InviteUtils from '@app/api/utils/InviteUtils'; @@ -103,9 +104,10 @@ export function extractURLs(inputText: string) { if (isFluxerAppExcludedURL(url)) continue; const encoded = idnaEncodeURL(url); if (!encoded) continue; - if (!seen.has(encoded)) { - seen.add(encoded); - result.push(encoded); + const canonical = stripOwnAttachmentSignature(encoded); + if (!seen.has(canonical)) { + seen.add(canonical); + result.push(canonical); if (result.length >= 5) break; } } diff --git a/fluxer_api/src/api/webhook/WebhookService.ts b/fluxer_api/src/api/webhook/WebhookService.ts index 5f79e5f97..f748a95ec 100644 --- a/fluxer_api/src/api/webhook/WebhookService.ts +++ b/fluxer_api/src/api/webhook/WebhookService.ts @@ -1,6 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import fs from 'node:fs/promises'; +import {stripOwnAttachmentSignature} from '@app/api/attachment/AttachmentUrls'; import type {ChannelID, GuildID, MessageID, UserID, WebhookID, WebhookToken} from '@app/api/BrandedTypes'; import {createChannelID, createGuildID, createWebhookID, createWebhookToken} from '@app/api/BrandedTypes'; import type {IChannelRepository} from '@app/api/channel/IChannelRepository'; @@ -571,12 +572,13 @@ export class WebhookService { private async getWebhookAvatar({ webhookId, - avatarUrl, + avatarUrl: requestedAvatarUrl, }: { webhookId: WebhookID; avatarUrl: string | null; }): Promise { - if (!avatarUrl) return null; + if (!requestedAvatarUrl) return null; + const avatarUrl = stripOwnAttachmentSignature(requestedAvatarUrl); try { const cacheKey = `webhook:${webhookId}:avatar:${avatarUrl}`; const avatarCache = await this.cacheService.get(cacheKey); diff --git a/fluxer_api/src/api/worker/tasks/HarvestGuildData.ts b/fluxer_api/src/api/worker/tasks/HarvestGuildData.ts index 52c0f515a..7f7e73fb6 100644 --- a/fluxer_api/src/api/worker/tasks/HarvestGuildData.ts +++ b/fluxer_api/src/api/worker/tasks/HarvestGuildData.ts @@ -11,9 +11,10 @@ import { createArchiveTask, throwIfArchiveTerminallyFailed, } from '@app/api/archive/ArchiveTask'; +import {makeDataPackageAttachmentCdnUrl} from '@app/api/attachment/AttachmentUrls'; import {type AttachmentID, type ChannelID, createGuildID, type MessageID} from '@app/api/BrandedTypes'; import {Config} from '@app/api/Config'; -import {makeAttachmentCdnKey, makeAttachmentCdnUrl} from '@app/api/channel/services/message/MessageHelpers'; +import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers'; import type {IStorageService} from '@app/api/infrastructure/IStorageService'; import {Logger} from '@app/api/Logger'; import {mapWithConcurrency} from '@app/api/utils/ConcurrencyUtils'; @@ -54,6 +55,32 @@ interface PendingAttachmentDownload { filename: string; } +export interface GuildHarvestAttachment { + id: AttachmentID; + filename: string; + size: bigint; + contentType: string; + width: number | null; + height: number | null; +} + +export function buildGuildHarvestAttachment( + channelId: ChannelID, + attachment: GuildHarvestAttachment, + includeAttachments: boolean, +): Record { + return { + attachment_id: attachment.id.toString(), + filename: attachment.filename, + size: attachment.size.toString(), + content_type: attachment.contentType, + archive_path: includeAttachments ? `attachments/${channelId}/${attachment.id}/${attachment.filename}` : null, + cdn_url: makeDataPackageAttachmentCdnUrl(channelId, attachment.id, attachment.filename), + width: attachment.width, + height: attachment.height, + }; +} + function cdnBucket(): string { return Config.s3.buckets.cdn; } @@ -182,16 +209,7 @@ const harvestGuildData: ArchiveTaskHandler = async (payload, helpers, attempt) = if (msg.authorId == null) continue; const attachments: Array = []; for (const att of msg.attachments) { - attachments.push({ - attachment_id: att.id.toString(), - filename: att.filename, - size: att.size.toString(), - content_type: att.contentType, - archive_path: validated.includeAttachments ? `attachments/${channel.id}/${att.id}/${att.filename}` : null, - cdn_url: makeAttachmentCdnUrl(channel.id, att.id, att.filename), - width: att.width, - height: att.height, - }); + attachments.push(buildGuildHarvestAttachment(channel.id, att, validated.includeAttachments)); if (validated.includeAttachments) { channelDownloads.push({ channelId: channel.id, diff --git a/fluxer_api/src/api/worker/tasks/HarvestUserData.ts b/fluxer_api/src/api/worker/tasks/HarvestUserData.ts index 91a54668c..aebfa0b5d 100644 --- a/fluxer_api/src/api/worker/tasks/HarvestUserData.ts +++ b/fluxer_api/src/api/worker/tasks/HarvestUserData.ts @@ -11,6 +11,7 @@ import { createArchiveTask, throwIfArchiveTerminallyFailed, } from '@app/api/archive/ArchiveTask'; +import {makeDataPackageAttachmentCdnUrl} from '@app/api/attachment/AttachmentUrls'; import { type ChannelID, createAttachmentID, @@ -21,7 +22,6 @@ import { type UserID, } from '@app/api/BrandedTypes'; import {Config} from '@app/api/Config'; -import {makeAttachmentCdnUrl} from '@app/api/channel/services/message/MessageHelpers'; import { isChannelEligible, isTimestampInWindow, @@ -340,7 +340,7 @@ export async function harvestMessages( content_type: attachment.contentType, content_hash: null, archive_path: null, - cdn_url: makeAttachmentCdnUrl(channelId, attachment.id, attachment.filename), + cdn_url: makeDataPackageAttachmentCdnUrl(channelId, attachment.id, attachment.filename), width: attachment.width, height: attachment.height, })), diff --git a/fluxer_app/src/features/app/constants/Endpoints.ts b/fluxer_app/src/features/app/constants/Endpoints.ts index c866ceb0c..712f2cb35 100644 --- a/fluxer_app/src/features/app/constants/Endpoints.ts +++ b/fluxer_app/src/features/app/constants/Endpoints.ts @@ -4,6 +4,7 @@ import {ME} from '@fluxer/constants/src/AppConstants'; export const Endpoints = { INSTANCE: '/instance', + ATTACHMENTS_REFRESH_URLS: '/attachments/refresh-urls', AUTH_LOGIN: '/auth/login', AUTH_LOGIN_MFA_TOTP: '/auth/login/mfa/totp', AUTH_LOGIN_MFA_WEBAUTHN_OPTIONS: '/auth/login/mfa/webauthn/authentication-options', diff --git a/fluxer_app/src/features/channel/components/AutocompleteMemePreview.tsx b/fluxer_app/src/features/channel/components/AutocompleteMemePreview.tsx index 58e12f29e..fc7f90f45 100644 --- a/fluxer_app/src/features/channel/components/AutocompleteMemePreview.tsx +++ b/fluxer_app/src/features/channel/components/AutocompleteMemePreview.tsx @@ -4,6 +4,7 @@ import {useAnimatedMediaVideoPlayback} from '@app/features/app/hooks/useAnimated import {useShouldAnimate} from '@app/features/app/hooks/useShouldAnimate'; import styles from '@app/features/channel/components/AutocompleteEmoji.module.css'; import type {FavoriteMeme} from '@app/features/expressions/models/FavoriteMeme'; +import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher'; import {buildStaticGifPreviewURL} from '@app/features/messaging/utils/MediaProxyUtils'; import {MusicNoteIcon} from '@phosphor-icons/react'; import {observer} from 'mobx-react-lite'; @@ -32,6 +33,7 @@ export const AutocompleteMemePreview = observer(({meme}: {meme: FavoriteMeme}) = const isVideoMeme = contentType.startsWith('video/'); const isAnimatedImageMeme = !isVideoMeme && !isAudioMeme && contentType.includes('gif'); const motionAllowed = useShouldAnimate({kind: 'gif', isAnimated: isVideoMeme || isAnimatedImageMeme}); + const memeUrl = AttachmentUrlRefresher.fresh(meme.url, {refreshUnsigned: true}); if (isAudioMeme) { return (
@@ -44,16 +46,14 @@ export const AutocompleteMemePreview = observer(({meme}: {meme: FavoriteMeme}) = ); } if (isVideoMeme && motionAllowed) { - return ( - - ); + return ; } const needsStillFrame = (isVideoMeme || isAnimatedImageMeme) && !motionAllowed; return ( {meme.name} | null, ): AttachmentRenderingState => { const attachments = snapshotAttachments ?? []; - const expiryApplied = mapAttachmentsWithExpiry(attachments, DeveloperOptions.mockAttachmentStates); + const expiryApplied = mapAttachmentsWithExpiry(attachments, DeveloperOptions.mockAttachmentStates).map((entry) => ({ + isExpired: entry.isExpired, + attachment: withFreshAttachmentUrls(entry.attachment), + })); const enrichedAttachments = expiryApplied.map((entry) => entry.attachment); const activeAttachments = expiryApplied.filter((entry) => !entry.isExpired).map((entry) => entry.attachment); const {mediaAttachments} = splitMediaAndFileAttachments(activeAttachments); diff --git a/fluxer_app/src/features/channel/components/MobileMemesPicker.tsx b/fluxer_app/src/features/channel/components/MobileMemesPicker.tsx index 00dc6854a..b00d76d3b 100644 --- a/fluxer_app/src/features/channel/components/MobileMemesPicker.tsx +++ b/fluxer_app/src/features/channel/components/MobileMemesPicker.tsx @@ -20,6 +20,7 @@ import FavoriteMemes from '@app/features/expressions/state/FavoriteMemes'; import {AUDIO_DESCRIPTOR, GIFS_DESCRIPTOR} from '@app/features/i18n/utils/CommonMessageDescriptors'; import {isKeyboardActivationKey} from '@app/features/input/utils/KeyboardUtils'; import {useNearViewport} from '@app/features/messaging/hooks/useNearViewport'; +import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher'; import {buildStaticGifPreviewURL} from '@app/features/messaging/utils/MediaProxyUtils'; import {ComponentBus} from '@app/features/platform/utils/ComponentBus'; import {DeleteIcon, EditIcon} from '@app/features/ui/action_menu/ContextMenuIcons'; @@ -428,7 +429,8 @@ const GridItem = observer( const isAudio = contentType.startsWith('audio/'); const isGifImage = !isVideo && contentType.toLowerCase().includes('gif'); const shouldAnimateGif = useShouldAnimate({kind: 'gif', isAnimated: !isAudio && (isVideo || isGifImage)}); - const thumbnailSrc = isGifImage && !shouldAnimateGif ? buildStaticGifPreviewURL(url) : url; + const memeUrl = AttachmentUrlRefresher.fresh(url, {refreshUnsigned: true}); + const thumbnailSrc = isGifImage && !shouldAnimateGif ? buildStaticGifPreviewURL(memeUrl) : memeUrl; const videoPlaybackAllowed = useAnimatedMediaVideoPlayback(videoRef, { enabled: isVisible && !isAudio && isVideo, shouldPlay: shouldAnimateGif, @@ -471,7 +473,7 @@ const GridItem = observer( disablePictureInPicture={true} disableRemotePlayback={true} preload={shouldAnimateGif ? 'auto' : 'metadata'} - src={url} + src={memeUrl} data-flx="channel.mobile-memes-picker.grid-item.gif" /> )} diff --git a/fluxer_app/src/features/channel/components/embeds/ChannelEmbed.tsx b/fluxer_app/src/features/channel/components/embeds/ChannelEmbed.tsx index 07772a6b0..e643e58b4 100644 --- a/fluxer_app/src/features/channel/components/embeds/ChannelEmbed.tsx +++ b/fluxer_app/src/features/channel/components/embeds/ChannelEmbed.tsx @@ -26,6 +26,7 @@ import Channels from '@app/features/channel/state/Channels'; import Guilds from '@app/features/guild/state/Guilds'; import {SUPPRESS_EMBEDS_DESCRIPTOR} from '@app/features/i18n/utils/CommonMessageDescriptors'; import * as MessageCommands from '@app/features/messaging/commands/MessageCommands'; +import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher'; import {getEmbedMediaDimensions} from '@app/features/messaging/utils/MediaDimensionConfig'; import {buildAnimatedImageProxyURL} from '@app/features/messaging/utils/MediaProxyUtils'; import {buildMessageEmbedCopyText} from '@app/features/messaging/utils/MessageCopyTextUtils'; @@ -39,6 +40,7 @@ import FocusRing from '@app/features/ui/focus_ring/FocusRing'; import MobileLayout from '@app/features/ui/state/MobileLayout'; import {MessageAttachmentFlags, MessageEmbedTypes, Permissions} from '@fluxer/constants/src/ChannelConstants'; import {GuildOperations} from '@fluxer/constants/src/GuildConstants'; +import type {EmbedMedia, MessageEmbed} from '@fluxer/schema/src/domains/message/EmbedSchemas'; import {useLingui} from '@lingui/react/macro'; import {XIcon} from '@phosphor-icons/react'; import {clsx} from 'clsx'; @@ -47,7 +49,29 @@ import type React from 'react'; import {type FC, useCallback, useMemo} from 'react'; const mediaFocusRingClass = messageStyles.mediaFocusRing; -export const Embed: FC = observer(({embed, message, embedIndex, contextualEmbeds, onDelete, isPreview}) => { + +function withFreshEmbedMediaUrls(media: EmbedMedia | undefined): EmbedMedia | undefined { + if (!media) return media; + const url = AttachmentUrlRefresher.fresh(media.url); + const proxyUrl = media.proxy_url === undefined ? undefined : AttachmentUrlRefresher.fresh(media.proxy_url); + if (url === media.url && proxyUrl === media.proxy_url) return media; + return {...media, url, proxy_url: proxyUrl}; +} + +function withFreshEmbedUrls(embed: MessageEmbed): MessageEmbed { + const image = withFreshEmbedMediaUrls(embed.image); + const thumbnail = withFreshEmbedMediaUrls(embed.thumbnail); + const video = withFreshEmbedMediaUrls(embed.video); + const audio = withFreshEmbedMediaUrls(embed.audio); + if (image === embed.image && thumbnail === embed.thumbnail && video === embed.video && audio === embed.audio) { + return embed; + } + return {...embed, image, thumbnail, video, audio}; +} + +export const Embed: FC = observer((props: EmbedProps) => { + const {message, embedIndex, contextualEmbeds, onDelete, isPreview} = props; + const embed = withFreshEmbedUrls(props.embed); const {i18n} = useLingui(); const {enabled: isMobile} = MobileLayout; const channel = Channels.getChannel(message.channelId); diff --git a/fluxer_app/src/features/channel/components/embeds/attachments/TextualAttachmentPreview.tsx b/fluxer_app/src/features/channel/components/embeds/attachments/TextualAttachmentPreview.tsx index cf29ddae6..d1e49a091 100644 --- a/fluxer_app/src/features/channel/components/embeds/attachments/TextualAttachmentPreview.tsx +++ b/fluxer_app/src/features/channel/components/embeds/attachments/TextualAttachmentPreview.tsx @@ -9,6 +9,10 @@ import {CsvAttachmentTablePanel} from '@app/features/channel/components/embeds/a import {TextualAttachmentCodePanel} from '@app/features/channel/components/embeds/attachments/TextualAttachmentCodePanel'; import styles from '@app/features/channel/components/embeds/attachments/TextualAttachmentPreview.module.css'; import {TextualAttachmentPreviewBottomSheet} from '@app/features/channel/components/embeds/attachments/TextualAttachmentPreviewBottomSheet'; +import { + fetchTextualPreviewText, + PreviewSizeLimitError, +} from '@app/features/channel/components/embeds/attachments/TextualAttachmentPreviewFetch'; import {TextualAttachmentPreviewFooter} from '@app/features/channel/components/embeds/attachments/TextualAttachmentPreviewFooter'; import {TextualAttachmentPreviewModal} from '@app/features/channel/components/embeds/attachments/TextualAttachmentPreviewModal'; import { @@ -45,56 +49,6 @@ import {type MouseEvent, useCallback, useEffect, useMemo, useState} from 'react' const logger = new Logger('TextualAttachmentPreview'); -class PreviewSizeLimitError extends Error { - constructor() { - super('Attachment preview exceeds the size limit'); - this.name = 'PreviewSizeLimitError'; - } -} - -async function readPreviewText(response: Response): Promise { - const contentLength = response.headers.get('content-length'); - if (contentLength !== null) { - const parsedContentLength = Number(contentLength); - if (Number.isFinite(parsedContentLength) && parsedContentLength > TEXT_PREVIEW_MAX_BYTES) { - throw new PreviewSizeLimitError(); - } - } - const body = response.body; - if (!body) { - throw new Error('Attachment preview response has no readable body'); - } - const reader = body.getReader(); - const chunks: Array = []; - let totalBytes = 0; - try { - while (true) { - const {done, value} = await reader.read(); - if (done) { - break; - } - if (!value) { - continue; - } - totalBytes += value.byteLength; - if (totalBytes > TEXT_PREVIEW_MAX_BYTES) { - await reader.cancel().catch(() => undefined); - throw new PreviewSizeLimitError(); - } - chunks.push(value); - } - } finally { - reader.releaseLock(); - } - const bytes = new Uint8Array(totalBytes); - let offset = 0; - for (const chunk of chunks) { - bytes.set(chunk, offset); - offset += chunk.byteLength; - } - return new TextDecoder().decode(bytes); -} - export const TextualAttachmentPreview = observer(function TextualAttachmentPreview({ attachment, spoilerHidden = false, @@ -146,13 +100,7 @@ export const TextualAttachmentPreview = observer(function TextualAttachmentPrevi setStatus('loading'); setPreviewError(null); const controller = new AbortController(); - fetch(attachment.url, {signal: controller.signal}) - .then((response) => { - if (!response.ok) { - throw new Error(`Attachment preview request failed: ${response.status} ${response.statusText}`); - } - return readPreviewText(response); - }) + fetchTextualPreviewText(attachment.url, controller.signal) .then((value) => { if (controller.signal.aborted) { return; diff --git a/fluxer_app/src/features/channel/components/embeds/attachments/TextualAttachmentPreviewFetch.ts b/fluxer_app/src/features/channel/components/embeds/attachments/TextualAttachmentPreviewFetch.ts new file mode 100644 index 000000000..27f93621d --- /dev/null +++ b/fluxer_app/src/features/channel/components/embeds/attachments/TextualAttachmentPreviewFetch.ts @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher'; +import {TEXT_PREVIEW_MAX_BYTES} from '@app/features/messaging/utils/AttachmentPreviewUtils'; + +const SIGNATURE_REFUSAL_STATUSES = new Set([401, 403, 404, 410]); + +export class PreviewSizeLimitError extends Error { + constructor() { + super('Attachment preview exceeds the size limit'); + this.name = 'PreviewSizeLimitError'; + } +} + +export function isAttachmentSignatureRefusal(status: number): boolean { + return SIGNATURE_REFUSAL_STATUSES.has(status); +} + +export async function readPreviewText(response: Response): Promise { + const contentLength = response.headers.get('content-length'); + if (contentLength !== null) { + const parsedContentLength = Number(contentLength); + if (Number.isFinite(parsedContentLength) && parsedContentLength > TEXT_PREVIEW_MAX_BYTES) { + throw new PreviewSizeLimitError(); + } + } + const body = response.body; + if (!body) { + throw new Error('Attachment preview response has no readable body'); + } + const reader = body.getReader(); + const chunks: Array = []; + let totalBytes = 0; + try { + while (true) { + const {done, value} = await reader.read(); + if (done) { + break; + } + if (!value) { + continue; + } + totalBytes += value.byteLength; + if (totalBytes > TEXT_PREVIEW_MAX_BYTES) { + await reader.cancel().catch(() => undefined); + throw new PreviewSizeLimitError(); + } + chunks.push(value); + } + } finally { + reader.releaseLock(); + } + const bytes = new Uint8Array(totalBytes); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return new TextDecoder().decode(bytes); +} + +function requestFailed(response: Response): Error { + return new Error(`Attachment preview request failed: ${response.status} ${response.statusText}`); +} + +export async function fetchTextualPreviewText(url: string, signal: AbortSignal): Promise { + const target = await AttachmentUrlRefresher.refresh(url); + const response = await fetch(target, {signal}); + if (response.ok) { + return readPreviewText(response); + } + if (!isAttachmentSignatureRefusal(response.status)) { + throw requestFailed(response); + } + const refreshed = await AttachmentUrlRefresher.refresh(url, {force: true}); + if (refreshed === target) { + throw requestFailed(response); + } + const retried = await fetch(refreshed, {signal}); + if (!retried.ok) { + throw requestFailed(retried); + } + return readPreviewText(retried); +} diff --git a/fluxer_app/src/features/channel/components/embeds/media/EmbedGifv.tsx b/fluxer_app/src/features/channel/components/embeds/media/EmbedGifv.tsx index a3d684eb2..a5de76ce2 100644 --- a/fluxer_app/src/features/channel/components/embeds/media/EmbedGifv.tsx +++ b/fluxer_app/src/features/channel/components/embeds/media/EmbedGifv.tsx @@ -19,6 +19,7 @@ import {safePause, safePlay} from '@app/features/channel/components/GifVideoPool import {useMaybeMessageViewContext} from '@app/features/channel/components/MessageViewContext'; import type {Channel} from '@app/features/channel/models/Channel'; import {isKeyboardActivationKey} from '@app/features/input/utils/KeyboardUtils'; +import {useAttachmentRefreshOnError} from '@app/features/messaging/hooks/useAttachmentRefreshOnError'; import {useDeleteAttachment} from '@app/features/messaging/hooks/useDeleteAttachment'; import {useMatureMedia} from '@app/features/messaging/hooks/useMatureMedia'; import {useMediaFavorite} from '@app/features/messaging/hooks/useMediaFavorite'; @@ -389,6 +390,7 @@ export const EmbedGifv: FC< const messageViewContext = useMaybeMessageViewContext(); const mediaCalculator = useEmbedMediaCalculator(); const videoRef = useRef(null); + const handleVideoError = useAttachmentRefreshOnError(videoProxyURL); const [containerElement, setContainerElement] = useState(null); const {isPointerInside, hasFocusInside} = useMediaSurfaceInteraction(containerElement); const {shouldBlur, gateReason, canReveal, reveal: revealSensitiveMedia} = useMatureMedia(nsfw, channelId); @@ -632,6 +634,7 @@ export const EmbedGifv: FC< height={dimensions.height} tabIndex={-1} aria-label={i18n._(ANIMATED_GIF_VIDEO_DESCRIPTOR)} + onError={handleVideoError} data-embed-media="gifv" data-flx="channel.embeds.media.embed-gifv.video-element" /> diff --git a/fluxer_app/src/features/channel/components/embeds/media/VoiceMessagePlayer.tsx b/fluxer_app/src/features/channel/components/embeds/media/VoiceMessagePlayer.tsx index 0247e7450..0187888f8 100644 --- a/fluxer_app/src/features/channel/components/embeds/media/VoiceMessagePlayer.tsx +++ b/fluxer_app/src/features/channel/components/embeds/media/VoiceMessagePlayer.tsx @@ -6,6 +6,7 @@ import styles from '@app/features/channel/components/embeds/media/VoiceMessagePl import {useMaybeMessageViewContext} from '@app/features/channel/components/MessageViewContext'; import {PAUSE_DESCRIPTOR, PLAY_DESCRIPTOR} from '@app/features/i18n/utils/CommonMessageDescriptors'; import {getCachedNumberFormat} from '@app/features/i18n/utils/IntlCache'; +import {useAttachmentRefreshOnError} from '@app/features/messaging/hooks/useAttachmentRefreshOnError'; import {buildMediaProxyURL} from '@app/features/messaging/utils/MediaProxyUtils'; import {Logger} from '@app/features/platform/utils/AppLogger'; import {remFromPx} from '@app/features/theme/layout/RemFromPx'; @@ -119,6 +120,7 @@ const VoiceMessagePlayer: React.FC = observer( const {i18n} = useLingui(); const messageViewContext = useMaybeMessageViewContext(); const effectiveSrc = buildMediaProxyURL(src); + const handleMediaError = useAttachmentRefreshOnError(effectiveSrc); const [hasStarted, setHasStarted] = useState(false); const [wantsMetadata, setWantsMetadata] = useState(false); const [prePlayCurrentTime, setPrePlayCurrentTime] = useState(0); @@ -324,6 +326,7 @@ const VoiceMessagePlayer: React.FC = observer( ref={mediaRef as React.RefObject} src={hasStarted || wantsMetadata ? effectiveSrc : undefined} preload={wantsMetadata ? 'metadata' : 'none'} + onError={handleMediaError} data-flx="channel.embeds.media.voice-message-player.audio" > diff --git a/fluxer_app/src/features/channel/components/pickers/gif/FavoriteGifTypes.ts b/fluxer_app/src/features/channel/components/pickers/gif/FavoriteGifTypes.ts index 286cf6e5c..da3af0610 100644 --- a/fluxer_app/src/features/channel/components/pickers/gif/FavoriteGifTypes.ts +++ b/fluxer_app/src/features/channel/components/pickers/gif/FavoriteGifTypes.ts @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +import {stripAttachmentSignature} from '@app/features/messaging/utils/AttachmentCdnUrl'; import {inferFormatContentType, PREVIEW_FORMAT_PRIORITY} from '@fluxer/schema/src/domains/gif/GifMediaFormatKeys'; export interface FavoriteGifMediaFormat { @@ -107,16 +108,36 @@ export function pickCanonicalPreviewFormat( export const STORED_PREVIEW_DEVICE_PIXEL_RATIO = 2; +export function stripFavoriteGifEntrySignatures(entry: FavoriteGifEntry): FavoriteGifEntry { + const url = stripAttachmentSignature(entry.url); + const proxyUrl = stripAttachmentSignature(entry.proxy_url); + const media: Record = {}; + let mediaChanged = false; + for (const [key, format] of Object.entries(entry.media)) { + const src = stripAttachmentSignature(format.src); + const proxySrc = stripAttachmentSignature(format.proxy_src); + if (src === format.src && proxySrc === format.proxy_src) { + media[key] = format; + continue; + } + mediaChanged = true; + media[key] = {...format, src, proxy_src: proxySrc}; + } + if (url === entry.url && proxyUrl === entry.proxy_url && !mediaChanged) return entry; + return {...entry, url, proxy_url: proxyUrl, media}; +} + export function slimFavoriteGifEntry(entry: FavoriteGifEntry): FavoriteGifEntry { - const best = pickBestPreviewFormat(entry.media, 'any', { + const stripped = stripFavoriteGifEntrySignatures(entry); + const best = pickBestPreviewFormat(stripped.media, 'any', { cssWidth: PREVIEW_TILE_CSS_WIDTH, devicePixelRatio: STORED_PREVIEW_DEVICE_PIXEL_RATIO, }); if (best == null) { - return Object.keys(entry.media).length === 0 ? entry : {...entry, media: {}}; + return Object.keys(stripped.media).length === 0 ? stripped : {...stripped, media: {}}; } return { - ...entry, + ...stripped, proxy_url: best.format.proxy_src, width: best.format.width, height: best.format.height, diff --git a/fluxer_app/src/features/channel/components/pickers/gif/GifPickerGrid.tsx b/fluxer_app/src/features/channel/components/pickers/gif/GifPickerGrid.tsx index 697969ae9..e3d64cfd8 100644 --- a/fluxer_app/src/features/channel/components/pickers/gif/GifPickerGrid.tsx +++ b/fluxer_app/src/features/channel/components/pickers/gif/GifPickerGrid.tsx @@ -16,6 +16,7 @@ import FavoriteGif from '@app/features/expressions/state/FavoriteGif'; import FavoriteMemes from '@app/features/expressions/state/FavoriteMemes'; import * as GifSlugUtils from '@app/features/expressions/utils/GifSlugUtils'; import {FAVORITES_DESCRIPTOR} from '@app/features/i18n/utils/CommonMessageDescriptors'; +import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher'; import QuickSwitcher from '@app/features/search/state/QuickSwitcher'; import {msg} from '@lingui/core/macro'; import {useLingui} from '@lingui/react/macro'; @@ -57,6 +58,7 @@ export const GifPickerGrid = observer( const favoriteGifs = FavoriteGif.favoriteGifs; const favoriteGifsVersion = favoriteGifs.length; const useSavedMediaForGifFavorites = FavoriteGif.saveGifFavoritesAsSavedMedia; + const attachmentUrlRevision = AttachmentUrlRefresher.revision; const data: Array = useMemo(() => { return buildGifPickerGridData({ surface: store.isShowingFavorites ? 'favorites' : store.isShowingFeatured ? 'featured' : 'results', @@ -88,6 +90,7 @@ export const GifPickerGrid = observer( selectGif, store.featuredFavoritePreviewSeed, i18n.locale, + attachmentUrlRevision, ]); const itemKeys = useMemo(() => data.filter((item) => item.type !== 'skeleton').map((item) => item.key), [data]); const itemByKey = useMemo(() => new Map(data.map((item) => [item.key, item])), [data]); diff --git a/fluxer_app/src/features/channel/components/pickers/gif/GifPickerGridData.ts b/fluxer_app/src/features/channel/components/pickers/gif/GifPickerGridData.ts index bca5a047e..ba3306214 100644 --- a/fluxer_app/src/features/channel/components/pickers/gif/GifPickerGridData.ts +++ b/fluxer_app/src/features/channel/components/pickers/gif/GifPickerGridData.ts @@ -8,6 +8,7 @@ import { import type {GifPickerGridItemData} from '@app/features/channel/components/pickers/gif/GifPickerTypes'; import type {Gif, GifFeatured} from '@app/features/expressions/commands/GifCommands'; import * as GifSlugUtils from '@app/features/expressions/utils/GifSlugUtils'; +import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher'; const CATEGORY_TILE_WIDTH = 200; const CATEGORY_TILE_HEIGHT = 96; @@ -34,6 +35,10 @@ export interface BuildGifPickerGridDataInput { trendingTitle: string; } +function freshStoredUrl(url: string): string { + return AttachmentUrlRefresher.fresh(url, {refreshUnsigned: true}); +} + export function buildSkeletonGifPickerItems(count: number): Array { return Array.from({length: count}, (_, i) => ({ type: 'skeleton', @@ -68,9 +73,9 @@ function buildFavoriteGifItems( for (let index = favoriteGifs.length - 1; index >= 0; index -= 1) { const entry = favoriteGifs[index]; const best = pickBestPreviewFormat(entry.media); - const fallbackSrc = GifSlugUtils.isUsableMediaSource(entry.proxy_url) ? entry.proxy_url : ''; - const previewSrc = best?.format.src ?? fallbackSrc; - const previewProxySrc = best?.format.proxy_src ?? fallbackSrc; + const fallbackSrc = GifSlugUtils.isUsableMediaSource(entry.proxy_url) ? freshStoredUrl(entry.proxy_url) : ''; + const previewSrc = best ? freshStoredUrl(best.format.src) : fallbackSrc; + const previewProxySrc = best ? freshStoredUrl(best.format.proxy_src) : fallbackSrc; const previewWidth = best?.format.width ?? entry.width; const previewHeight = best?.format.height ?? entry.height; const previewContentType = best ? inferFormatContentType(best.key) : entry.content_type; @@ -107,14 +112,21 @@ function buildFeaturedItems(input: BuildGifPickerGridDataInput): Array 0 ? (gifvMemes[Math.floor(input.featuredFavoritePreviewSeed * gifvMemes.length)] ?? null) : null; - const favoriteMemePreview = favoriteMemeCandidate?.url ?? ''; + const favoriteMemePreview = favoriteMemeCandidate ? freshStoredUrl(favoriteMemeCandidate.url) : ''; const usesFavoriteMemePreview = input.useSavedMediaForGifFavorites && input.favoriteGifs.length === 0; const favoriteTilePreview = usesFavoriteMemePreview ? favoriteMemePreview - : favoriteGifPreview?.format.src || favoriteGifPreviewEntry?.proxy_url || favoriteGifPreviewEntry?.url || ''; + : freshStoredUrl( + favoriteGifPreview?.format.src || favoriteGifPreviewEntry?.proxy_url || favoriteGifPreviewEntry?.url || '', + ); const favoriteTileProxyPreview = usesFavoriteMemePreview ? favoriteTilePreview - : favoriteGifPreview?.format.proxy_src || favoriteGifPreviewEntry?.proxy_url || favoriteGifPreviewEntry?.url || ''; + : freshStoredUrl( + favoriteGifPreview?.format.proxy_src || + favoriteGifPreviewEntry?.proxy_url || + favoriteGifPreviewEntry?.url || + '', + ); const favoriteTileContentType = (() => { if (usesFavoriteMemePreview) return favoriteMemeCandidate?.contentType ?? ''; if (favoriteGifPreview) return inferFormatContentType(favoriteGifPreview.key); diff --git a/fluxer_app/src/features/channel/components/pickers/memes/MemeGridItem.tsx b/fluxer_app/src/features/channel/components/pickers/memes/MemeGridItem.tsx index 0f6e14c93..ad06630d0 100644 --- a/fluxer_app/src/features/channel/components/pickers/memes/MemeGridItem.tsx +++ b/fluxer_app/src/features/channel/components/pickers/memes/MemeGridItem.tsx @@ -13,6 +13,7 @@ import * as FavoriteMemeCommands from '@app/features/expressions/commands/Favori import {EditFavoriteMemeModal} from '@app/features/expressions/components/modals/EditFavoriteMemeModal'; import type {FavoriteMeme} from '@app/features/expressions/models/FavoriteMeme'; import {isKeyboardActivationKey} from '@app/features/input/utils/KeyboardUtils'; +import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher'; import {buildMediaProxyURL, buildStaticGifPreviewURL} from '@app/features/messaging/utils/MediaProxyUtils'; import {ComponentBus} from '@app/features/platform/utils/ComponentBus'; import * as ModalCommands from '@app/features/ui/commands/ModalCommands'; @@ -113,10 +114,14 @@ export const MemeGridItem = observer( isHovering: isVideoPreviewActive, }); const shouldRenderVideoPreview = !isAudio && isVideo && isVideoPreviewActive && shouldAnimateGif; - const thumbnailSrc = isGifImage && !shouldAnimateGif ? buildStaticGifPreviewURL(meme.url) : meme.url; + const memeUrl = AttachmentUrlRefresher.fresh(meme.url, {refreshUnsigned: true}); + const thumbnailSrc = + isGifImage && !shouldAnimateGif + ? AttachmentUrlRefresher.fresh(buildStaticGifPreviewURL(meme.url), {refreshUnsigned: true}) + : memeUrl; const videoPreviewStartTime = meme.duration && meme.duration > 0 ? meme.duration / 2 : null; usePooledVideo({ - src: shouldRenderVideoPreview ? meme.url : null, + src: shouldRenderVideoPreview ? memeUrl : null, containerRef: videoContainerRef, videoPool, autoPlay: shouldRenderVideoPreview, @@ -219,7 +224,7 @@ export const MemeGridItem = observer( data-flx="channel.pickers.memes.meme-grid-item.div--2" > diff --git a/fluxer_app/src/features/expressions/state/FavoriteGif.ts b/fluxer_app/src/features/expressions/state/FavoriteGif.ts index 4a6366252..4c77f310d 100644 --- a/fluxer_app/src/features/expressions/state/FavoriteGif.ts +++ b/fluxer_app/src/features/expressions/state/FavoriteGif.ts @@ -4,7 +4,9 @@ import { type FavoriteGifEntry, type FavoriteGifMediaFormat, slimFavoriteGifEntry, + stripFavoriteGifEntrySignatures, } from '@app/features/channel/components/pickers/gif/FavoriteGifTypes'; +import {stripAttachmentSignature} from '@app/features/messaging/utils/AttachmentCdnUrl'; import {makeSyncedField} from '@app/features/user/state/SyncedField'; import {FAVORITE_GIF_MAX_ENCODED_BYTES} from '@app/features/user/state/SyncedFieldBudget'; import type {FavoriteGifMediaFormat as FavoriteGifMediaFormatProto} from '@fluxer/schema/src/gen/fluxer/user/preferences/v1/pickers_pb'; @@ -91,25 +93,29 @@ class FavoriteGif { } hasUrl(url: string): boolean { - return this.favoriteGifs.some((entry) => entry.url === url); + const target = stripAttachmentSignature(url); + return this.favoriteGifs.some((entry) => stripAttachmentSignature(entry.url) === target); } findByUrl(url: string): FavoriteGifEntry | null { - return this.favoriteGifs.find((entry) => entry.url === url) ?? null; + const target = stripAttachmentSignature(url); + return this.favoriteGifs.find((entry) => stripAttachmentSignature(entry.url) === target) ?? null; } addEntry(entry: FavoriteGifEntry): void { - if (this.hasUrl(entry.url)) return; - this.favoriteGifs = [...this.favoriteGifs, entry]; + const stored = stripFavoriteGifEntrySignatures(entry); + if (this.hasUrl(stored.url)) return; + this.favoriteGifs = [...this.favoriteGifs, stored]; } removeByUrl(url: string): void { if (!this.hasUrl(url)) return; - this.favoriteGifs = this.favoriteGifs.filter((entry) => entry.url !== url); + const target = stripAttachmentSignature(url); + this.favoriteGifs = this.favoriteGifs.filter((entry) => stripAttachmentSignature(entry.url) !== target); } replaceAll(entries: ReadonlyArray): void { - this.favoriteGifs = [...entries]; + this.favoriteGifs = entries.map(stripFavoriteGifEntrySignatures); } setSaveGifFavoritesAsSavedMedia(value: boolean): void { diff --git a/fluxer_app/src/features/gateway/transport/GatewayConnection.ts b/fluxer_app/src/features/gateway/transport/GatewayConnection.ts index 39ab06352..1d701f49b 100644 --- a/fluxer_app/src/features/gateway/transport/GatewayConnection.ts +++ b/fluxer_app/src/features/gateway/transport/GatewayConnection.ts @@ -28,6 +28,7 @@ import {selectGuildActivationTarget} from '@app/features/gateway/transport/Guild import GuildMatureContentAgree from '@app/features/guild/state/GuildMatureContentAgree'; import GuildMembers from '@app/features/member/state/GuildMembers'; import MemberSearch from '@app/features/member/state/MemberSearch'; +import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher'; import Messages from '@app/features/messaging/state/MessagingMessages'; import Navigation from '@app/features/navigation/state/Navigation'; import SelectedGuild from '@app/features/navigation/state/SelectedGuild'; @@ -587,6 +588,7 @@ class GatewayConnection { Presence.handleSessionInvalidated(); Messages.handleSessionInvalidated(); FavoriteMemes.reset(); + AttachmentUrlRefresher.reset(); GuildMatureContentAgree.reset(); Initialization.reset(); MemberSearch.handleLogout(); diff --git a/fluxer_app/src/features/messaging/components/markdown/renderers/LinkRenderer.tsx b/fluxer_app/src/features/messaging/components/markdown/renderers/LinkRenderer.tsx index 39c880d23..b19da2e9b 100644 --- a/fluxer_app/src/features/messaging/components/markdown/renderers/LinkRenderer.tsx +++ b/fluxer_app/src/features/messaging/components/markdown/renderers/LinkRenderer.tsx @@ -28,6 +28,7 @@ import { type RendererProps, } from '@app/features/messaging/components/markdown/renderers/RendererTypes'; import {ExternalLinkWarningModal} from '@app/features/messaging/components/modals/ExternalLinkWarningModal'; +import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher'; import {openExternalUrlWithWarning} from '@app/features/messaging/utils/ExternalLinkUtils'; import {goToMessage} from '@app/features/messaging/utils/MessageNavigator'; import type {LinkNode} from '@app/features/messaging/utils/markdown/parser/Nodes'; @@ -889,12 +890,16 @@ export const LinkRenderer = observer(function LinkRenderer({ logger.warn('Invalid URL in link:', url); } } + const href = AttachmentUrlRefresher.fresh(url); + const warmAttachmentUrl = () => AttachmentUrlRefresher.warm(url); return ( { e.stopPropagation(); if (handleClick) { diff --git a/fluxer_app/src/features/messaging/components/modals/MediaViewerModal.tsx b/fluxer_app/src/features/messaging/components/modals/MediaViewerModal.tsx index b25e9cd35..b9ff4d294 100644 --- a/fluxer_app/src/features/messaging/components/modals/MediaViewerModal.tsx +++ b/fluxer_app/src/features/messaging/components/modals/MediaViewerModal.tsx @@ -18,6 +18,7 @@ import {getMediaViewerPortalRoot} from '@app/features/messaging/components/modal import {useDeleteAttachment} from '@app/features/messaging/hooks/useDeleteAttachment'; import {useMediaFavorite} from '@app/features/messaging/hooks/useMediaFavorite'; import type {Message} from '@app/features/messaging/models/MessagingMessage'; +import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher'; import {formatAttachmentDate} from '@app/features/messaging/utils/AttachmentExpiryUtils'; import {openExternalUrlWithWarning} from '@app/features/messaging/utils/ExternalLinkUtils'; import {createDownloadHandler} from '@app/features/messaging/utils/FileDownloadUtils'; @@ -273,13 +274,20 @@ const MobileMediaOptionsSheet: FC = observer(funct /> ); }); +function withFreshViewerItemUrls(item: MediaViewerItem): MediaViewerItem { + if (!item) return item; + const src = AttachmentUrlRefresher.fresh(item.src); + const originalSrc = AttachmentUrlRefresher.fresh(item.originalSrc); + if (src === item.src && originalSrc === item.originalSrc) return item; + return {...item, src, originalSrc}; +} const MediaViewerModalComponent: FC = observer(() => { const {i18n} = useLingui(); const {isOpen, items, currentIndex, channelId, messageId, message, sourceChannel, allowAttachmentDelete} = MediaViewer; const {enabled: isMobile} = MobileLayout; const [isMediaMenuOpen, setIsMediaMenuOpen] = useState(false); - const currentItem = items[currentIndex]; + const currentItem = withFreshViewerItemUrls(items[currentIndex]); const currentGifvIsActualGif = currentItem != null && isGifvRenderedAsImage(currentItem); useBottomSheetBackHandler(isOpen, MediaViewerCommands.closeMediaViewer); useEffect(() => { @@ -296,7 +304,10 @@ const MediaViewerModalComponent: FC = observer(() => { count === 2 ? [(currentIndex + 1) % count] : [(currentIndex - 1 + count) % count, (currentIndex + 1) % count]; const unpinCallbacks = adjacentIndices.map((index) => { const item = items[index]; - if (!item || !isViewerImageItem(item) || item.src.startsWith('blob:')) return () => {}; + if (!item) return () => {}; + AttachmentUrlRefresher.warm(item.src); + AttachmentUrlRefresher.warm(item.originalSrc); + if (!isViewerImageItem(item) || item.src.startsWith('blob:')) return () => {}; return ImageCacheUtils.pinImage(buildViewerMediaURL(item)); }); return () => { diff --git a/fluxer_app/src/features/messaging/hooks/useAttachmentRefreshOnError.ts b/fluxer_app/src/features/messaging/hooks/useAttachmentRefreshOnError.ts new file mode 100644 index 000000000..7d84622b4 --- /dev/null +++ b/fluxer_app/src/features/messaging/hooks/useAttachmentRefreshOnError.ts @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher'; +import {useCallback, useRef} from 'react'; + +export function useAttachmentRefreshOnError(src: string | undefined): () => void { + const refreshedSourceRef = useRef(null); + return useCallback(() => { + if (src === undefined || src.length === 0) return; + if (refreshedSourceRef.current === src) return; + refreshedSourceRef.current = src; + void AttachmentUrlRefresher.refresh(src, {force: true}); + }, [src]); +} diff --git a/fluxer_app/src/features/messaging/hooks/useMediaLoading.ts b/fluxer_app/src/features/messaging/hooks/useMediaLoading.ts index e31bcf2eb..a2837bbed 100644 --- a/fluxer_app/src/features/messaging/hooks/useMediaLoading.ts +++ b/fluxer_app/src/features/messaging/hooks/useMediaLoading.ts @@ -1,6 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import DeveloperOptions from '@app/features/devtools/state/DeveloperOptions'; +import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher'; import * as ImageCacheUtils from '@app/features/messaging/utils/ImageCacheUtils'; import {decodeThumbHashDataURL} from '@app/features/messaging/utils/ThumbHashUtils'; import {type SyntheticEvent, useCallback, useLayoutEffect, useMemo, useRef, useState} from 'react'; @@ -123,6 +124,7 @@ export function useMediaLoading( const {enabled = true} = options; const shouldForcePlaceholder = DeveloperOptions.forceRenderPlaceholders || DeveloperOptions.forceMediaLoading; const mediaElementRef = useRef(null); + const refreshedSourceRef = useRef(null); const loadEnabled = enabled && src.length > 0 && !shouldForcePlaceholder; const sourceIdentity = useMemo(() => ({src, loadEnabled}), [loadEnabled, src]); const committedSourceIdentityRef = useRef(sourceIdentity); @@ -184,6 +186,10 @@ export function useMediaLoading( if (mediaElementRef.current !== element) return; if (!mediaElementMatchesSource(element, src)) return; ImageCacheUtils.forgetImage(src); + if (refreshedSourceRef.current !== src) { + refreshedSourceRef.current = src; + void AttachmentUrlRefresher.refresh(src, {force: true}); + } setSourceState((currentState) => { if (currentState.src === src && currentState.loadEnabled === loadEnabled && currentState.loaded) { return {...currentState, cached: false}; diff --git a/fluxer_app/src/features/messaging/hooks/useNearViewport.ts b/fluxer_app/src/features/messaging/hooks/useNearViewport.ts index ec352d879..505ce64c5 100644 --- a/fluxer_app/src/features/messaging/hooks/useNearViewport.ts +++ b/fluxer_app/src/features/messaging/hooks/useNearViewport.ts @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +import {stripAttachmentSignature} from '@app/features/messaging/utils/AttachmentCdnUrl'; import {observeIntersection} from '@app/features/platform/utils/SharedIntersectionObserver'; import {LRUCache} from 'lru-cache'; import {createContext, useCallback, useContext, useEffect, useState} from 'react'; @@ -23,6 +24,11 @@ export function resolveObserverRoot(resolve: ScrollSurfaceResolver, element: Ele return surface; } +export function resolveViewportKey(rememberKey: string | null | undefined): string | null { + if (!rememberKey) return null; + return stripAttachmentSignature(rememberKey); +} + interface UseNearViewportOptions { disabled?: boolean; rememberKey?: string | null; @@ -37,8 +43,9 @@ export function useNearViewport({ threshold = 0, }: UseNearViewportOptions = {}): {ref: (node: T | null) => void; isNearViewport: boolean} { const resolveScrollSurface = useContext(NearViewportSurfaceContext); + const viewportKey = resolveViewportKey(rememberKey); const loadImmediately = disabled || typeof IntersectionObserver === 'undefined'; - const wasRemembered = rememberKey ? rememberedViewportKeys.has(rememberKey) : false; + const wasRemembered = viewportKey ? rememberedViewportKeys.has(viewportKey) : false; const [element, setElement] = useState(null); const [isNearViewport, setIsNearViewport] = useState(loadImmediately || wasRemembered); const ref = useCallback((node: T | null) => { @@ -50,13 +57,13 @@ export function useNearViewport({ } }, [disabled]); useEffect(() => { - if (!isNearViewport || !rememberKey) return; + if (!isNearViewport || !viewportKey) return; if (disabled || typeof IntersectionObserver === 'undefined') return; - rememberedViewportKeys.set(rememberKey, true); - }, [disabled, isNearViewport, rememberKey]); + rememberedViewportKeys.set(viewportKey, true); + }, [disabled, isNearViewport, viewportKey]); useEffect(() => { if (disabled || isNearViewport || !element) return undefined; - if (rememberKey && rememberedViewportKeys.has(rememberKey)) { + if (viewportKey && rememberedViewportKeys.has(viewportKey)) { setIsNearViewport(true); return undefined; } @@ -70,14 +77,14 @@ export function useNearViewport({ (entry) => { if (!entry.isIntersecting && entry.intersectionRatio <= 0) return; stopObserving(); - if (rememberKey) { - rememberedViewportKeys.set(rememberKey, true); + if (viewportKey) { + rememberedViewportKeys.set(viewportKey, true); } setIsNearViewport(true); }, {root: resolveObserverRoot(resolveScrollSurface, element), rootMargin, threshold}, ); return stopObserving; - }, [disabled, element, isNearViewport, rememberKey, resolveScrollSurface, rootMargin, threshold]); + }, [disabled, element, isNearViewport, viewportKey, resolveScrollSurface, rootMargin, threshold]); return {ref, isNearViewport}; } diff --git a/fluxer_app/src/features/messaging/state/AttachmentUrlRefresher.ts b/fluxer_app/src/features/messaging/state/AttachmentUrlRefresher.ts new file mode 100644 index 000000000..239a4b861 --- /dev/null +++ b/fluxer_app/src/features/messaging/state/AttachmentUrlRefresher.ts @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {Endpoints} from '@app/features/app/constants/Endpoints'; +import RuntimeConfig from '@app/features/app/state/RuntimeConfig'; +import { + ATTACHMENT_URL_REFRESH_MARGIN_MS, + applyAttachmentSignature, + attachmentCacheKey, + attachmentUrlNeedsRefresh, + isAttachmentCdnUrl, + readAttachmentSignatureFields, + readAttachmentUrlSignature, + stripAttachmentSignature, +} from '@app/features/messaging/utils/AttachmentCdnUrl'; +import {resolveRetryAfterMs} from '@app/features/messaging/utils/RetryAfterUtils'; +import {http} from '@app/features/platform/transport/RestTransport'; +import {HttpError} from '@app/features/platform/types/EndpointError'; +import {Logger} from '@app/features/platform/utils/AppLogger'; +import type {RefreshAttachmentUrlsResponse} from '@fluxer/schema/src/domains/message/AttachmentSchemas'; +import {makeAutoObservable, runInAction, untracked} from 'mobx'; + +const logger = new Logger('AttachmentUrlRefresher'); + +const FLUSH_WINDOW_MS = 16; +const MAX_URLS_PER_REQUEST = 50; +const UNSIGNED_RESULT_COOLDOWN_MS = 600_000; +const TRANSPORT_ERROR_COOLDOWN_MS = 30_000; +const RATE_LIMIT_FALLBACK_COOLDOWN_MS = 10_000; + +type CacheEntry = {kind: 'signature'; fields: string; goodUntilMs: number} | {kind: 'blocked'; untilMs: number}; + +interface QueuedRequest { + url: string; + resolvers: Array<(fields: string | null) => void>; +} + +class AttachmentUrlRefresher { + entries = new Map(); + readonly queued = new Map(); + readonly inFlight = new Map>(); + flushTimer: ReturnType | null = null; + cooldownUntilMs = 0; + generation = 0; + revision = 0; + + constructor() { + makeAutoObservable( + this, + { + queued: false, + inFlight: false, + flushTimer: false, + cooldownUntilMs: false, + generation: false, + }, + {autoBind: true}, + ); + } + + fresh(url: string, options: {refreshUnsigned?: boolean} = {}): string { + if (!url || !isAttachmentCdnUrl(url, RuntimeConfig.mediaEndpoint)) return url; + const signature = readAttachmentUrlSignature(url); + const nowMs = Date.now(); + if (signature !== null && signature.expiresAtMs > nowMs + ATTACHMENT_URL_REFRESH_MARGIN_MS) return url; + const key = attachmentCacheKey(url); + if (key === null) return url; + const entry = this.entries.get(key); + if (entry?.kind === 'signature' && entry.goodUntilMs > nowMs) { + return applyAttachmentSignature(url, entry.fields); + } + if (signature === null && options.refreshUnsigned !== true) return url; + if (entry?.kind === 'blocked' && entry.untilMs > nowMs) return url; + this.enqueue(key, url, nowMs); + return url; + } + + warm(url: string): void { + if (!url || !isAttachmentCdnUrl(url, RuntimeConfig.mediaEndpoint)) return; + const nowMs = Date.now(); + if (!attachmentUrlNeedsRefresh(url, nowMs)) return; + const key = attachmentCacheKey(url); + if (key === null) return; + const entry = untracked(() => this.entries.get(key)); + if (entry?.kind === 'signature' && entry.goodUntilMs > nowMs) return; + if (entry?.kind === 'blocked' && entry.untilMs > nowMs) return; + this.enqueue(key, url, nowMs); + } + + async refresh(url: string, options: {force?: boolean} = {}): Promise { + if (!url || !isAttachmentCdnUrl(url, RuntimeConfig.mediaEndpoint)) return url; + const key = attachmentCacheKey(url); + if (key === null) return url; + const nowMs = Date.now(); + const entry = untracked(() => this.entries.get(key)); + if (entry?.kind === 'signature' && entry.goodUntilMs > nowMs) { + return applyAttachmentSignature(url, entry.fields); + } + if (options.force !== true) { + if (entry?.kind === 'blocked' && entry.untilMs > nowMs) return url; + if (!attachmentUrlNeedsRefresh(url, nowMs)) return url; + } + const fields = await this.request(key, url, nowMs); + return fields === null ? url : applyAttachmentSignature(url, fields); + } + + reset(): void { + this.generation += 1; + this.revision += 1; + this.entries = new Map(); + for (const request of this.queued.values()) { + for (const resolve of request.resolvers) resolve(null); + } + this.queued.clear(); + this.inFlight.clear(); + this.cooldownUntilMs = 0; + if (this.flushTimer !== null) { + clearTimeout(this.flushTimer); + this.flushTimer = null; + } + } + + private forgetExpired(nowMs: number): void { + for (const [key, entry] of this.entries) { + const untilMs = entry.kind === 'signature' ? entry.goodUntilMs : entry.untilMs; + if (untilMs <= nowMs) this.entries.delete(key); + } + } + + private enqueue(key: string, url: string, nowMs: number): void { + if (this.inFlight.has(key)) return; + if (nowMs < this.cooldownUntilMs) return; + if (this.queued.has(key)) return; + this.queued.set(key, {url: stripAttachmentSignature(url), resolvers: []}); + this.scheduleFlush(); + } + + private request(key: string, url: string, nowMs: number): Promise { + const existing = this.inFlight.get(key); + if (existing) return existing; + if (nowMs < this.cooldownUntilMs) return Promise.resolve(null); + const queued = this.queued.get(key) ?? {url: stripAttachmentSignature(url), resolvers: []}; + this.queued.set(key, queued); + this.scheduleFlush(); + return new Promise((resolve) => { + queued.resolvers.push(resolve); + }); + } + + private scheduleFlush(): void { + if (this.flushTimer !== null) return; + this.flushTimer = setTimeout(() => { + this.flushTimer = null; + void this.flush(); + }, FLUSH_WINDOW_MS); + } + + private async flush(): Promise { + if (this.queued.size === 0) return; + const nowMs = Date.now(); + if (nowMs < this.cooldownUntilMs) { + this.drainQueue(); + return; + } + const batch = Array.from(this.queued.entries()).slice(0, MAX_URLS_PER_REQUEST); + for (const [key] of batch) this.queued.delete(key); + if (this.queued.size > 0) this.scheduleFlush(); + const generation = this.generation; + const send = this.send(batch, generation); + for (const [key] of batch) { + this.inFlight.set( + key, + send.then((outcome) => outcome.get(key) ?? null), + ); + } + const outcome = await send; + for (const [key, request] of batch) { + if (generation === this.generation) this.inFlight.delete(key); + const fields = outcome.get(key) ?? null; + for (const resolve of request.resolvers) resolve(fields); + } + } + + private async send(batch: Array<[string, QueuedRequest]>, generation: number): Promise> { + const resolved = new Map(); + try { + const response = await http.post(Endpoints.ATTACHMENTS_REFRESH_URLS, { + body: {attachment_urls: batch.map(([, request]) => request.url)}, + }); + if (generation !== this.generation) return resolved; + const byOriginal = new Map( + (response.body?.refreshed_urls ?? []).map((entry) => [entry.original, entry.refreshed]), + ); + const nowMs = Date.now(); + const updates: Array<[string, CacheEntry]> = []; + for (const [key, request] of batch) { + const refreshed = byOriginal.get(request.url); + const fields = refreshed === undefined ? null : readAttachmentSignatureFields(refreshed); + const expiry = refreshed === undefined ? null : readAttachmentUrlSignature(refreshed); + if (fields === null || expiry === null) { + updates.push([key, {kind: 'blocked', untilMs: nowMs + UNSIGNED_RESULT_COOLDOWN_MS}]); + continue; + } + resolved.set(key, fields); + updates.push([ + key, + {kind: 'signature', fields, goodUntilMs: expiry.expiresAtMs - ATTACHMENT_URL_REFRESH_MARGIN_MS}, + ]); + } + runInAction(() => { + this.forgetExpired(nowMs); + for (const [key, entry] of updates) this.entries.set(key, entry); + this.revision += 1; + }); + return resolved; + } catch (error) { + if (generation !== this.generation) return resolved; + const nowMs = Date.now(); + if (error instanceof HttpError && error.status === 429) { + this.cooldownUntilMs = nowMs + (resolveRetryAfterMs(error) ?? RATE_LIMIT_FALLBACK_COOLDOWN_MS); + return resolved; + } + logger.warn('Failed to refresh attachment URLs', error); + runInAction(() => { + this.forgetExpired(nowMs); + for (const [key] of batch) { + this.entries.set(key, {kind: 'blocked', untilMs: nowMs + TRANSPORT_ERROR_COOLDOWN_MS}); + } + this.revision += 1; + }); + return resolved; + } + } + + private drainQueue(): void { + for (const request of this.queued.values()) { + for (const resolve of request.resolvers) resolve(null); + } + this.queued.clear(); + } +} + +export default new AttachmentUrlRefresher(); diff --git a/fluxer_app/src/features/messaging/utils/AttachmentCdnUrl.ts b/fluxer_app/src/features/messaging/utils/AttachmentCdnUrl.ts new file mode 100644 index 000000000..901a6eed0 --- /dev/null +++ b/fluxer_app/src/features/messaging/utils/AttachmentCdnUrl.ts @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +export const ATTACHMENT_URL_REFRESH_MARGIN_MS = 3_600_000; + +export interface EndpointInfo { + basePath: string; + origin: string; +} + +export interface AttachmentUrlSignature { + expiresAtMs: number; +} + +const ATTACHMENT_PATH_SEGMENT = '/attachments/'; +const WINDOW_HEX_PATTERN = /^[0-9a-f]{8}$/u; +const MAC_HEX_PATTERN = /^[0-9a-f]{64}$/u; +const DIGITS_PATTERN = /^[0-9]+$/u; +const PLUS_PATTERN = /\+/gu; +const DATA_PACKAGE_USE_CASE = 'dp'; +const DATA_PACKAGE_EXPIRY = '0'; + +export function parseEndpoint(endpoint: string): EndpointInfo | null { + if (!endpoint) return null; + try { + const parsedEndpoint = new URL(endpoint); + const basePath = + parsedEndpoint.pathname.length > 1 && parsedEndpoint.pathname.endsWith('/') + ? parsedEndpoint.pathname.slice(0, -1) + : parsedEndpoint.pathname || '/'; + return { + basePath, + origin: parsedEndpoint.origin, + }; + } catch { + return null; + } +} + +export function isUrlOnEndpoint(targetUrl: URL, endpoint: EndpointInfo): boolean { + if (targetUrl.origin !== endpoint.origin) return false; + if (endpoint.basePath === '/') return true; + return targetUrl.pathname === endpoint.basePath || targetUrl.pathname.startsWith(`${endpoint.basePath}/`); +} + +export function isAttachmentCdnUrl(url: string, mediaEndpoint: string): boolean { + if (!url.includes(ATTACHMENT_PATH_SEGMENT)) return false; + const endpoint = parseEndpoint(mediaEndpoint); + if (endpoint === null) return false; + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return false; + } + if (parsed.username !== '' || parsed.password !== '') return false; + if (parsed.origin !== endpoint.origin) return false; + const prefix = endpoint.basePath === '/' ? ATTACHMENT_PATH_SEGMENT : `${endpoint.basePath}${ATTACHMENT_PATH_SEGMENT}`; + if (!parsed.pathname.startsWith(prefix)) return false; + const segments = parsed.pathname.slice(prefix.length).split('/'); + if (segments.length < 3) return false; + const [channelId, attachmentId] = segments; + return ( + DIGITS_PATTERN.test(channelId) && + DIGITS_PATTERN.test(attachmentId) && + segments.slice(2).every((segment) => segment.length > 0) + ); +} + +export function attachmentCacheKey(url: string): string | null { + try { + const parsed = new URL(url); + return `${parsed.origin}${parsed.pathname}`; + } catch { + return null; + } +} + +interface SplitUrl { + base: string; + query: string; + fragment: string; +} + +function splitUrl(url: string): SplitUrl { + const fragmentIndex = url.indexOf('#'); + const head = fragmentIndex < 0 ? url : url.slice(0, fragmentIndex); + const fragment = fragmentIndex < 0 ? '' : url.slice(fragmentIndex); + const queryIndex = head.indexOf('?'); + if (queryIndex < 0) return {base: head, query: '', fragment}; + return {base: head.slice(0, queryIndex), query: head.slice(queryIndex + 1), fragment}; +} + +function formDecodeName(name: string): string { + try { + return decodeURIComponent(name.replace(PLUS_PATTERN, ' ')); + } catch { + return name; + } +} + +function fieldName(field: string): string { + const separator = field.indexOf('='); + return formDecodeName(separator < 0 ? field : field.slice(0, separator)); +} + +function fieldValue(field: string): string { + const separator = field.indexOf('='); + return separator < 0 ? '' : field.slice(separator + 1); +} + +type SignatureFieldName = 'ex' | 'is' | 'hm' | 'uc'; + +function isSignatureFieldName(name: string): name is SignatureFieldName { + return name === 'ex' || name === 'is' || name === 'hm' || name === 'uc'; +} + +function preservedFields(query: string): Array { + if (query.length === 0) return []; + return query.split('&').filter((field) => { + if (field.length === 0 || field === '=') return false; + return !isSignatureFieldName(fieldName(field)); + }); +} + +interface ScannedSignature { + ex: Array; + is: Array; + hm: Array; + uc: Array; +} + +function scanSignatureFields(url: string): ScannedSignature { + const found: ScannedSignature = {ex: [], is: [], hm: [], uc: []}; + const {query} = splitUrl(url); + if (query.length === 0) return found; + for (const field of query.split('&')) { + if (field.length === 0) continue; + const name = fieldName(field); + if (!isSignatureFieldName(name)) continue; + found[name].push(fieldValue(field)); + } + return found; +} + +interface AttachmentSignatureFields { + ex: string; + is: string; + hm: string; + uc: string; + expiresAtMs: number; +} + +function parseAttachmentSignature(url: string): AttachmentSignatureFields | null { + const found = scanSignatureFields(url); + if (found.ex.length !== 1 || found.is.length !== 1 || found.hm.length !== 1 || found.uc.length > 1) return null; + const ex = found.ex[0]; + const is = found.is[0]; + const hm = found.hm[0]; + if (!WINDOW_HEX_PATTERN.test(is) || !MAC_HEX_PATTERN.test(hm)) return null; + if (found.uc.length === 1) { + if (found.uc[0] !== DATA_PACKAGE_USE_CASE || ex !== DATA_PACKAGE_EXPIRY) return null; + return {ex, is, hm, uc: DATA_PACKAGE_USE_CASE, expiresAtMs: Number.POSITIVE_INFINITY}; + } + if (!WINDOW_HEX_PATTERN.test(ex)) return null; + return {ex, is, hm, uc: '', expiresAtMs: Number.parseInt(ex, 16) * 1000}; +} + +export function readAttachmentUrlSignature(url: string): AttachmentUrlSignature | null { + const signature = parseAttachmentSignature(url); + return signature === null ? null : {expiresAtMs: signature.expiresAtMs}; +} + +export function readAttachmentSignatureFields(url: string): string | null { + const signature = parseAttachmentSignature(url); + if (signature === null) return null; + const fields = `ex=${signature.ex}&is=${signature.is}&hm=${signature.hm}`; + return signature.uc === '' ? fields : `${fields}&uc=${signature.uc}`; +} + +export function stripAttachmentSignature(url: string): string { + const {base, query, fragment} = splitUrl(url); + const preserved = preservedFields(query); + if (preserved.length === 0) return `${base}${fragment}`; + return `${base}?${preserved.join('&')}${fragment}`; +} + +export function applyAttachmentSignature(url: string, signatureFields: string): string { + const {base, query, fragment} = splitUrl(url); + const preserved = preservedFields(query); + if (preserved.length === 0) return `${base}?${signatureFields}${fragment}`; + return `${base}?${signatureFields}&${preserved.join('&')}${fragment}`; +} + +export function attachmentUrlNeedsRefresh(url: string, nowMs: number): boolean { + const signature = readAttachmentUrlSignature(url); + if (signature === null) return true; + return signature.expiresAtMs <= nowMs + ATTACHMENT_URL_REFRESH_MARGIN_MS; +} diff --git a/fluxer_app/src/features/messaging/utils/FileDownloadUtils.ts b/fluxer_app/src/features/messaging/utils/FileDownloadUtils.ts index d803e920c..efdcaccf0 100644 --- a/fluxer_app/src/features/messaging/utils/FileDownloadUtils.ts +++ b/fluxer_app/src/features/messaging/utils/FileDownloadUtils.ts @@ -1,6 +1,8 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import RuntimeConfig from '@app/features/app/state/RuntimeConfig'; +import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher'; +import {isUrlOnEndpoint, parseEndpoint} from '@app/features/messaging/utils/AttachmentCdnUrl'; import {isMobileOrTabletUserAgent} from '@app/features/platform/notifications/NotificationAlertOptions'; import {supportsShowSaveFilePicker} from '@app/features/platform/types/Browser'; import {Logger} from '@app/features/platform/utils/AppLogger'; @@ -10,34 +12,6 @@ const logger = new Logger('FileDownloadUtils'); type MediaType = 'image' | 'gif' | 'video' | 'audio' | 'file'; -interface EndpointInfo { - basePath: string; - origin: string; -} - -function parseEndpoint(endpoint: string): EndpointInfo | null { - if (!endpoint) return null; - try { - const parsedEndpoint = new URL(endpoint); - const basePath = - parsedEndpoint.pathname.length > 1 && parsedEndpoint.pathname.endsWith('/') - ? parsedEndpoint.pathname.slice(0, -1) - : parsedEndpoint.pathname || '/'; - return { - basePath, - origin: parsedEndpoint.origin, - }; - } catch { - return null; - } -} - -function isUrlOnEndpoint(targetUrl: URL, endpoint: EndpointInfo): boolean { - if (targetUrl.origin !== endpoint.origin) return false; - if (endpoint.basePath === '/') return true; - return targetUrl.pathname === endpoint.basePath || targetUrl.pathname.startsWith(`${endpoint.basePath}/`); -} - function appendMediaProxyDownloadParam(src: string): string { let parsedSrc: URL; try { @@ -221,23 +195,21 @@ function downloadViaAnchor(src: string, suggestedName: string, options?: {append export async function downloadFile(src: string, type: MediaType, providedFilename?: string): Promise { if (!src) return; + const suggestedName = deriveSuggestedName(src, type, providedFilename); + const target = await AttachmentUrlRefresher.refresh(src); if (isElectron()) { try { - const outcome = await downloadWithNative({ - url: src, - suggestedName: deriveSuggestedName(src, type, providedFilename), - }); + const outcome = await downloadWithNative({url: target, suggestedName}); if (outcome !== 'unavailable') return; } catch (error) { logger.warn('Native download failed', error); return; } } - const suggestedName = deriveSuggestedName(src, type, providedFilename); - if (await downloadViaFileSystemAccess(src, suggestedName, type)) return; - if (await downloadViaFetchBlob(src, suggestedName)) return; - if (downloadViaAnchor(src, suggestedName)) return; - await openExternalUrl(appendMediaProxyDownloadParam(src)); + if (await downloadViaFileSystemAccess(target, suggestedName, type)) return; + if (await downloadViaFetchBlob(target, suggestedName)) return; + if (downloadViaAnchor(target, suggestedName)) return; + await openExternalUrl(appendMediaProxyDownloadParam(target)); } export function createDownloadHandler(src: string, type: MediaType, providedFilename?: string) { diff --git a/fluxer_app/src/features/messaging/utils/SpoilerUtils.ts b/fluxer_app/src/features/messaging/utils/SpoilerUtils.ts index a7bc6cc39..aa3df2337 100644 --- a/fluxer_app/src/features/messaging/utils/SpoilerUtils.ts +++ b/fluxer_app/src/features/messaging/utils/SpoilerUtils.ts @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +import {stripAttachmentSignature} from '@app/features/messaging/utils/AttachmentCdnUrl'; import Permission from '@app/features/permissions/state/Permission'; import UserSettings from '@app/features/user/state/UserSettings'; import {Permissions} from '@fluxer/constants/src/ChannelConstants'; @@ -49,7 +50,7 @@ export function canonicalizeMediaUrl(url: string | null | undefined): string | n if (youtubeId) { return `youtube:${youtubeId}`; } - return parsed.href.replace(/\/$/, ''); + return stripAttachmentSignature(parsed.href).replace(/\/$/, ''); } catch { return null; } diff --git a/fluxer_app/src/features/ui/action_menu/items/MediaMenuData.tsx b/fluxer_app/src/features/ui/action_menu/items/MediaMenuData.tsx index 4458266c3..88f451c5d 100644 --- a/fluxer_app/src/features/ui/action_menu/items/MediaMenuData.tsx +++ b/fluxer_app/src/features/ui/action_menu/items/MediaMenuData.tsx @@ -18,6 +18,7 @@ import { } from '@app/features/i18n/utils/CommonMessageDescriptors'; import {EditAltTextModal} from '@app/features/messaging/components/modals/EditAltTextModal'; import type {Message} from '@app/features/messaging/models/MessagingMessage'; +import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher'; import {createDownloadHandler} from '@app/features/messaging/utils/FileDownloadUtils'; import {buildMediaProxyURL, stripMediaProxyParams} from '@app/features/messaging/utils/MediaProxyUtils'; import Permission from '@app/features/permissions/state/Permission'; @@ -429,17 +430,18 @@ export async function copyMediaToClipboard({ ); return; } + const freshSrc = await AttachmentUrlRefresher.refresh(originalSrc); if (type === 'file') { - await TextCopyCommands.copy(i18n, originalSrc, true); + await TextCopyCommands.copy(i18n, freshSrc, true); ToastCommands.createToast({type: 'success', children: i18n._(LINK_COPIED_TO_CLIPBOARD_DESCRIPTOR)}); return; } - const baseProxyURL = proxyURL ? stripMediaProxyParams(proxyURL) : null; + const baseProxyURL = proxyURL ? await AttachmentUrlRefresher.refresh(stripMediaProxyParams(proxyURL)) : null; const clipboardFileMediaType = getClipboardFileMediaType(type); if (clipboardFileMediaType) { const electronApi = getElectronAPI(); if (!electronApi?.clipboardWriteFile) { - await TextCopyCommands.copy(i18n, originalSrc, true); + await TextCopyCommands.copy(i18n, freshSrc, true); ToastCommands.createToast({type: 'success', children: i18n._(URL_COPIED_TO_CLIPBOARD_DESCRIPTOR)}); return; } @@ -451,7 +453,7 @@ export async function copyMediaToClipboard({ timeout: 0, }); const result = await electronApi.clipboardWriteFile({ - url: baseProxyURL || originalSrc, + url: baseProxyURL || freshSrc, suggestedName: defaultName, mediaType: clipboardFileMediaType, }); @@ -470,7 +472,7 @@ export async function copyMediaToClipboard({ const urlsToTry: Array = []; if (baseProxyURL) urlsToTry.push(buildMediaProxyURL(baseProxyURL, {format: 'png'})); if (baseProxyURL) urlsToTry.push(baseProxyURL); - urlsToTry.push(originalSrc); + urlsToTry.push(freshSrc); let toastId: string | null = null; try { toastId = ToastCommands.createToast({ @@ -494,7 +496,7 @@ export async function copyMediaToClipboard({ } catch (error) { logger.error('Failed to copy image to clipboard:', error); if (toastId) ToastCommands.destroyToast(toastId); - await TextCopyCommands.copy(i18n, originalSrc, true); + await TextCopyCommands.copy(i18n, freshSrc, true); ToastCommands.createToast({ type: 'success', children: i18n._(URL_COPIED_TO_CLIPBOARD_DESCRIPTOR), @@ -511,7 +513,7 @@ export async function copyMediaLinkToClipboard({i18n, originalSrc}: {i18n: I18n; ); return; } - await TextCopyCommands.copy(i18n, originalSrc, true); + await TextCopyCommands.copy(i18n, await AttachmentUrlRefresher.refresh(originalSrc), true); ToastCommands.createToast({ type: 'success', children: i18n._(LINK_COPIED_TO_CLIPBOARD_DESCRIPTOR), diff --git a/fluxer_app/src/features/ui/utils/NativeUtils.ts b/fluxer_app/src/features/ui/utils/NativeUtils.ts index b4e1d61c8..c1be6bc6a 100644 --- a/fluxer_app/src/features/ui/utils/NativeUtils.ts +++ b/fluxer_app/src/features/ui/utils/NativeUtils.ts @@ -196,20 +196,26 @@ const getSafeExternalUrl = (href: string | null): string | null => { } }; +async function refreshAttachmentUrl(url: string): Promise { + const {default: AttachmentUrlRefresher} = await import('@app/features/messaging/state/AttachmentUrlRefresher'); + return AttachmentUrlRefresher.refresh(url); +} + export async function openExternalUrl(url: string, target: string = '_blank') { const safeUrl = getSafeExternalUrl(url); if (!safeUrl) return; + const refreshedUrl = await refreshAttachmentUrl(safeUrl); const electronApi = getElectronAPI(); if (electronApi) { try { - await electronApi.openExternal(safeUrl); + await electronApi.openExternal(refreshedUrl); return; } catch (error) { logger.error(' Failed to open external URL via Electron', error); return; } } - window.open(safeUrl, target, 'noopener,noreferrer'); + window.open(refreshedUrl, target, 'noopener,noreferrer'); } interface ExternalLinkClickEvent { diff --git a/fluxer_app/src/features/voice/components/media_player/components/AudioPlayer.tsx b/fluxer_app/src/features/voice/components/media_player/components/AudioPlayer.tsx index de8713082..5ff507785 100644 --- a/fluxer_app/src/features/voice/components/media_player/components/AudioPlayer.tsx +++ b/fluxer_app/src/features/voice/components/media_player/components/AudioPlayer.tsx @@ -1,6 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import {PAUSE_DESCRIPTOR, PLAY_DESCRIPTOR} from '@app/features/i18n/utils/CommonMessageDescriptors'; +import {useAttachmentRefreshOnError} from '@app/features/messaging/hooks/useAttachmentRefreshOnError'; import FocusRing from '@app/features/ui/focus_ring/FocusRing'; import {Tooltip} from '@app/features/ui/tooltip/Tooltip'; import styles from '@app/features/voice/components/media_player/AudioPlayer.module.css'; @@ -77,6 +78,7 @@ export function AudioPlayer({ }); const {volume, isMuted, setVolume, toggleMute} = useMediaVolume({mediaRef}); const {escalateToMetadata, sourceAttribute, preloadAttribute} = useMetadataPreload(src, hasStarted); + const handleMediaError = useAttachmentRefreshOnError(src); useEffect(() => { if (hasStarted && pendingPlayRef.current) { const timer = setTimeout(() => { @@ -210,6 +212,7 @@ export function AudioPlayer({ ref={mediaRef as React.RefObject} src={sourceAttribute} preload={preloadAttribute} + onError={handleMediaError} data-flx="voice.media-player.audio-player.audio" /> {title && ( diff --git a/fluxer_app/src/features/voice/components/media_player/components/InlineAudioPlayer.tsx b/fluxer_app/src/features/voice/components/media_player/components/InlineAudioPlayer.tsx index 3fa6137b6..146519950 100644 --- a/fluxer_app/src/features/voice/components/media_player/components/InlineAudioPlayer.tsx +++ b/fluxer_app/src/features/voice/components/media_player/components/InlineAudioPlayer.tsx @@ -8,6 +8,7 @@ import { PLAY_DESCRIPTOR, REMOVE_FROM_FAVORITES_DESCRIPTOR, } from '@app/features/i18n/utils/CommonMessageDescriptors'; +import {useAttachmentRefreshOnError} from '@app/features/messaging/hooks/useAttachmentRefreshOnError'; import {formatFileSize} from '@app/features/messaging/utils/FileUtils'; import {remFromPx} from '@app/features/theme/layout/RemFromPx'; import FocusRing from '@app/features/ui/focus_ring/FocusRing'; @@ -111,6 +112,7 @@ export function InlineAudioPlayer({ mediaRef, }); const {escalateToMetadata, sourceAttribute, preloadAttribute} = useMetadataPreload(src, hasStarted); + const handleMediaError = useAttachmentRefreshOnError(src); const displayDuration = duration > 0 ? duration : (initialDuration ?? 0); useLayoutEffect(() => { const media = mediaRef.current; @@ -205,6 +207,7 @@ export function InlineAudioPlayer({ ref={mediaRef as React.RefObject} src={sourceAttribute} preload={preloadAttribute} + onError={handleMediaError} data-flx="voice.media-player.inline-audio-player.audio" > @@ -332,6 +335,7 @@ export function InlineAudioPlayer({ ref={mediaRef as React.RefObject} src={sourceAttribute} preload={preloadAttribute} + onError={handleMediaError} data-flx="voice.media-player.inline-audio-player.audio--2" > diff --git a/fluxer_common/Cargo.toml b/fluxer_common/Cargo.toml index eb0f9f7e5..eacbcd6ae 100644 --- a/fluxer_common/Cargo.toml +++ b/fluxer_common/Cargo.toml @@ -17,6 +17,7 @@ tempfile = "3.27.0" time = { version = "0.3.55", features = ["formatting", "macros", "parsing"] } tracing = "0.1.44" base64 = "0.23" +hex = "0.4.3" hmac = "0.13.0" sha2 = "0.11.0" thiserror = "2" diff --git a/fluxer_common/src/attachment_url_signature.rs b/fluxer_common/src/attachment_url_signature.rs new file mode 100644 index 000000000..6d5b3e092 --- /dev/null +++ b/fluxer_common/src/attachment_url_signature.rs @@ -0,0 +1,1127 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use crate::external_media_path::percent_decode; +use hmac::{KeyInit, Mac}; + +pub const ATTACHMENT_URL_TTL_SECS: u64 = 86_400; +pub const ATTACHMENT_URL_BUCKET_SECS: u64 = 43_200; +pub const SIGNATURE_PARAMETER_NAMES: [&str; 4] = ["ex", "is", "hm", "uc"]; + +const DOMAIN: &str = "fluxer-attachment-url-v1"; +const DATA_PACKAGE_USAGE: &str = "dp"; +const DATA_PACKAGE_EXPIRES: &str = "0"; +const WINDOW_HEX_LEN: usize = 8; +const WINDOW_MAX_SECS: u64 = 0xffff_ffff; +const SIGNATURE_HEX_LEN: usize = 64; + +type HmacSha256 = hmac::Hmac; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Verdict { + Valid, + Missing, + Malformed, + Mismatch, + Expired, +} + +impl Verdict { + pub fn label(self) -> &'static str { + match self { + Self::Valid => "valid", + Self::Missing => "missing", + Self::Malformed => "malformed", + Self::Mismatch => "mismatch", + Self::Expired => "expired", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct Verification { + pub verdict: Verdict, + pub remaining_secs: Option, +} + +impl Verification { + fn without_expiry(verdict: Verdict) -> Self { + Self { + verdict, + remaining_secs: None, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum UrlKind { + Ordinary, + DataPackage, +} + +impl UrlKind { + pub fn usage(self) -> &'static str { + match self { + Self::Ordinary => "", + Self::DataPackage => DATA_PACKAGE_USAGE, + } + } +} + +pub fn issue_window(anchor_secs: u64, now_secs: u64) -> (u64, u64) { + let elapsed = now_secs.saturating_sub(anchor_secs); + let issued = anchor_secs + .saturating_add((elapsed / ATTACHMENT_URL_BUCKET_SECS) * ATTACHMENT_URL_BUCKET_SECS); + (issued, issued.saturating_add(ATTACHMENT_URL_TTL_SECS)) +} + +pub fn canonical_input(storage_key: &str, ex: u64, is: u64, kind: UrlKind) -> String { + format!( + "{DOMAIN}\n{ex:08x}\n{is:08x}\n{}\n{storage_key}", + kind.usage() + ) +} + +pub fn sign(storage_key: &str, ex: u64, is: u64, kind: UrlKind, secret: &[u8]) -> String { + let input = canonical_input(storage_key, ex, is, kind); + hex::encode(keyed_mac(secret, &input).finalize().into_bytes()) +} + +pub fn is_signature_parameter_name(name: &str) -> bool { + let decoded = percent_decode(name, true); + SIGNATURE_PARAMETER_NAMES + .iter() + .any(|candidate| candidate.as_bytes() == decoded.as_slice()) +} + +pub fn strip_signature(url: &str) -> String { + let (head, fragment) = split_fragment(url); + let (base, query) = split_query(head); + let preserved = preserved_fields(query); + if preserved.is_empty() { + return format!("{base}{fragment}"); + } + format!("{base}?{}{fragment}", preserved.join("&")) +} + +pub fn with_signature( + url: &str, + storage_key: &str, + anchor_secs: u64, + now_secs: u64, + secret: &[u8], +) -> String { + let (issued, expires) = issue_window(anchor_secs, now_secs); + if expires > WINDOW_MAX_SECS { + return url.to_owned(); + } + let signature = sign(storage_key, expires, issued, UrlKind::Ordinary, secret); + replace_signature( + url, + &format!("ex={expires:08x}&is={issued:08x}&hm={signature}"), + ) +} + +pub fn with_data_package_signature( + url: &str, + storage_key: &str, + anchor_secs: u64, + now_secs: u64, + secret: &[u8], +) -> String { + let (issued, expires) = issue_window(anchor_secs, now_secs); + if expires > WINDOW_MAX_SECS { + return url.to_owned(); + } + let signature = sign(storage_key, 0, issued, UrlKind::DataPackage, secret); + replace_signature( + url, + &format!( + "ex={DATA_PACKAGE_EXPIRES}&is={issued:08x}&hm={signature}&uc={DATA_PACKAGE_USAGE}" + ), + ) +} + +pub fn decode_key(path: &str) -> Option { + let decoded = percent_decode(path.trim_start_matches('/'), false); + std::str::from_utf8(&decoded).ok().map(ToOwned::to_owned) +} + +pub fn verify( + storage_key: &str, + raw_query: Option<&str>, + secrets: &[&[u8]], + now_secs: u64, +) -> Verification { + let Some(raw_query) = raw_query else { + return Verification::without_expiry(Verdict::Missing); + }; + let Some(fields) = signature_fields(raw_query) else { + return Verification::without_expiry(Verdict::Malformed); + }; + let (expires, issued, signature) = match ( + fields.expires, + fields.issued, + fields.signature, + fields.usage, + ) { + (None, None, None, None) => return Verification::without_expiry(Verdict::Missing), + (Some(expires), Some(issued), Some(signature), _) => (expires, issued, signature), + _ => return Verification::without_expiry(Verdict::Malformed), + }; + let kind = match fields.usage { + None => UrlKind::Ordinary, + Some(DATA_PACKAGE_USAGE) => UrlKind::DataPackage, + Some(_) => return Verification::without_expiry(Verdict::Malformed), + }; + let expires = match kind { + UrlKind::Ordinary => parse_window(expires), + UrlKind::DataPackage => (expires == DATA_PACKAGE_EXPIRES).then_some(0), + }; + let (Some(expires), Some(issued)) = (expires, parse_window(issued)) else { + return Verification::without_expiry(Verdict::Malformed); + }; + if !is_lowercase_hex(signature, SIGNATURE_HEX_LEN) { + return Verification::without_expiry(Verdict::Malformed); + } + let Ok(provided) = hex::decode(signature) else { + return Verification::without_expiry(Verdict::Malformed); + }; + if kind == UrlKind::Ordinary && issued > expires { + return Verification::without_expiry(Verdict::Malformed); + } + let input = canonical_input(storage_key, expires, issued, kind); + let matched = secrets + .iter() + .any(|secret| keyed_mac(secret, &input).verify_slice(&provided).is_ok()); + if !matched { + return Verification::without_expiry(Verdict::Mismatch); + } + if kind == UrlKind::DataPackage { + return Verification::without_expiry(Verdict::Valid); + } + if now_secs >= expires { + return Verification::without_expiry(Verdict::Expired); + } + Verification { + verdict: Verdict::Valid, + remaining_secs: Some(expires - now_secs), + } +} + +#[derive(Default)] +struct SignatureFields<'a> { + expires: Option<&'a str>, + issued: Option<&'a str>, + signature: Option<&'a str>, + usage: Option<&'a str>, +} + +fn signature_fields(raw_query: &str) -> Option> { + let mut fields = SignatureFields::default(); + for field in raw_query.split('&').filter(|field| !field.is_empty()) { + let (name, value) = field.split_once('=').unwrap_or((field, "")); + let slot = match percent_decode(name, true).as_slice() { + b"ex" => &mut fields.expires, + b"is" => &mut fields.issued, + b"hm" => &mut fields.signature, + b"uc" => &mut fields.usage, + _ => continue, + }; + if slot.replace(value).is_some() { + return None; + } + } + Some(fields) +} + +fn keyed_mac(secret: &[u8], input: &str) -> HmacSha256 { + let mut mac = HmacSha256::new_from_slice(secret).expect("hmac accepts any key length"); + mac.update(input.as_bytes()); + mac +} + +fn parse_window(value: &str) -> Option { + if !is_lowercase_hex(value, WINDOW_HEX_LEN) { + return None; + } + u64::from_str_radix(value, 16).ok() +} + +fn is_lowercase_hex(value: &str, len: usize) -> bool { + value.len() == len + && value + .bytes() + .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')) +} + +fn replace_signature(url: &str, signature_fields: &str) -> String { + let (head, fragment) = split_fragment(url); + let (base, query) = split_query(head); + let preserved = preserved_fields(query); + if preserved.is_empty() { + return format!("{base}?{signature_fields}{fragment}"); + } + format!( + "{base}?{signature_fields}&{}{fragment}", + preserved.join("&") + ) +} + +fn split_fragment(url: &str) -> (&str, &str) { + match url.find('#') { + Some(index) => (&url[..index], &url[index..]), + None => (url, ""), + } +} + +fn split_query(head: &str) -> (&str, &str) { + head.split_once('?').unwrap_or((head, "")) +} + +fn preserved_fields(query: &str) -> Vec<&str> { + query + .split('&') + .filter(|field| { + if field.is_empty() || *field == "=" { + return false; + } + let (name, _) = field.split_once('=').unwrap_or((*field, "")); + !is_signature_parameter_name(name) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use base64::prelude::*; + use serde_json::Value; + + const VECTORS: &str = include_str!("testdata/attachment_url_signature_vectors.json"); + const KEY: &str = "attachments/1544725486800732163/1544971349200470016/cat.gif"; + const ANCHOR: u64 = 1_788_420_273; + + type Signer = fn(&str, &str, u64, u64, &[u8]) -> String; + + fn fixture() -> Value { + serde_json::from_str(VECTORS).expect("the signature vectors parse as json") + } + + fn fixture_secrets(fixture: &Value) -> Vec> { + fixture["secrets_base64"] + .as_array() + .expect("the fixture carries a secret list") + .iter() + .map(|entry| { + BASE64_STANDARD + .decode(entry.as_str().expect("a secret is a string")) + .expect("a secret is standard base64") + }) + .collect() + } + + fn cases<'a>(fixture: &'a Value, name: &str) -> &'a [Value] { + fixture[name] + .as_array() + .expect("the fixture carries the case list") + .as_slice() + } + + fn text<'a>(case: &'a Value, field: &str) -> &'a str { + case[field] + .as_str() + .unwrap_or_else(|| panic!("case carries {field}")) + } + + fn number(case: &Value, field: &str) -> u64 { + case[field] + .as_u64() + .unwrap_or_else(|| panic!("case carries {field}")) + } + + fn secret_bytes() -> Vec { + (0u8..32).collect() + } + + fn other_secret_bytes() -> Vec { + (32u8..64).collect() + } + + fn verdict_of( + storage_key: &str, + raw_query: Option<&str>, + secrets: &[&[u8]], + now_secs: u64, + ) -> Verdict { + verify(storage_key, raw_query, secrets, now_secs).verdict + } + + #[test] + fn a_valid_ordinary_signature_reports_the_seconds_left_and_a_data_package_reports_none() { + let secret = secret_bytes(); + let now = ANCHOR + 10; + let (issued, expires) = issue_window(ANCHOR, now); + let query = ordinary_query(KEY, issued, expires, &secret); + for probe in [issued, now, expires - 1] { + assert_eq!( + Some(expires - probe), + verify(KEY, Some(&query), &[&secret], probe).remaining_secs, + "{probe}" + ); + } + for refused in [ + verify(KEY, Some(&query), &[&secret], expires), + verify(KEY, None, &[&secret], now), + verify(KEY, Some(&query), &[&other_secret_bytes()], now), + ] { + assert_ne!(Verdict::Valid, refused.verdict); + assert_eq!(None, refused.remaining_secs); + } + let package = verify( + KEY, + Some(&data_package_query(KEY, issued, &secret)), + &[&secret], + now, + ); + assert_eq!(Verdict::Valid, package.verdict); + assert_eq!(None, package.remaining_secs); + } + + fn ordinary_query(key: &str, issued: u64, expires: u64, secret: &[u8]) -> String { + format!( + "ex={expires:08x}&is={issued:08x}&hm={}", + sign(key, expires, issued, UrlKind::Ordinary, secret) + ) + } + + fn data_package_query(key: &str, issued: u64, secret: &[u8]) -> String { + format!( + "ex=0&is={issued:08x}&hm={}&uc=dp", + sign(key, 0, issued, UrlKind::DataPackage, secret) + ) + } + + #[test] + fn the_fixture_declares_the_constants_the_code_uses() { + let fixture = fixture(); + assert_eq!(3, fixture["version"].as_u64().expect("version")); + assert_eq!( + ATTACHMENT_URL_TTL_SECS, + fixture["ttl_secs"].as_u64().expect("ttl") + ); + assert_eq!( + ATTACHMENT_URL_BUCKET_SECS, + fixture["bucket_secs"].as_u64().expect("bucket") + ); + assert_eq!( + 1_420_070_400_000, + fixture["fluxer_epoch_ms"].as_u64().expect("epoch") + ); + } + + #[test] + fn every_sign_vector_reproduces_byte_for_byte() { + let fixture = fixture(); + let secrets = fixture_secrets(&fixture); + let parsed = cases(&fixture, "sign"); + let mut run = 0; + let mut data_packages = 0; + for case in parsed { + let name = text(case, "name"); + let storage_key = text(case, "storage_key"); + let anchor = number(case, "anchor"); + let now = number(case, "now"); + let (issued, expires) = issue_window(anchor, now); + assert_eq!(format!("{issued:08x}"), text(case, "is"), "{name} issued"); + let (kind, signed_expires, signer): (_, _, Signer) = match text(case, "uc") { + "" => { + assert_eq!(format!("{expires:08x}"), text(case, "ex"), "{name} expires"); + (UrlKind::Ordinary, expires, with_signature) + } + "dp" => { + assert_eq!("0", text(case, "ex"), "{name} expires"); + data_packages += 1; + (UrlKind::DataPackage, 0, with_data_package_signature) + } + other => panic!("{name} carries an unknown uc {other}"), + }; + assert_eq!( + text(case, "signature_input"), + canonical_input(storage_key, signed_expires, issued, kind), + "{name} input" + ); + assert_eq!( + text(case, "hm"), + sign(storage_key, signed_expires, issued, kind, &secrets[0]), + "{name} signature" + ); + assert_eq!( + text(case, "signed"), + signer(text(case, "url"), storage_key, anchor, now, &secrets[0]), + "{name} signed url" + ); + assert_eq!( + text(case, "url"), + strip_signature(text(case, "signed")), + "{name} strips back" + ); + let signed = text(case, "signed"); + assert!(!signed.ends_with('&'), "{name} trailing separator"); + assert!(!signed.contains("&&"), "{name} doubled separator"); + run += 1; + } + assert_eq!(parsed.len(), run); + assert!(run >= 8); + assert!(data_packages >= 3); + } + + #[test] + fn every_strip_vector_reproduces_byte_for_byte() { + let fixture = fixture(); + let parsed = cases(&fixture, "strip"); + let mut run = 0; + for case in parsed { + assert_eq!( + text(case, "stripped"), + strip_signature(text(case, "url")), + "{}", + text(case, "name") + ); + run += 1; + } + assert_eq!(parsed.len(), run); + assert!(run >= 10); + } + + #[test] + fn every_verify_vector_reaches_its_declared_verdict() { + let fixture = fixture(); + let secrets = fixture_secrets(&fixture); + let parsed = cases(&fixture, "verify"); + let mut run = 0; + let mut rotations = 0; + let mut trailing_separators = 0; + let mut doubled_separators = 0; + for case in parsed { + let name = text(case, "name"); + let held: Vec<&[u8]> = case["verifier_secret_indices"] + .as_array() + .expect("case names the secrets the verifier holds") + .iter() + .map(|index| { + let index = index.as_u64().expect("a secret index is a number") as usize; + secrets[index].as_slice() + }) + .collect(); + assert!(!held.is_empty(), "{name}"); + if held.len() > 1 { + rotations += 1; + } + let now = number(case, "now"); + let query = case["query"].as_str(); + let verdict = match decode_key(text(case, "path")) { + Some(key) => verdict_of(&key, query, &held, now), + None => Verdict::Malformed, + }; + assert_eq!(text(case, "verdict"), verdict.label(), "{name}"); + if verdict == Verdict::Valid { + if query.is_some_and(|query| query.ends_with('&')) { + trailing_separators += 1; + } + if query.is_some_and(|query| query.contains("&&")) { + doubled_separators += 1; + } + } + run += 1; + } + assert_eq!(parsed.len(), run); + assert!(run >= 50); + assert!(rotations >= 3); + assert!(trailing_separators >= 2); + assert!(doubled_separators >= 2); + } + + #[test] + fn a_freshly_signed_url_verifies_at_every_second_of_its_life() { + let secret = secret_bytes(); + let now = ANCHOR + 10; + let signed = with_signature("https://media.test/x.gif", KEY, ANCHOR, now, &secret); + let query = signed + .split_once('?') + .expect("a signed url carries a query") + .1; + let (issued, expires) = issue_window(ANCHOR, now); + assert!(issued <= now && now < expires); + for probe in [issued, now, expires - 1] { + assert_eq!( + Verdict::Valid, + verdict_of(KEY, Some(query), &[&secret], probe), + "{probe}" + ); + } + for probe in [expires, expires + 1] { + assert_eq!( + Verdict::Expired, + verdict_of(KEY, Some(query), &[&secret], probe), + "{probe}" + ); + } + } + + #[test] + fn a_data_package_url_never_expires() { + let secret = secret_bytes(); + let now = ANCHOR + 3 * ATTACHMENT_URL_BUCKET_SECS + 7; + let signed = + with_data_package_signature("https://media.test/x.gif", KEY, ANCHOR, now, &secret); + let query = signed + .split_once('?') + .expect("a signed url carries a query") + .1; + let (issued, expires) = issue_window(ANCHOR, now); + assert!(query.starts_with(&format!("ex=0&is={issued:08x}&hm="))); + assert!(query.ends_with("&uc=dp")); + for probe in [ + 0, + issued, + now, + expires, + expires + 1, + u64::from(u32::MAX), + u64::MAX, + ] { + assert_eq!( + Verdict::Valid, + verdict_of(KEY, Some(query), &[&secret], probe), + "{probe}" + ); + } + assert_eq!( + Verdict::Mismatch, + verdict_of(KEY, Some(query), &[&other_secret_bytes()], now) + ); + } + + #[test] + fn the_issue_window_stays_on_the_anchor_grid() { + for offset in [0, 1, ATTACHMENT_URL_BUCKET_SECS - 1] { + for bucket in 0..8u64 { + let now = ANCHOR + bucket * ATTACHMENT_URL_BUCKET_SECS + offset; + let (issued, expires) = issue_window(ANCHOR, now); + assert_eq!(ANCHOR + bucket * ATTACHMENT_URL_BUCKET_SECS, issued); + assert_eq!(issued + ATTACHMENT_URL_TTL_SECS, expires); + assert!(expires - now >= ATTACHMENT_URL_TTL_SECS - ATTACHMENT_URL_BUCKET_SECS); + } + } + } + + #[test] + fn a_clock_behind_the_anchor_issues_the_first_bucket() { + let (issued, expires) = issue_window(ANCHOR, ANCHOR - 5_000); + assert_eq!(ANCHOR, issued); + assert_eq!(ANCHOR + ATTACHMENT_URL_TTL_SECS, expires); + } + + #[test] + fn a_window_beyond_eight_hex_digits_leaves_the_url_unchanged() { + let secret = secret_bytes(); + let signers: [Signer; 2] = [with_signature, with_data_package_signature]; + let last_fitting_anchor = WINDOW_MAX_SECS - ATTACHMENT_URL_TTL_SECS; + for signer in signers { + for url in [ + "https://media.test/attachments/1/2/cat.gif", + "https://media.test/attachments/1/2/cat.gif?width=64&ex=1&&is=2&hm=3&uc=4#top", + ] { + for anchor in [last_fitting_anchor + 1, WINDOW_MAX_SECS, u64::MAX] { + assert_eq!(url, signer(url, KEY, anchor, 0, &secret), "{anchor} {url}"); + } + assert_eq!( + url, + signer( + url, + KEY, + last_fitting_anchor, + last_fitting_anchor + ATTACHMENT_URL_BUCKET_SECS, + &secret + ), + "{url}" + ); + let signed = signer(url, KEY, last_fitting_anchor, 0, &secret); + assert_ne!(url, signed, "{url}"); + assert!(signed.contains(&format!("is={last_fitting_anchor:08x}&"))); + } + } + let signed = with_signature( + "https://media.test/x.gif", + KEY, + last_fitting_anchor, + 0, + &secret, + ); + assert!(signed.contains("ex=ffffffff&"), "{signed}"); + } + + #[test] + fn verification_tries_every_configured_secret() { + let first = secret_bytes(); + let second = other_secret_bytes(); + let now = ANCHOR; + let (issued, expires) = issue_window(ANCHOR, now); + for query in [ + ordinary_query(KEY, issued, expires, &second), + data_package_query(KEY, issued, &second), + ] { + assert_eq!( + Verdict::Mismatch, + verdict_of(KEY, Some(&query), &[&first], now) + ); + assert_eq!( + Verdict::Valid, + verdict_of(KEY, Some(&query), &[&first, &second], now) + ); + assert_eq!( + Verdict::Valid, + verdict_of(KEY, Some(&query), &[&second, &first], now) + ); + assert_eq!(Verdict::Mismatch, verdict_of(KEY, Some(&query), &[], now)); + } + } + + #[test] + fn a_signature_never_carries_across_keys_or_windows() { + let secret = secret_bytes(); + let now = ANCHOR; + let (issued, expires) = issue_window(ANCHOR, now); + let signature = sign(KEY, expires, issued, UrlKind::Ordinary, &secret); + let query = ordinary_query(KEY, issued, expires, &secret); + let package = data_package_query(KEY, issued, &secret); + assert_eq!( + Verdict::Valid, + verdict_of(KEY, Some(&query), &[&secret], now) + ); + assert_eq!( + Verdict::Valid, + verdict_of(KEY, Some(&package), &[&secret], now) + ); + for other in [ + "attachments/1544725486800732163/1544971349200470016/cat.gi", + "attachments/1544725486800732163/1544971349200470017/cat.gif", + "attachments/1544725486800732163/1544971349200470016/cat.gif ", + "attachments/1544725486800732163/1544971349200470016/sub/cat.gif", + ] { + for query in [&query, &package] { + assert_eq!( + Verdict::Mismatch, + verdict_of(other, Some(query), &[&secret], now), + "{other} {query}" + ); + } + } + let shifted = format!( + "ex={:08x}&is={issued:08x}&hm={signature}", + expires + ATTACHMENT_URL_BUCKET_SECS + ); + assert_eq!( + Verdict::Mismatch, + verdict_of(KEY, Some(&shifted), &[&secret], now) + ); + let shifted_package = format!( + "ex=0&is={:08x}&hm={}&uc=dp", + issued + 1, + sign(KEY, 0, issued, UrlKind::DataPackage, &secret) + ); + assert_eq!( + Verdict::Mismatch, + verdict_of(KEY, Some(&shifted_package), &[&secret], now) + ); + } + + #[test] + fn a_signature_is_bound_to_its_usage() { + let secret = secret_bytes(); + let now = ANCHOR; + let (issued, expires) = issue_window(ANCHOR, now); + assert_ne!( + canonical_input(KEY, 0, issued, UrlKind::Ordinary), + canonical_input(KEY, 0, issued, UrlKind::DataPackage) + ); + let ordinary = sign(KEY, expires, issued, UrlKind::Ordinary, &secret); + let package = sign(KEY, 0, issued, UrlKind::DataPackage, &secret); + for (label, query) in [ + ( + "ordinary signature relabelled as a data package", + format!("ex=0&is={issued:08x}&hm={ordinary}&uc=dp"), + ), + ( + "data package signature relabelled as ordinary", + format!("ex={expires:08x}&is={issued:08x}&hm={package}"), + ), + ( + "ordinary input with a zero expiry signed under the data package", + format!( + "ex=0&is={issued:08x}&hm={}&uc=dp", + sign(KEY, 0, issued, UrlKind::Ordinary, &secret) + ), + ), + ] { + assert_eq!( + Verdict::Mismatch, + verdict_of(KEY, Some(&query), &[&secret], now), + "{label}" + ); + } + } + + #[test] + fn every_parameter_shape_outside_the_grammar_is_refused() { + let secret = secret_bytes(); + let now = ANCHOR; + let (issued, expires) = issue_window(ANCHOR, now); + let signature = sign(KEY, expires, issued, UrlKind::Ordinary, &secret); + let package = sign(KEY, 0, issued, UrlKind::DataPackage, &secret); + let valid = ordinary_query(KEY, issued, expires, &secret); + let valid_package = data_package_query(KEY, issued, &secret); + assert_eq!( + Verdict::Valid, + verdict_of(KEY, Some(&valid), &[&secret], now) + ); + assert_eq!( + Verdict::Valid, + verdict_of(KEY, Some(&valid_package), &[&secret], now) + ); + for (label, query, expected) in [ + ("no query", None, Verdict::Missing), + ("empty query", Some(String::new()), Verdict::Missing), + ( + "only transform parameters", + Some("width=64&format=webp".to_owned()), + Verdict::Missing, + ), + ( + "only the signature", + Some(format!("hm={signature}")), + Verdict::Malformed, + ), + ( + "only the window", + Some(format!("ex={expires:08x}&is={issued:08x}")), + Verdict::Malformed, + ), + ( + "repeated signature", + Some(format!("{valid}&hm={signature}")), + Verdict::Malformed, + ), + ( + "percent escaped second name", + Some(format!("%65x={expires:08x}&{valid}")), + Verdict::Malformed, + ), + ( + "uppercase signature", + Some(format!( + "ex={expires:08x}&is={issued:08x}&hm={}", + signature.to_ascii_uppercase() + )), + Verdict::Malformed, + ), + ( + "percent escaped value", + Some(format!( + "ex={expires:08x}&is={issued:08x}&hm=%{}", + &signature[2..] + )), + Verdict::Malformed, + ), + ( + "short window field", + Some(format!("ex={:07x}&is={issued:08x}&hm={signature}", 1)), + Verdict::Malformed, + ), + ( + "issued after expiry", + Some(format!("ex={issued:08x}&is={expires:08x}&hm={signature}")), + Verdict::Malformed, + ), + ( + "name without a value", + Some(format!("ex&is={issued:08x}&hm={signature}")), + Verdict::Malformed, + ), + ( + "zero expiry without uc", + Some(format!("ex=0&is={issued:08x}&hm={package}")), + Verdict::Malformed, + ), + ( + "padded zero expiry with uc", + Some(format!("ex=00000000&is={issued:08x}&hm={package}&uc=dp")), + Verdict::Malformed, + ), + ( + "eight digit expiry with uc", + Some(format!("{valid}&uc=dp")), + Verdict::Malformed, + ), + ( + "double zero expiry with uc", + Some(format!("ex=00&is={issued:08x}&hm={package}&uc=dp")), + Verdict::Malformed, + ), + ( + "another uc value", + Some(format!("ex=0&is={issued:08x}&hm={package}&uc=dq")), + Verdict::Malformed, + ), + ( + "uppercase uc", + Some(format!("ex=0&is={issued:08x}&hm={package}&uc=DP")), + Verdict::Malformed, + ), + ( + "empty uc", + Some(format!("ex=0&is={issued:08x}&hm={package}&uc=")), + Verdict::Malformed, + ), + ( + "uc without a value", + Some(format!("ex=0&is={issued:08x}&hm={package}&uc")), + Verdict::Malformed, + ), + ( + "percent escaped uc value", + Some(format!("ex=0&is={issued:08x}&hm={package}&uc=%64p")), + Verdict::Malformed, + ), + ("uc alone", Some("uc=dp".to_owned()), Verdict::Malformed), + ( + "uc among transform parameters", + Some("width=64&uc=dp&format=webp".to_owned()), + Verdict::Malformed, + ), + ( + "data package without its signature", + Some(format!("ex=0&is={issued:08x}&uc=dp")), + Verdict::Malformed, + ), + ( + "data package without its issued field", + Some(format!("ex=0&hm={package}&uc=dp")), + Verdict::Malformed, + ), + ( + "repeated uc", + Some(format!("{valid_package}&uc=dp")), + Verdict::Malformed, + ), + ( + "percent escaped repeated uc", + Some(format!("{valid_package}&%75c=dp")), + Verdict::Malformed, + ), + ( + "repeated zero expiry", + Some(format!("ex=0&{valid_package}")), + Verdict::Malformed, + ), + ( + "short data package issued field", + Some(format!("ex=0&is={:07x}&hm={package}&uc=dp", 1)), + Verdict::Malformed, + ), + ( + "uppercase data package signature", + Some(format!( + "ex=0&is={issued:08x}&hm={}&uc=dp", + package.to_ascii_uppercase() + )), + Verdict::Malformed, + ), + ] { + assert_eq!( + expected, + verdict_of(KEY, query.as_deref(), &[&secret], now), + "{label}" + ); + } + } + + #[test] + fn tolerated_input_shapes_still_verify() { + let secret = secret_bytes(); + let now = ANCHOR; + let (issued, expires) = issue_window(ANCHOR, now); + let signature = sign(KEY, expires, issued, UrlKind::Ordinary, &secret); + let package = sign(KEY, 0, issued, UrlKind::DataPackage, &secret); + for (label, query) in [ + ( + "a trailing separator", + format!("ex={expires:08x}&is={issued:08x}&hm={signature}&"), + ), + ( + "a doubled separator between the fields", + format!("ex={expires:08x}&&is={issued:08x}&hm={signature}"), + ), + ( + "empty pairs around the signature", + format!("&=&ex={expires:08x}&&is={issued:08x}&hm={signature}&&"), + ), + ( + "signature after other parameters", + format!("width=64&ex={expires:08x}&is={issued:08x}&hm={signature}"), + ), + ( + "unknown parameters between the signature fields", + format!("ex={expires:08x}&width=64&is={issued:08x}&format=webp&hm={signature}"), + ), + ( + "a data package with a trailing separator", + format!("ex=0&is={issued:08x}&hm={package}&uc=dp&"), + ), + ( + "a doubled separator between the data package fields", + format!("ex=0&is={issued:08x}&&hm={package}&uc=dp"), + ), + ( + "data package fields in another order", + format!("uc=dp&width=64&hm={package}&is={issued:08x}&ex=0"), + ), + ( + "percent escaped uc name", + format!("ex=0&is={issued:08x}&hm={package}&%75c=dp"), + ), + ( + "empty pairs around a data package", + format!("&=&ex=0&&is={issued:08x}&hm={package}&=&uc=dp&&"), + ), + ] { + assert_eq!( + Verdict::Valid, + verdict_of(KEY, Some(&query), &[&secret], now), + "{label}" + ); + } + } + + #[test] + fn every_signature_parameter_name_is_recognised_after_form_decoding() { + for name in ["ex", "is", "hm", "uc", "%65x", "%75c", "u%63", "%68%6D"] { + assert!(is_signature_parameter_name(name), "{name}"); + } + for name in [ + "", "e", "exp", "UC", "u+c", "u%2Bc", "%2575c", "width", "dp", + ] { + assert!(!is_signature_parameter_name(name), "{name}"); + } + } + + #[test] + fn a_decoded_key_names_one_object_across_every_path_spelling() { + for (path, expected) in [ + ("/attachments/1/2/cat.gif", "attachments/1/2/cat.gif"), + ("///attachments/1/2/cat.gif", "attachments/1/2/cat.gif"), + ("attachments/1/2/cat.gif", "attachments/1/2/cat.gif"), + ("/attachments/1/2/caf%C3%A9.gif", "attachments/1/2/café.gif"), + ("/attachments/1/2/café.gif", "attachments/1/2/café.gif"), + ("/attachments/1/2/a%2Fb.gif", "attachments/1/2/a/b.gif"), + ("/attachments/1/2/a+b.gif", "attachments/1/2/a+b.gif"), + ("/attachments/1/2/a%25b.gif", "attachments/1/2/a%b.gif"), + ("/attachments/1/2/a%2.gif", "attachments/1/2/a%2.gif"), + ] { + assert_eq!(Some(expected.to_owned()), decode_key(path), "{path}"); + } + for path in [ + "/attachments/1/2/caf%C3%28.gif", + "/attachments/1/2/%FF.gif", + "/attachments/1/2/%ED%A0%80.gif", + ] { + assert_eq!(None, decode_key(path), "{path}"); + } + } + + #[test] + fn signing_replaces_an_existing_signature_and_keeps_everything_else() { + let secret = secret_bytes(); + let now = ANCHOR; + let canonical = "https://media.test/attachments/1/2/cat.gif?width=64&height=64#anchor"; + let key = "attachments/1/2/cat.gif"; + let signed = with_signature(canonical, key, ANCHOR, now, &secret); + assert!(signed.contains("?ex=")); + assert!(signed.ends_with("&width=64&height=64#anchor")); + let resigned = with_signature( + &signed, + key, + ANCHOR, + now + ATTACHMENT_URL_BUCKET_SECS, + &secret, + ); + let package = with_data_package_signature(&resigned, key, ANCHOR, now, &secret); + let back = with_signature(&package, key, ANCHOR, now, &secret); + assert_eq!(signed, back); + for url in [&resigned, &package, &back] { + assert_eq!(1, url.matches("ex=").count(), "{url}"); + assert_eq!(1, url.matches("is=").count(), "{url}"); + assert_eq!(1, url.matches("hm=").count(), "{url}"); + assert!(url.ends_with("&width=64&height=64#anchor"), "{url}"); + assert_eq!(canonical, strip_signature(url), "{url}"); + } + assert_eq!(1, package.matches("uc=dp&").count(), "{package}"); + assert!(!resigned.contains("uc="), "{resigned}"); + assert!(!back.contains("uc="), "{back}"); + } + + #[test] + fn a_bare_signed_url_ends_with_its_last_signature_field() { + let secret = secret_bytes(); + let url = "https://media.test/attachments/1/2/cat.gif"; + let key = "attachments/1/2/cat.gif"; + for (signed, names) in [ + ( + with_signature(url, key, ANCHOR, ANCHOR, &secret), + &["ex=", "is=", "hm="][..], + ), + ( + with_data_package_signature(url, key, ANCHOR, ANCHOR, &secret), + &["ex=0", "is=", "hm=", "uc=dp"][..], + ), + ] { + assert!(!signed.ends_with('&'), "{signed}"); + assert!(!signed.contains("&&"), "{signed}"); + let query = signed.split_once('?').expect("a signed url has a query").1; + let fields: Vec<&str> = query.split('&').collect(); + assert_eq!(names.len(), fields.len(), "{signed}"); + for (field, name) in fields.iter().zip(names) { + assert!(field.starts_with(name), "{signed}"); + } + } + } + + #[test] + fn a_preserved_parameter_follows_one_separator() { + let secret = secret_bytes(); + let key = "attachments/1/2/cat.gif"; + let signers: [Signer; 2] = [with_signature, with_data_package_signature]; + for signer in signers { + for (url, tail) in [ + ( + "https://media.test/attachments/1/2/cat.gif?width=100", + "&width=100", + ), + ( + "https://media.test/attachments/1/2/cat.gif?width=100&height=64#top", + "&width=100&height=64#top", + ), + ("https://media.test/attachments/1/2/cat.gif#top", "#top"), + ("https://media.test/attachments/1/2/cat.gif?&=&", ""), + ] { + let signed = signer(url, key, ANCHOR, ANCHOR, &secret); + assert!(signed.ends_with(tail), "{signed}"); + assert!(!signed.contains("&&"), "{signed}"); + let head = signed + .split_once('#') + .map_or(signed.as_str(), |(head, _)| head); + assert!(!head.ends_with('&'), "{signed}"); + } + } + } +} diff --git a/fluxer_common/src/lib.rs b/fluxer_common/src/lib.rs index 6582122b0..9c821ffa8 100644 --- a/fluxer_common/src/lib.rs +++ b/fluxer_common/src/lib.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +pub mod attachment_url_signature; pub mod config; pub mod external_media_path; pub mod geoip; diff --git a/fluxer_common/src/testdata/attachment_url_signature_vectors.json b/fluxer_common/src/testdata/attachment_url_signature_vectors.json new file mode 100644 index 000000000..8abed9876 --- /dev/null +++ b/fluxer_common/src/testdata/attachment_url_signature_vectors.json @@ -0,0 +1,793 @@ +{ + "version": 3, + "ttl_secs": 86400, + "bucket_secs": 43200, + "fluxer_epoch_ms": 1420070400000, + "secrets_base64": ["AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", "ICEiIyQlJicoKSorLC0uLzAxMjM0NTY3ODk6Ozw9Pj8="], + "sign": [ + { + "name": "first issue at the snowflake second", + "channel_id": "1544725486800732163", + "attachment_id": "1544971349200470016", + "filename": "cat.gif", + "storage_key": "attachments/1544725486800732163/1544971349200470016/cat.gif", + "anchor": 1788420273, + "now": 1788420273, + "ex": "6a9a7231", + "is": "6a9920b1", + "hm": "2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "uc": "", + "signature_input": "fluxer-attachment-url-v1\n6a9a7231\n6a9920b1\n\nattachments/1544725486800732163/1544971349200470016/cat.gif", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif", + "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27" + }, + { + "name": "first issue with a slash inside the filename", + "channel_id": "1544725486800732163", + "attachment_id": "1544971349200470016", + "filename": "a/b.gif", + "storage_key": "attachments/1544725486800732163/1544971349200470016/a/b.gif", + "anchor": 1788420273, + "now": 1788420273, + "ex": "6a9a7231", + "is": "6a9920b1", + "hm": "5d594e9ce78bc3c82e735dd29bcfdf7cee9ebb4391304b1df90d454207feb258", + "uc": "", + "signature_input": "fluxer-attachment-url-v1\n6a9a7231\n6a9920b1\n\nattachments/1544725486800732163/1544971349200470016/a/b.gif", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/a/b.gif", + "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/a/b.gif?ex=6a9a7231&is=6a9920b1&hm=5d594e9ce78bc3c82e735dd29bcfdf7cee9ebb4391304b1df90d454207feb258" + }, + { + "name": "first issue with a unicode filename", + "channel_id": "1544725486800732163", + "attachment_id": "1544971349200470016", + "filename": "café.gif", + "storage_key": "attachments/1544725486800732163/1544971349200470016/café.gif", + "anchor": 1788420273, + "now": 1788420273, + "ex": "6a9a7231", + "is": "6a9920b1", + "hm": "37efc4faf535ba29ecf6580cb47a88d80b14cd6319e29931c06ddffa761abd9c", + "uc": "", + "signature_input": "fluxer-attachment-url-v1\n6a9a7231\n6a9920b1\n\nattachments/1544725486800732163/1544971349200470016/café.gif", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/café.gif", + "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/café.gif?ex=6a9a7231&is=6a9920b1&hm=37efc4faf535ba29ecf6580cb47a88d80b14cd6319e29931c06ddffa761abd9c" + }, + { + "name": "second bucket", + "channel_id": "1544725486800732163", + "attachment_id": "1544971349200470016", + "filename": "cat.gif", + "storage_key": "attachments/1544725486800732163/1544971349200470016/cat.gif", + "anchor": 1788420273, + "now": 1788463473, + "ex": "6a9b1af1", + "is": "6a99c971", + "hm": "d5f9d09526e0be7825b32ee9522927e0786d3fcb576e566ed93d642b6fc05bf5", + "uc": "", + "signature_input": "fluxer-attachment-url-v1\n6a9b1af1\n6a99c971\n\nattachments/1544725486800732163/1544971349200470016/cat.gif", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif", + "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=6a9b1af1&is=6a99c971&hm=d5f9d09526e0be7825b32ee9522927e0786d3fcb576e566ed93d642b6fc05bf5" + }, + { + "name": "far future issue stays on the bucket grid", + "channel_id": "1544725486800732163", + "attachment_id": "1544971349200470016", + "filename": "cat.gif", + "storage_key": "attachments/1544725486800732163/1544971349200470016/cat.gif", + "anchor": 1788420273, + "now": 1792740280, + "ex": "6adc5d31", + "is": "6adb0bb1", + "hm": "efbfba51976929974135d10c7760fc1dfa1a2e21bfd8e005493fb568a4a532b1", + "uc": "", + "signature_input": "fluxer-attachment-url-v1\n6adc5d31\n6adb0bb1\n\nattachments/1544725486800732163/1544971349200470016/cat.gif", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif", + "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=6adc5d31&is=6adb0bb1&hm=efbfba51976929974135d10c7760fc1dfa1a2e21bfd8e005493fb568a4a532b1" + }, + { + "name": "transform parameters and a fragment follow the signature", + "channel_id": "1544725486800732163", + "attachment_id": "1544971349200470016", + "filename": "cat.gif", + "storage_key": "attachments/1544725486800732163/1544971349200470016/cat.gif", + "anchor": 1788420273, + "now": 1788420273, + "ex": "6a9a7231", + "is": "6a9920b1", + "hm": "2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "uc": "", + "signature_input": "fluxer-attachment-url-v1\n6a9a7231\n6a9920b1\n\nattachments/1544725486800732163/1544971349200470016/cat.gif", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64&height=64#top", + "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&width=64&height=64#top" + }, + { + "name": "data package at the snowflake second", + "channel_id": "1544725486800732163", + "attachment_id": "1544971349200470016", + "filename": "cat.gif", + "storage_key": "attachments/1544725486800732163/1544971349200470016/cat.gif", + "anchor": 1788420273, + "now": 1788420273, + "ex": "0", + "is": "6a9920b1", + "hm": "0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a", + "uc": "dp", + "signature_input": "fluxer-attachment-url-v1\n00000000\n6a9920b1\ndp\nattachments/1544725486800732163/1544971349200470016/cat.gif", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif", + "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp" + }, + { + "name": "data package with a unicode filename", + "channel_id": "1544725486800732163", + "attachment_id": "1544971349200470016", + "filename": "café.gif", + "storage_key": "attachments/1544725486800732163/1544971349200470016/café.gif", + "anchor": 1788420273, + "now": 1788420273, + "ex": "0", + "is": "6a9920b1", + "hm": "89e63377486a58781d1f168c80a438e99a1c87f71f8bb5a7401f2512770286c2", + "uc": "dp", + "signature_input": "fluxer-attachment-url-v1\n00000000\n6a9920b1\ndp\nattachments/1544725486800732163/1544971349200470016/café.gif", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/café.gif", + "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/café.gif?ex=0&is=6a9920b1&hm=89e63377486a58781d1f168c80a438e99a1c87f71f8bb5a7401f2512770286c2&uc=dp" + }, + { + "name": "data package issued in a later bucket", + "channel_id": "1544725486800732163", + "attachment_id": "1544971349200470016", + "filename": "cat.gif", + "storage_key": "attachments/1544725486800732163/1544971349200470016/cat.gif", + "anchor": 1788420273, + "now": 1792740280, + "ex": "0", + "is": "6adb0bb1", + "hm": "34bdd1bc2fdb906efa5ff6bb2db8be256318be7457b60dfa593ee0507eec0881", + "uc": "dp", + "signature_input": "fluxer-attachment-url-v1\n00000000\n6adb0bb1\ndp\nattachments/1544725486800732163/1544971349200470016/cat.gif", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif", + "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=0&is=6adb0bb1&hm=34bdd1bc2fdb906efa5ff6bb2db8be256318be7457b60dfa593ee0507eec0881&uc=dp" + }, + { + "name": "data package keeps transform parameters after uc", + "channel_id": "1544725486800732163", + "attachment_id": "1544971349200470016", + "filename": "cat.gif", + "storage_key": "attachments/1544725486800732163/1544971349200470016/cat.gif", + "anchor": 1788420273, + "now": 1788420273, + "ex": "0", + "is": "6a9920b1", + "hm": "0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a", + "uc": "dp", + "signature_input": "fluxer-attachment-url-v1\n00000000\n6a9920b1\ndp\nattachments/1544725486800732163/1544971349200470016/cat.gif", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64#top", + "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp&width=64#top" + } + ], + "strip": [ + { + "name": "a signed url strips back to its canonical form", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif" + }, + { + "name": "a trailing separator on a signed url is tolerated", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&", + "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif" + }, + { + "name": "transform parameters survive the strip", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&width=64&height=64", + "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64&height=64" + }, + { + "name": "a signature after other parameters is still removed", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64&ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64" + }, + { + "name": "empty pairs are dropped", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?&=&ex=6a9a7231&&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&", + "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif" + }, + { + "name": "an unsigned url is unchanged", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64", + "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64" + }, + { + "name": "a fragment survives the strip", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27#top", + "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif#top" + }, + { + "name": "a percent escaped signature name is removed too", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?%65x=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&width=64", + "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64" + }, + { + "name": "a data package url strips back to its canonical form", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp", + "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif" + }, + { + "name": "transform parameters survive a data package strip", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp&width=64#top", + "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64#top" + }, + { + "name": "uc is removed whatever its value", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?uc=zz&width=64&uc", + "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64" + }, + { + "name": "a percent escaped uc name is removed too", + "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&%75c=dp&width=64", + "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64" + } + ], + "verify": [ + { + "name": "the emitted form verifies", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "valid one second before expiry", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788506672, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "expired at the expiry second", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788506673, + "verifier_secret_indices": [0], + "verdict": "expired" + }, + { + "name": "expired long after the window", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788549873, + "verifier_secret_indices": [0], + "verdict": "expired" + }, + { + "name": "a trailing separator is tolerated", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "a doubled separator between the fields is tolerated", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "empty pairs are tolerated", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "&=&ex=6a9a7231&&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&&", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "the signature may follow other parameters", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "width=64&ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "transform parameters are not signed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&width=64&format=webp", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "no query at all is missing", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": null, + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "missing" + }, + { + "name": "an empty query is missing", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "missing" + }, + { + "name": "only transform parameters is missing", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "width=64", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "missing" + }, + { + "name": "a partial set is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "a repeated signature is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "a percent escaped duplicate name is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "%65x=6a9a7231&ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "an uppercase signature is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2F82FAFD058D10847AFFCCC91735BC5B95753F4081F4DA68AE054C221D674F27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "a percent escaped value is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=%82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "a seven digit window field is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "issued after expiry is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9920b1&is=6a9a7231&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "a path whose bytes are not utf-8 is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/caf%C3%28.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "a signature over another object mismatches", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=37efc4faf535ba29ecf6580cb47a88d80b14cd6319e29931c06ddffa761abd9c", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "mismatch" + }, + { + "name": "a signature from another key mismatches", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [1], + "verdict": "mismatch" + }, + { + "name": "an altered expiry mismatches", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7232&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "mismatch" + }, + { + "name": "the rotation key verifies its own signature", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=f1963bb94826bb42d68224ea35c317a07fd3cfd7853cd5346ca85ec47611d04d", + "now": 1788420273, + "verifier_secret_indices": [1], + "verdict": "valid" + }, + { + "name": "a signature from the second key mismatches when only the first is held", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=f1963bb94826bb42d68224ea35c317a07fd3cfd7853cd5346ca85ec47611d04d", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "mismatch" + }, + { + "name": "a signature from the second key verifies while both are held", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=f1963bb94826bb42d68224ea35c317a07fd3cfd7853cd5346ca85ec47611d04d", + "now": 1788420273, + "verifier_secret_indices": [0, 1], + "verdict": "valid" + }, + { + "name": "a signature from the first key verifies whatever order the keys are held in", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [1, 0], + "verdict": "valid" + }, + { + "name": "a percent encoded unicode filename verifies", + "path": "/attachments/1544725486800732163/1544971349200470016/caf%C3%A9.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=37efc4faf535ba29ecf6580cb47a88d80b14cd6319e29931c06ddffa761abd9c", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "a raw unicode filename verifies", + "path": "/attachments/1544725486800732163/1544971349200470016/café.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=37efc4faf535ba29ecf6580cb47a88d80b14cd6319e29931c06ddffa761abd9c", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "a leading double slash names the same object", + "path": "//attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "an encoded slash names the decoded key", + "path": "/attachments/1544725486800732163/1544971349200470016/a%2Fb.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=5d594e9ce78bc3c82e735dd29bcfdf7cee9ebb4391304b1df90d454207feb258", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "the decoded key is what is signed", + "path": "/attachments/1544725486800732163/1544971349200470016/a/b.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=5d594e9ce78bc3c82e735dd29bcfdf7cee9ebb4391304b1df90d454207feb258", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "a plus is not a space in the key", + "path": "/attachments/1544725486800732163/1544971349200470016/a+b.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=5d594e9ce78bc3c82e735dd29bcfdf7cee9ebb4391304b1df90d454207feb258", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "mismatch" + }, + { + "name": "the emitted data package form verifies", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "a data package verifies long after an ordinary url would expire", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp", + "now": 4102444800, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "a data package tolerates a trailing separator", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp&", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "a doubled separator between the data package fields is tolerated", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "a data package tolerates its fields in any position", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "uc=dp&width=64&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&is=6a9920b1&ex=0", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "transform parameters after a data package are not signed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp&width=64&format=webp", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "a percent escaped uc name counts as uc", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&%75c=dp", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "a data package with a percent encoded unicode filename verifies", + "path": "/attachments/1544725486800732163/1544971349200470016/caf%C3%A9.gif", + "query": "ex=0&is=6a9920b1&hm=89e63377486a58781d1f168c80a438e99a1c87f71f8bb5a7401f2512770286c2&uc=dp", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "a data package from the second key verifies while both are held", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&hm=0a442f8c3a83e6075764ada1d3b833fa9c1977354861c12e55ba04502af77cdf&uc=dp", + "now": 1788420273, + "verifier_secret_indices": [0, 1], + "verdict": "valid" + }, + { + "name": "a data package from a key removed from rotation mismatches", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&hm=0a442f8c3a83e6075764ada1d3b833fa9c1977354861c12e55ba04502af77cdf&uc=dp", + "now": 4102444800, + "verifier_secret_indices": [0], + "verdict": "mismatch" + }, + { + "name": "a data package from another key mismatches", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp", + "now": 1788420273, + "verifier_secret_indices": [1], + "verdict": "mismatch" + }, + { + "name": "a data package over another object mismatches", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&hm=89e63377486a58781d1f168c80a438e99a1c87f71f8bb5a7401f2512770286c2&uc=dp", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "mismatch" + }, + { + "name": "an altered issued field on a data package mismatches", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b2&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "mismatch" + }, + { + "name": "an ordinary signature relabelled as a data package mismatches", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&uc=dp", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "mismatch" + }, + { + "name": "a data package signature relabelled as an ordinary url mismatches", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "mismatch" + }, + { + "name": "a data package with uc removed is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "a tampered uc value is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dq", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "an uppercase uc value is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=DP", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "an empty uc value is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "a percent escaped uc value is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=%64p", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "a zero padded expiry with uc is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=00000000&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "uc on an ordinary url is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&uc=dp", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "uc alone is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "uc=dp", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "uc among transform parameters is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "width=64&uc=dp&format=webp", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "a data package without its signature is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&uc=dp", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "a repeated uc is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp&uc=dp", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "a percent escaped repeated uc is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp&%75c=dp", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "a repeated zero expiry is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "a seven digit data package issued field is malformed", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=0&is=a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "malformed" + }, + { + "name": "a leading triple slash names the same object", + "path": "///attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "percent encoded slashes in the prefix name the same object", + "path": "/attachments%2F1544725486800732163%2F1544971349200470016%2Fcat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "valid" + }, + { + "name": "a dot segment after the prefix is part of the key", + "path": "/attachments/./1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "mismatch" + }, + { + "name": "a percent encoded dot segment after the prefix is part of the key", + "path": "/attachments/%2E/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "mismatch" + }, + { + "name": "a dot segment between the id components is part of the key", + "path": "/attachments/1544725486800732163/./1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "mismatch" + }, + { + "name": "a doubled slash inside the path is part of the key", + "path": "/attachments/1544725486800732163//1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "mismatch" + }, + { + "name": "a parent segment before the prefix is part of the key", + "path": "/x/../attachments/1544725486800732163/1544971349200470016/cat.gif", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "mismatch" + }, + { + "name": "a trailing dot segment is part of the key", + "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif/.", + "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27", + "now": 1788420273, + "verifier_secret_indices": [0], + "verdict": "mismatch" + } + ] +} diff --git a/fluxer_docs/scripts/VerifyDocsCoverage.ts b/fluxer_docs/scripts/VerifyDocsCoverage.ts index dbedbd22a..48ce40e8b 100644 --- a/fluxer_docs/scripts/VerifyDocsCoverage.ts +++ b/fluxer_docs/scripts/VerifyDocsCoverage.ts @@ -192,6 +192,7 @@ const MEDIA_PROXY_ROUTES = new Map([ ['HEAD /_health', '.route("/_health", get(routes::ops::health))'], ['GET /_metrics', '.route("/_metrics", get(routes::ops::metrics_handler))'], ['POST /_metadata', '.route("/_metadata", post(routes::internal::metadata_handler))'], + ['POST /_sniff', '.route("/_sniff", post(routes::internal::sniff_handler))'], ['POST /_thumbnail', '.route("/_thumbnail", post(routes::internal::thumbnail_handler))'], ['POST /_frames', '.route("/_frames", post(routes::internal::frames_handler))'], [ @@ -439,6 +440,25 @@ for (const [route, {documentedIn}] of OUT_OF_BAND_CREDENTIAL) { } failures += section('stale anchors (the code moved, update this script)', staleAnchors); +const mediaProxyRouterSource = await readFile(path.join(MEDIA_PROXY_SERVER_DIR, 'runtime.rs'), 'utf8'); +const mediaProxyRegisteredPaths = [...mediaProxyRouterSource.matchAll(/\.route\(\s*"([^"]+)"/gu)].map((match) => + match[1].replace(/\{[^}]*\}/gu, '{}'), +); +const mediaProxyListedPaths = new Set([...MEDIA_PROXY_ROUTES.keys()].map((shape) => shape.split(' ')[1])); +const unlistedMediaProxyPaths = [...new Set(mediaProxyRegisteredPaths)] + .filter((routePath) => !mediaProxyListedPaths.has(routePath)) + .map((routePath) => `${routePath}: build_router registers it and MEDIA_PROXY_ROUTES does not name it`) + .sort(); +if (mediaProxyRegisteredPaths.length === 0) { + unlistedMediaProxyPaths.push( + 'fluxer_media_proxy/src/server/runtime.rs registers no route, so this check has gone blind', + ); +} +failures += section('registered by fluxer_media_proxy but absent from this script', unlistedMediaProxyPaths); +console.log( + ` routes registered in fluxer_media_proxy/src/server/runtime.rs: ${mediaProxyRegisteredPaths.length.toString()}`, +); + const mediaProxyDocumented = documented.filter((route) => route.file.startsWith('media-proxy/')); const adminDocumented = documented.filter( (route) => !route.file.startsWith('media-proxy/') && stripVersionPrefix(route.path).startsWith('/admin'), diff --git a/fluxer_docs/src/content/docs/http-api/memes.mdx b/fluxer_docs/src/content/docs/http-api/memes.mdx index 1e20a7692..0813c7512 100644 --- a/fluxer_docs/src/content/docs/http-api/memes.mdx +++ b/fluxer_docs/src/content/docs/http-api/memes.mdx @@ -54,7 +54,7 @@ One media asset owned by one account. Fluxer resolves every media field at creat 2 [Save meme from message](#save-meme-from-message) falls back to the selected attachment's own description when the body supplies no non-empty alt text -3 `url` is the media endpoint followed by `attachments/{user_id}/{attachment_id}/{filename}` +3 `url` is the media endpoint followed by `attachments/{user_id}/{attachment_id}/{filename}`, and Fluxer signs it on every read as a [signed attachment URL](/http-api/messages/#signed-attachment-urls) 4 Set when the source attachment or embed was already animated, and when the Media Proxy reports the stored copy as animated diff --git a/fluxer_docs/src/content/docs/http-api/messages.mdx b/fluxer_docs/src/content/docs/http-api/messages.mdx index 24f96c80a..97dd6a95a 100644 --- a/fluxer_docs/src/content/docs/http-api/messages.mdx +++ b/fluxer_docs/src/content/docs/http-api/messages.mdx @@ -150,8 +150,8 @@ An attachment is one file stored against a message and served from media deliver | content_type1 | string | The media type detected for the attachment | | content_hash | ?string | The hash of the attachment's stored bytes, or null when they were never hashed | | size | integer | The stored size of the attachment in bytes | -| url2 | ?string | Public attachment URL, or null once the attachment has expired | -| proxy_url2 | ?string | Media Proxy URL, or null once the attachment has expired | +| url2 | ?string | Signed public attachment URL, or null once the attachment has expired | +| proxy_url2 | ?string | Signed Media Proxy URL, or null once the attachment has expired | | width?3 | integer | Pixel width for visual media | | height?3 | integer | Pixel height for visual media | | placeholder? | string | Encoded media placeholder | @@ -164,14 +164,47 @@ An attachment is one file stored against a message and served from media deliver 1 Always present. It falls back to a type guessed from the stored filename when the attachment has no recorded media type, and to `application/octet-stream` when the filename yields no guess -2 Both fields are the same canonical media endpoint URL, and both become null together once the attachment has expired +2 Both fields are the same media endpoint URL with the same [signature](#signed-attachment-urls), and both become null together once the attachment has expired 3 Omitted for an audio media type even when a stored dimension exists -4 The key is always present. Its value is the recorded decay deadline, or null when the attachment has none +4 The key is always present. Its value is the recorded decay deadline, or null when the attachment has none. It is unrelated to the `ex` time of a signed URL 5 Present and true only when the attachment has already expired, so an absent key must be read as false +A new attachment starts with the media type derived from the filename given when its upload was requested. For a pre-uploaded attachment that is the `filename` of its [attachment upload request item](#attachment-upload-request-item-object), which can differ from the `filename` the message sends. When the starting type is not a supported image, video, or audio type, Fluxer checks the first 8 KiB of the file for one. For an image, video, or audio file the Media Proxy can read, `content_type` is then the type detected from its bytes. When the starting type is not a media type, SVG bytes keep it. So does a file Fluxer cannot check or the Media Proxy cannot read. + +### Signed attachment URLs + +An instance configured with attachment URL secrets signs `url` and `proxy_url`. Fluxer adds the signature parameters to the attachment path, before any other parameter. An instance without those secrets returns the URL with none of them. + +| Field | Type | Description | +| --- | --- | --- | +| ex | string | The Unix time in seconds at which the URL expires, as 8 lowercase hexadecimal digits, or `0` on a data package URL | +| is | string | The Unix time in seconds at which the URL was issued, as 8 lowercase hexadecimal digits | +| hm | string | The signature over `ex`, `is`, `uc`, and the attachment path, as 64 lowercase hexadecimal digits | +| uc | string | Present only on a data package URL, where the value is always `dp` | + +An ordinary URL has `ex`, `is`, and `hm`, and no `uc`. It is valid for 24 hours from `is`. `is` starts at the attachment's creation time and moves forward in steps of 12 hours, so a URL Fluxer returns has more than 12 and at most 24 hours left. Fluxer signs the URL each time it returns the attachment, so a later read of the message returns a URL for the current step. [Refresh attachment URLs](#refresh-attachment-urls) signs a URL again without reading the message. + +A client MUST keep `ex`, `is`, `hm`, and `uc` exactly as issued. It MAY add any other query parameter, such as a [transformation](/media-proxy/transformations/), because the signature covers no other parameter. A client SHOULD refresh a URL whose `ex` is absent or less than one hour away. Once the operator enforces signatures, a read with a missing, altered, or expired signature returns 404 from the Media Proxy, as [Signed attachment URLs](/media-proxy/overview/#signed-attachment-urls) defines. + +Fluxer never rewrites message `content` or embed text. An attachment URL pasted into either keeps the parameters it was sent with, and it stops working when its own `ex` passes. + +### Data package attachment URLs + +A [data harvest](/http-api/users/data-harvest/) archive records a data package URL for each attachment instead of an ordinary one. It has `uc=dp`, and its `ex` is the single character `0`: + +```text +?ex=0&is={issued}&hm={signature}&uc=dp +``` + +A data package URL does not expire. The Media Proxy checks its signature against every configured secret and skips the expiry check, so it reads its object for as long as the secret that signed it stays in the instance's secret list. Removing that secret is the only thing that ends it, and it ends every data package URL that secret signed at once. + +`is` is set the way it is on an ordinary URL, and the signature covers `uc`, so an ordinary URL relabelled `uc=dp` does not verify, and neither does a data package URL with `uc` removed. + +A client treats a data package URL as never expiring and does not refresh it before use. A refused read still triggers the same forced refresh as any other attachment URL, and that returns an ordinary URL. + ### Example ```json @@ -183,8 +216,8 @@ An attachment is one file stored against a message and served from media deliver "content_type": "image/png", "content_hash": "9f86d081884c7d659a2feaa0c55ad015", "size": 18422, - "url": "https://media.example.com/attachments/1501314428688998182/1501320000000000001/diagram.png", - "proxy_url": "https://media.example.com/attachments/1501314428688998182/1501320000000000001/diagram.png", + "url": "https://media.example.com/attachments/1501314428688998182/1501320000000000001/diagram.png?ex=69fba4b5&is=69fa5335&hm=5586be3ae70411fe6582431649e2f28abc6c201e4f12a4f43a3256b72b90d3de", + "proxy_url": "https://media.example.com/attachments/1501314428688998182/1501320000000000001/diagram.png?ex=69fba4b5&is=69fa5335&hm=5586be3ae70411fe6582431649e2f28abc6c201e4f12a4f43a3256b72b90d3de", "width": 1280, "height": 720, "flags": 0, @@ -440,6 +473,10 @@ Every optional field above is omitted entirely. 1 Always derived from `url`, so every serialised media object has it +When `url` is an attachment URL of this instance, Fluxer signs both fields, so `url` and `proxy_url` are the same URL with the same fresh [signature](#signed-attachment-urls). A rich embed can name such a URL with or without `ex`, `is`, `hm`, and `uc`. Fluxer removes them before it stores the URL, and signs both fields again each time it returns the embed. + +Fluxer signs every other embed field that can hold an attachment URL of this instance the same way. That is `url` on the embed itself, `url`, `icon_url`, and `proxy_icon_url` on an [embed author](#embed-author-object), `icon_url` and `proxy_icon_url` on an [embed footer](#embed-footer-object), and `url` on an [embed provider](#embed-provider-object). An icon URL and its proxy twin are then the same signed URL. A field whose URL is not an attachment URL of this instance is returned as it was stored, and a proxy field of an external URL is the [signed external media](/media-proxy/routes/#get-signed-external-media) path instead. + ## Embed footer object ### Structure @@ -660,7 +697,7 @@ One declared upload inside a [Request attachment upload URLs](#request-attachmen 1 A decimal string is converted to the integer -2 The issued upload URL and the stored attachment both use the media type derived from `filename` +2 The issued upload URL uses the media type derived from `filename`. The [message attachment object](#message-attachment-object) states how the stored attachment is typed ### Singlepart attachment upload object @@ -726,6 +763,28 @@ One planned upload named in a [Complete attachment upload](#complete-attachment- | upload_filename | string | Opaque upload key returned when the upload was planned | | upload_id | string | Multipart upload identifier returned when the upload was planned | +## Refreshed attachment URL object + +One result of [Refresh attachment URLs](#refresh-attachment-urls). + +### Structure + +| Field | Type | Description | +| --- | --- | --- | +| original | string | The URL exactly as it was sent | +| refreshed1 | string | The URL with a fresh [signature](#signed-attachment-urls), or `original` unchanged | + +1 Equal to `original` when the URL is not an attachment URL of this instance, or when the instance has no attachment URL secrets + +A URL is an attachment URL of this instance when all of these hold: + +- It parses as an absolute URL whose scheme, host, and port equal those of the media endpoint. The host is compared without regard to case, and a default port equals no port. +- Its path, as written in the string, starts with the media endpoint path followed by `/attachments/`. +- The rest of the path, percent-decoded, is valid UTF-8 and a safe storage key with no empty, `.`, or `..` segment. +- That key is `attachments/{channel_id}/{attachment_id}/{filename}`, where both IDs are decimal digits and the filename is not empty. + +To sign a URL, Fluxer removes every `ex`, `is`, `hm`, and `uc` parameter and every empty pair, then places the fresh `ex`, `is`, and `hm` first. Every other parameter follows in its original order and bytes, and a fragment is kept. A [data package URL](#data-package-attachment-urls) loses its `uc` this way, so the result is an ordinary URL valid for 24 hours. + ## Channel pin object A channel pin pairs one pinned message with the time it was pinned. @@ -1019,6 +1078,51 @@ The operation makes each completed upload available for a message. It creates no 10 requests per 10 seconds for each authenticated user and channel ID, on the `channel:attachment:upload::channel_id` bucket, which is shared with [Request attachment upload URLs](#request-attachment-upload-urls). +## Refresh attachment URLs + + + +Signs from 1 through 50 attachment URLs again. Returns one [refreshed attachment URL](#refreshed-attachment-url-object) object for each URL sent. A [data package URL](#data-package-attachment-urls) is accepted like any other, and its result is an ordinary URL valid for 24 hours. + +### Limitations + +- An OAuth2 bearer token is refused with 403 `ACCESS_DENIED`. +- Fluxer checks no channel membership, no permission, and no attachment existence, and reads no stored data. A URL is signed whether or not its object exists or the caller can see it. + +### JSON body + +| Field | Type | Description | +| --- | --- | --- | +| attachment_urls1 | array[string] | URLs to sign again (1-50 entries, each at most 2,048 characters) | + +1 Each entry is read exactly as sent, with no trimming or normalisation + +### Response body + +| Field | Type | Description | +| --- | --- | --- | +| refreshed_urls1 | array[[refreshed attachment URL](#refreshed-attachment-url-object) object] | Results in request order | + +1 Contains exactly one entry for each requested URL, in the order the entries were sent, and a duplicated URL therefore produces a duplicated entry + +### Response + +| Status | Body | Condition | +| --- | --- | --- | +| 200 | response body | Every URL was processed | +| 400 | [error response](/http-api/#error-response) | Body, URL count, or a URL length is invalid and the request returns `INVALID_FORM_BODY` | +| 401 | [error response](/http-api/#error-response) | The credential is missing or invalid and the request returns `UNAUTHORIZED` | +| 403 | [error response](/http-api/#error-response) | Caller presents a bearer credential and the request returns `ACCESS_DENIED` | +| 403 | [error response](/http-api/#error-response) | The account has an outstanding required action and the request returns `ACCOUNT_SUSPICIOUS_ACTIVITY` | + +:::caution[A refreshed URL is not an access check] +Anyone who can authenticate can sign any attachment URL of the instance whose path they know. A signature limits how long a copied URL works. It does not decide who can read the file. +::: + +### Rate limit + +20 requests per 10 seconds for each authenticated user, on the `attachment:refresh_urls` bucket. + ## Create message diff --git a/fluxer_docs/src/content/docs/http-api/users/data-harvest.mdx b/fluxer_docs/src/content/docs/http-api/users/data-harvest.mdx index 5e701be88..248f12c8a 100644 --- a/fluxer_docs/src/content/docs/http-api/users/data-harvest.mdx +++ b/fluxer_docs/src/content/docs/http-api/users/data-harvest.mdx @@ -189,7 +189,7 @@ Fluxer prepares the archive in the background. It contains: - `account/security.json` - the account avatar and banner under `assets/user/` when present -Attachment metadata appears with its message and has the attachment ID, filename, size, content type, CDN URL, and pixel dimensions. The attachment files themselves are never included. +Attachment metadata appears with its message and has the attachment ID, filename, size, content type, CDN URL, and pixel dimensions. The CDN URL is a [data package URL](/http-api/messages/#data-package-attachment-urls), so it does not expire and still reads its attachment after the archive's own seven day deadline passes. Removing the secret that signed it is the only thing that ends it. The attachment files themselves are never included. Every authored message is collected, with no ceiling on the count. Messages that no longer exist are omitted. Other read failures fail the attempt. After completion, Fluxer attempts to email a download URL if the account has an email address and the instance has email delivery enabled. That URL expires seven days after it was issued. Email failure does not prevent downloading a completed archive. diff --git a/fluxer_docs/src/content/docs/media-proxy/overview.md b/fluxer_docs/src/content/docs/media-proxy/overview.md index bb5f1731b..554843835 100644 --- a/fluxer_docs/src/content/docs/media-proxy/overview.md +++ b/fluxer_docs/src/content/docs/media-proxy/overview.md @@ -17,7 +17,7 @@ The Media Proxy serves Fluxer attachments, image assets, themes, entrance sound | Image asset1 | `/avatars`, `/icons`, `/branding`, `/banners`, `/splashes`, `/embed-splashes`, `/emojis`, `/stickers` | [Image asset contract](/media-proxy/routes/#image-asset-contract) | | Static object2 | `/{key}` | [Get static object](/media-proxy/routes/#get-static-object) | | Upload relay | `/v1/relay/{key}` | [Upload relay](/media-proxy/upload-relay/) | -| Operator and internal | `/_health`, `/_metrics`, `/_metadata`, `/_thumbnail`, `/_frames` | [Operator and internal endpoints](/media-proxy/routes/#operator-and-internal-endpoints) | +| Operator and internal | `/_health`, `/_metrics`, `/_metadata`, `/_sniff`, `/_thumbnail`, `/_frames` | [Operator and internal endpoints](/media-proxy/routes/#operator-and-internal-endpoints) | 1 Guild member avatars and banners are image assets too, at `/guilds/{guild_id}/users/{user_id}/avatars` and `/guilds/{guild_id}/users/{user_id}/banners` @@ -33,10 +33,10 @@ The upload relay is the exception in both respects. It is served below `/v1/rela ## Authorisation -No public Media Proxy route reads the HTTP API `Authorization` header. A stored object is authorised by its path, signed external media by its path signature, and an upload by the capability in its URL. Only the internal `POST` endpoints read `Authorization`, and they require the exact value `Bearer {secret}` built from the deployment secret. +No public Media Proxy route reads the HTTP API `Authorization` header. An attachment is authorised by its path and, under the [signed attachment URL policy](#signed-attachment-urls), by the signature in its query string. Any other stored object is authorised by its path, signed external media by its path signature, and an upload by the capability in its URL. Only the internal `POST` endpoints read `Authorization`, and they require the exact value `Bearer {secret}` built from the deployment secret. :::caution[A media URL is a bearer capability] -An attachment path has no signature and no expiry, so anyone holding the URL reads the object for as long as it exists. Leaving the channel does not revoke a URL already issued. +Anyone holding a URL reads the object without an account. Under `enforce`, an ordinary signed attachment URL works until its `ex` time, which is at most one day after it was signed, and a [data package URL](/http-api/messages/#data-package-attachment-urls) works until the secret that signed it is removed. Under `off` or `report`, an attachment path works for as long as the object exists. Leaving the channel does not revoke a URL already issued. ::: The Media Proxy has no request-count rate limit, so no response has the [rate limit headers](/topics/rate-limits/#rate-limit-headers) of the versioned HTTP API. A failure returns a short `text/plain` body, and [media error response](/media-proxy/responses-and-limits/#media-error-response) defines that shape. @@ -50,10 +50,11 @@ One Media Proxy process serves exactly one mode. The mode is fixed at startup an | `mp` | Attachments, signed external media, themes, entrance sounds, and every image asset route | | `static` | Every read route as a raw object read from the static bucket, with no transformation and no SVG rasterisation1 | | `upload` | The [upload relay](/media-proxy/upload-relay/), plus every `mp` read route from the same buckets | +| `relay` | The [upload relay](/media-proxy/upload-relay/) alone. Every read path returns 404 | 1 A `static` mode endpoint also strips `X-Robots-Tag` from every response and sends no `Content-Disposition` -The relay `PUT` is the only route with a mode gate, and it returns 404 outside `upload` mode. On a read that requests no transformation, only an `mp` endpoint rasterises SVG, so an `upload` endpoint returns the original SVG bytes. The [operator and internal endpoints](/media-proxy/routes/#operator-and-internal-endpoints) behave the same in every mode. +The relay `PUT` returns 404 outside `upload` and `relay` mode. A `relay` endpoint returns 404 for a `GET` or `HEAD` of any read path and reads no object for it. On a read that requests no transformation, only an `mp` endpoint rasterises SVG, so an `upload` endpoint returns the original SVG bytes. The [operator and internal endpoints](/media-proxy/routes/#operator-and-internal-endpoints) behave the same in every mode. The operator chooses which mode serves each published base URL. The reference self-hosted deployment serves `endpoints.media` from an `upload` mode process. @@ -67,32 +68,102 @@ On the [signed external route](/media-proxy/routes/#get-signed-external-media), ## Request headers -Only `Range` affects the representation a public read returns. `X-Forwarded-For` decides which address the [media access allowlist](#access-restrictions) evaluates, so it can turn a 200 into a 403 without changing the representation. Every other request header, including `Authorization`, `Cookie`, `Accept`, `Origin`, `If-Range`, `If-None-Match`, `If-Modified-Since`, `If-Match`, and `If-Unmodified-Since`, is ignored on a public read route. +Only `Range` affects the representation a public read returns. The [cross-origin read policy](#cross-origin-reads) reads `Origin`, and under `enforce` that header decides `Access-Control-Allow-Origin` and can turn a 200 into a 403. Every other request header, including `Authorization`, `Cookie`, `Accept`, `If-Range`, `If-None-Match`, `If-Modified-Since`, `If-Match`, and `If-Unmodified-Since`, is ignored on a public read route. ### Common request headers | Field | Type | Description | | --- | --- | --- | | Range?1 | string | One byte range under the [byte-range contract](#byte-ranges) | -| X-Forwarded-For?2 | string | Client address chain read by the [media access allowlist](#access-restrictions) | +| Origin?2 | string | Serialised request origin read by the [cross-origin read policy](#cross-origin-reads) | 1 A value whose unit is not the exact lowercase `bytes`, or that names more than one range, is ignored, and the response has the complete representation -2 Read only when the allowlist gate is enabled and the peer address is a configured trusted proxy. A value from any other peer is ignored +2 Read only when the cross-origin read policy is `report` or `enforce` -## Access restrictions +## Cross-origin reads -An operator can restrict public reads to a CDN edge address allowlist. This restriction is disabled by default. +An operator can limit which web origins read media through CORS. `FLUXER_MEDIA_PROXY_CORS_MODE` sets this cross-origin read policy, and it defaults to `off`. -A request from an address outside the list returns 403. `/_health`, `/_metrics`, `/_metadata`, `/_thumbnail`, `/_frames`, and every path below `/v1/relay/` are exempt. +| Value | Description | +| --- | --- | +| `off` | A media representation has `Access-Control-Allow-Origin: *`, and no `Origin` is read | +| `report` | Every response is the same as under `off`, and each read `enforce` would refuse is logged | +| `enforce` | A read from an origin outside the allowlist returns 403 | -When the peer address is a configured trusted proxy, the client address is the rightmost `X-Forwarded-For` hop that is not itself a trusted proxy. Otherwise the Media Proxy uses the peer address and ignores `X-Forwarded-For`. It does the same when there is no `X-Forwarded-For`, or when every hop in it is unparseable or is itself a trusted proxy. +Under `enforce`, an `Origin` is allowed when it equals the serialised form of an entry in `FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS` byte for byte. Anything else is refused, including `null`, a repeated `Origin` header, and a value that is not ASCII. A refused read returns 403 on every read path, and the Media Proxy reads no object and fetches no external URL for it. [Media Proxy settings](/operator/configuration/#media-proxy-settings) defines how an entry is parsed and serialised. + +A read with no `Origin` is served without `Access-Control-Allow-Origin`. A browser sends no `Origin` for an `` or a `