From 1f810ba04d2bed9509d35b14e4111feccc586357 Mon Sep 17 00:00:00 2001 From: Hampus Date: Fri, 18 Sep 2026 17:35:58 +0200 Subject: [PATCH] fix(api): drop the upload segment signal and dead exports (#2831) --- deploy/self-hosting/.env.example | 302 ++++++------------ .../src/api/attachment/AttachmentUrls.ts | 4 - .../message/AttachmentProcessingService.ts | 21 +- .../services/message/UploadSegmentSignal.ts | 209 ------------ 4 files changed, 96 insertions(+), 440 deletions(-) delete mode 100644 fluxer_api/src/api/channel/services/message/UploadSegmentSignal.ts diff --git a/deploy/self-hosting/.env.example b/deploy/self-hosting/.env.example index a87b89529..89dd904ab 100644 --- a/deploy/self-hosting/.env.example +++ b/deploy/self-hosting/.env.example @@ -1,108 +1,64 @@ -# Every variable docker-compose.yml reads is named here, uncommented when it has -# no default and commented with its default when it has one. A name absent from -# this file reaches a service only through a Compose override. Compose expands -# top to bottom, so a line using ${...} must sit below every name it reads. +# Every variable docker-compose.yml reads, uncommented when it has no default and +# commented with its default when it has one. Compose expands top to bottom, so a +# line using ${...} must sit below every name it reads. FLUXER_DOMAIN=chat.example.com FLUXER_PUBLIC_SCHEME=https FLUXER_PUBLIC_PORT=443 -# The lines above are the address browsers use, and every advertised endpoint -# carries FLUXER_PUBLIC_PORT. They do not move what the host publishes. -# FLUXER_HTTP_PORT and FLUXER_HTTPS_PORT below do that, and a non-default port -# needs the matching one set as well. Complete recipes sit beside them. +# The address browsers use. FLUXER_HTTP_PORT and FLUXER_HTTPS_PORT below decide +# which host ports Fluxer binds. -# How browsers reach this instance. -# -# Default: Fluxer binds 80 and 443 and gets its own Let's Encrypt certificate. -# Point DNS at this host. -# -# Behind your own reverse proxy (nginx, Traefik, HAProxy, Cloudflare Tunnel, -# another Caddy): uncomment COMPOSE_FILE below. Fluxer then serves plain HTTP on -# 127.0.0.1:8080 instead, and your proxy forwards everything to it. Keep -# FLUXER_PUBLIC_SCHEME and FLUXER_PUBLIC_PORT describing the PUBLIC address your -# proxy serves, not this local port. +# By default Fluxer binds 80 and 443 and gets its own certificate. Point DNS here. +# Behind your own reverse proxy, uncomment this instead: Fluxer then serves plain +# HTTP on 127.0.0.1:8080. Keep the scheme and port above describing the public +# address, not this one. #COMPOSE_FILE=docker-compose.yml:docker-compose.proxy.yml -# Where the plain-HTTP port binds when the proxy overlay is in use. Leave it on -# loopback when the proxy runs on this host. Use 0.0.0.0:8080 only when the proxy -# is on another machine, and firewall the port to that machine. +# Where that plain-HTTP port binds. Use 0.0.0.0:8080 only when the proxy is on +# another machine, and firewall it to that machine. #FLUXER_EDGE_BIND=127.0.0.1:8080 -# Which upstream hops may set X-Forwarded-For. Fluxer rewrites the header from -# this to the real client address, so IP bans, rate limits and abuse detection -# see the caller rather than the proxy. The default covers proxies on private or -# loopback addresses, which is every same-host setup. Set it to your proxy's -# address if it reaches Fluxer from a public IP. +# Which hops may set X-Forwarded-For. The default covers private and loopback +# addresses. Set your proxy's address if it reaches Fluxer from a public IP. #FLUXER_EDGE_TRUSTED_PROXIES=private_ranges -# The origin browsers see, without a trailing slash. Leave it unset and each -# service builds one from the three values at the top of this file. Set it and it -# wins: every service reads the host, the scheme and the port out of it and -# ignores those three names. Use it when browsers reach the instance on a host -# FLUXER_DOMAIN does not name. It has to be a bare origin, a scheme and a host -# and an optional port and nothing after them, or the services refuse to start. -# It does not move the edge listener or the published ports either, so set the -# publish below to the port written here. +# The origin browsers see, no trailing slash. Set it when browsers reach the +# instance on a host FLUXER_DOMAIN does not name, and it wins over the three +# values above. Scheme, host and optional port only. It does not move the +# published ports. #FLUXER_PUBLIC_ORIGIN=https://chat.example.com -# Overrides the address the edge listens on inside its container. Compose builds -# it from FLUXER_PUBLIC_SCHEME and FLUXER_DOMAIN with no port, and the edge keeps -# its container ports at 80 and 443 whatever the public port is. Caddy matches a -# site by host and ignores the port in the Host header, so a request arriving on -# a non-default published port still lands on this site. Put a port in this value -# only if you also publish that same container port below, or nothing will be -# listening where the publish points. Honoured in the default mode only: -# docker-compose.proxy.yml sets the literal :8080 and tunnel.compose.yml the -# literal :80, and Compose lets the last file win, so a value here is discarded -# under either overlay with no warning. Set it for an unusual default-mode -# layout, such as serving several hostnames. Write the scheme into it: a bare -# hostname means automatic HTTPS on 443 whatever FLUXER_PUBLIC_SCHEME says. +# The address the edge listens on inside its container. Both proxy overlays set +# this themselves, so a value here is ignored under either. Include the scheme. #FLUXER_EDGE_SITE_ADDRESS=https://chat.example.com -# The old name for the value above, read only when FLUXER_EDGE_SITE_ADDRESS is -# unset, so an existing .env keeps the listener it already had. +# The old name for the line above, read only when it is unset. #FLUXER_CADDY_SITE_ADDRESS= -# Host side of the edge's publishes, and the only names that decide which host -# ports Fluxer binds. The container side is fixed. Container 80 carries the -# HTTP to HTTPS redirect and the Let's Encrypt HTTP challenge under an https -# scheme, and the site itself under an http one. Container 443 carries the TLS -# site. FLUXER_HTTPS_PORT moves the TCP and the UDP publish together, because -# HTTP/3 needs both on the same port. Both take an optional bind address in front -# of the port, and 127.0.0.1 keeps the publish off every public interface. Give -# them different host ports: the same host port on both is two publishes of one -# port and the edge refuses to start. +# Host ports. Container 80 handles the redirect and the certificate challenge, +# container 443 the TLS site. FLUXER_HTTPS_PORT moves TCP and UDP together, since +# HTTP/3 needs both. Both accept a bind address. Give them different host ports. #FLUXER_HTTP_PORT=80 #FLUXER_HTTPS_PORT=443 #FLUXER_HTTP_PORT=127.0.0.1:80 #FLUXER_HTTPS_PORT=127.0.0.1:443 -# HTTPS on 8443, complete. Host 80 stays published and still answers the ACME -# challenge. Let's Encrypt only ever connects to the public 80 or 443, so the -# certificate is issued if a router in front forwards public 80 to this host and -# is not issued otherwise. Serve your own certificate from the Caddyfile when it -# cannot. +# HTTPS on 8443. Host 80 stays published for the certificate challenge, which +# only ever arrives on public 80 or 443. Serve your own certificate if nothing +# forwards those. #FLUXER_PUBLIC_PORT=8443 #FLUXER_HTTPS_PORT=8443 -# Plain HTTP on 19080, complete. The port 80 publish moves to 19080, so nothing -# binds host 80. Under an http scheme nothing listens on container 443, so the -# last line parks that publish on loopback for a host that wants 443 for -# something else. Drop it and 443 is published and idle, which is what earlier -# releases did. +# Plain HTTP on 19080. Nothing binds host 80, and the last line parks the idle +# 443 publish on loopback. #FLUXER_PUBLIC_SCHEME=http #FLUXER_PUBLIC_PORT=19080 #FLUXER_HTTP_PORT=19080 #FLUXER_HTTPS_PORT=127.0.0.1:443 -# A tunnel or another proxy in front of the stack needs no HTTPS publish at all. -# tunnel.compose.yml ships beside this file and replaces Caddy's published ports -# with a single loopback HTTP publish, so nothing binds 443, and points the edge -# at plain HTTP on that publish so it stops redirecting to https. FLUXER_HTTP_PORT -# still moves that one publish. Set the line below and plain docker compose -# commands pick the file up, or add it to your own -f flags if you pass any. The -# file uses the !override tag, which needs Compose 2.24.4 or newer. +# A tunnel needs no HTTPS publish. tunnel.compose.yml ships beside this file and +# leaves one loopback HTTP publish. Needs Compose 2.24.4 or newer. #COMPOSE_FILE=docker-compose.yml:tunnel.compose.yml FLUXER_REGISTRY_OWNER=fluxerapp @@ -111,11 +67,8 @@ FLUXER_IMAGE_TAG=v1 POSTGRES_PASSWORD=CHANGE_ME MEILI_MASTER_KEY=CHANGE_ME -# The stack ships its own Postgres and its own object store, and points at both -# by service name. Set these to run either one outside the stack. Leave them -# unset and the bundled services are used. Taking a service out of the stack -# means an upgrade skips the backup step that reaches into it, and backing that -# store up belongs to whoever runs it. +# Set these to run Postgres or the object store outside the stack. Backing up a +# store you moved out is yours to arrange, and an upgrade skips it. #FLUXER_POSTGRES_HOST=db.example.com #FLUXER_POSTGRES_PORT=5432 #FLUXER_POSTGRES_DATABASE=fluxer @@ -125,8 +78,7 @@ MEILI_MASTER_KEY=CHANGE_ME #FLUXER_S3_PUBLIC_ENDPOINT=https://cdn.example.com #FLUXER_S3_REGION=eu-central-1 #FLUXER_S3_FORCE_PATH_STYLE=false -# Bucket names. The bundled object store creates whichever names these hold, so -# the two stay in step. An object store outside the stack needs the buckets to +# Bucket names. The bundled store creates these. An outside store needs them to # exist already. #FLUXER_S3_BUCKET_CDN=fluxer #FLUXER_S3_BUCKET_UPLOADS=fluxer-uploads @@ -134,10 +86,8 @@ MEILI_MASTER_KEY=CHANGE_ME #FLUXER_S3_BUCKET_REPORTS=fluxer-reports #FLUXER_S3_BUCKET_HARVESTS=fluxer-harvests -# The rest of the bundled services, pointed somewhere else the same way. Leave a -# line unset and the service in the stack is used. Taking a service out of the -# stack goes in an override file listed in COMPOSE_FILE, because an upgrade -# replaces docker-compose.yml. +# The other bundled services, pointed elsewhere. Removing a service from the +# stack belongs in an override file, since an upgrade replaces docker-compose.yml. #FLUXER_KV_URL=redis://cache.example.com:6379/0 #FLUXER_NATS_URL=nats://mq.example.com:4222 #FLUXER_NATS_JETSTREAM_URL=nats://mq.example.com:4222 @@ -145,24 +95,21 @@ MEILI_MASTER_KEY=CHANGE_ME #FLUXER_SEARCH_URL=https://search.example.com #FLUXER_LIVEKIT_INTERNAL_URL=http://livekit.example.com:7880 -# Voice off. The livekit service still runs until an override file takes it out. +# Voice off. The livekit service still runs until an override removes it. #FLUXER_LIVEKIT_ENABLED=false -# Optional systems, each off unless the instance is configured for it. +# Optional systems, each off unless configured. #FLUXER_SMS_ENABLED=false #FLUXER_STRIPE_ENABLED=false #FLUXER_NCMEC_ENABLED=false #FLUXER_CLAMAV_ENABLED=false -# The client address. Set the header name a proxy in front actually writes, and -# turn the trust off when nothing sits in front, because a trusted header an -# attacker can set is a spoofed client address. +# The client address. Name the header your proxy actually writes, and turn the +# trust off when nothing sits in front. #FLUXER_CLIENT_IP_HEADER_NAME=cf-connecting-ip #FLUXER_TRUST_CLIENT_IP_HEADER=true -# How much the services write. trace, debug, info, warn, error or fatal. Every -# service names the object storage endpoint and its addressing at info on start, -# so a bucket that answers 404 is visible without raising this. +# How much the services write. trace, debug, info, warn, error or fatal. #LOG_LEVEL=debug FLUXER_S3_ACCESS_KEY=fluxer @@ -177,62 +124,43 @@ FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64=CHANGE_ME FLUXER_ADMIN_SECRET_KEY_BASE=CHANGE_ME FLUXER_ADMIN_OAUTH_CLIENT_SECRET=CHANGE_ME -# The token every service sends to NATS. The bundled NATS runs without -# authentication, so this stays empty unless a Compose override points the stack -# at an external NATS that requires a token. Compose forwards the name to every -# container that connects. +# The token every service sends to NATS. The bundled NATS needs none, so this +# stays empty unless an override points at an external one. #FLUXER_NATS_AUTH_TOKEN= FLUXER_VAPID_PUBLIC_KEY=CHANGE_ME FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME -# The VAPID contact address defaults to admin@ followed by FLUXER_DOMAIN. Set it -# only if that mailbox does not exist. +# Defaults to admin@ followed by FLUXER_DOMAIN. Set it if that mailbox does not +# exist. #FLUXER_VAPID_EMAIL=admin@chat.example.com -# Passkeys follow FLUXER_DOMAIN by default. Set these only if browsers reach the -# instance on a different host, and note that changing FLUXER_PASSKEY_RP_ID -# invalidates every passkey already registered against the old value. +# Passkeys follow FLUXER_DOMAIN. Set these only if browsers use another host. +# Changing the RP ID invalidates every passkey registered against the old value. #FLUXER_PASSKEY_RP_ID=chat.example.com #FLUXER_PASSKEY_RP_NAME=Fluxer #FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS=https://chat.example.com #FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS=http://chat.example.com:19080 -# Two optional media policies, both off unless you turn them on. Nothing below is -# needed for a working instance, and an upgrade never adds any of it. +# Optional media policies, both off by default. See the operator docs. # -# The first limits which web origins may read media through CORS. A request with -# no Origin header is always served, so direct links, image tags and native -# clients keep working. The allowlist defaults to the public origin above, which -# is where this instance serves its web app. The hosted web client and the -# desktop app run on https://web.fluxer.app, so add that origin, as in the second -# allowlist line, if people use them with this instance. +# CORS limits which web origins may read media. A request with no Origin is +# always served. Add https://web.fluxer.app if people use the hosted client. # -# The second makes an attachment read need a signed URL, which stops a copied -# link working forever elsewhere. Turning it on takes two settings: a secret, and -# the mode. Generate the secret with openssl rand -base64 32. It is a comma -# separated list, the first entry signs and every entry verifies. To rotate, add -# the new secret second and run docker compose up -d, then move it first and run -# again. Remove the old secret no sooner than a day after that, because an -# ordinary URL it signed stays valid for up to a day. A data package URL inside a -# harvest export never expires, so removing a secret ends every data package URL -# it signed and those exports have to be rebuilt. +# Signatures make an attachment read need a signed URL, so a copied link stops +# working. Needs a secret from openssl rand -base64 32, first entry signs and +# every entry verifies. # -# Each mode is off, report or enforce on its own. Set a mode to report first to -# log what enforce would refuse while refusing nothing. media-proxy reads these -# at container start, so apply a change with docker compose up -d media-proxy. -# docker compose restart media-proxy keeps the old environment. +# Each mode is off, report or enforce. Start at report. media-proxy reads these +# at start, so apply with docker compose up -d media-proxy. #FLUXER_MEDIA_PROXY_CORS_MODE=enforce -#FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS=https://chat.example.com #FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS=https://chat.example.com,https://web.fluxer.app #FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64= #FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE=enforce -# Extra Content-Security-Policy sources, appended to the built-in ones. Set these -# only when a browser must reach an origin the defaults do not cover, such as a -# voice server hosted on a domain other than FLUXER_DOMAIN. Separate several -# sources with spaces or commas. Every one of them is empty by default, and the -# three carrying a value below are illustrations, not defaults. +# Extra Content-Security-Policy sources, appended to the built-in ones. Set one +# only when a browser must reach an origin the defaults do not cover. Separate +# several with spaces or commas. The three values below are illustrations. #FLUXER_CSP_EXTRA_DEFAULT_SRC= #FLUXER_CSP_EXTRA_CONNECT_SRC=wss://livekit.example.com:7881 #FLUXER_CSP_EXTRA_IMG_SRC=https://cdn.example.com @@ -244,39 +172,29 @@ FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME #FLUXER_CSP_EXTRA_WORKER_SRC= #FLUXER_CSP_EXTRA_MANIFEST_SRC= -# One report-uri for Content-Security-Policy violation reports. Empty leaves the -# directive off the header. +# One report-uri for CSP violation reports. Empty leaves the directive off. #FLUXER_CSP_REPORT_URI= -# Allow the SSO identity provider to resolve to a private or internal address. -# Off by default: the API refuses to call non-public addresses so a misconfigured -# provider URL cannot be used to reach internal services. Turn it on only when the -# provider genuinely lives on your own network, such as split-horizon DNS or a LAN -# identity provider, and only when you trust everyone who can configure SSO. +# Let the SSO provider resolve to a private address. Off by default, so a +# misconfigured provider URL cannot reach internal services. Turn it on only for +# a provider on your own network. #FLUXER_SSO_ALLOW_PRIVATE_ADDRESSES=true -# Both reach LiveKit as LIVEKIT_KEYS and the webhook signing key, and the API as -# FLUXER_LIVEKIT_API_KEY and FLUXER_LIVEKIT_API_SECRET. Change them together. +# These reach both LiveKit and the api. Change them together. LIVEKIT_API_KEY=fluxer LIVEKIT_API_SECRET=CHANGE_ME -# The URL browsers use for voice signalling. Compose builds it from -# FLUXER_PUBLIC_ORIGIN, or from FLUXER_PUBLIC_SCHEME, FLUXER_DOMAIN and -# FLUXER_PUBLIC_PORT, as that origin followed by /livekit. The client rewrites a -# leading http to ws itself. Set it only when LiveKit is served from another -# host. +# The URL browsers use for voice signalling. Built from the public origin plus +# /livekit. Set it only when LiveKit is served from another host. #FLUXER_LIVEKIT_URL= -# Media ports. LiveKit advertises these in ICE candidates, so the host must -# forward the same numbers. +# Media ports. LiveKit advertises these, so forward the same numbers. #FLUXER_LIVEKIT_TCP_PORT=7881 #FLUXER_LIVEKIT_UDP_PORT=7882 -# LiveKit finds the address browsers dial by asking a STUN server. A host that -# cannot reach one over UDP stops with "could not resolve external IP", and the -# address is then set by hand: put it in FLUXER_LIVEKIT_NODE_IP and set -# FLUXER_LIVEKIT_USE_EXTERNAL_IP to false. Point the STUN entries at another -# server to keep the lookup and leave Google out of it. +# LiveKit finds its public address over STUN. A host that cannot reach one stops +# with "could not resolve external IP", so set the address by hand instead, or +# point STUN elsewhere. #FLUXER_LIVEKIT_USE_EXTERNAL_IP=false #FLUXER_LIVEKIT_NODE_IP=203.0.113.10 #FLUXER_LIVEKIT_STUN_PRIMARY=stun.l.google.com:19302 @@ -303,11 +221,9 @@ FLUXER_CAPTCHA_TURNSTILE_SITE_KEY= FLUXER_CAPTCHA_TURNSTILE_SECRET_KEY= FLUXER_DISCOVERY_ENABLED=true -# Container memory. The limits sum to 18.25 GiB, which is a sum of ceilings and -# not an allocation, so the defaults fit a host with 8 GB and are sized for 16 GB. -# The reservations are cgroup memory.low, which biases the kernel away from -# reclaiming from services whose death takes the instance down. They reserve -# nothing. Lower the limits on a smaller host. +# Container memory. These are ceilings, not allocations, and the defaults suit a +# 16 GB host. The reservations bias the kernel away from reclaiming from services +# whose death takes the instance down. Lower the limits on a smaller host. #FLUXER_CADDY_MEMORY_LIMIT=256mb #FLUXER_POSTGRES_MEMORY_LIMIT=5gb #FLUXER_POSTGRES_MEMORY_RESERVATION=3gb @@ -338,32 +254,22 @@ FLUXER_DISCOVERY_ENABLED=true #FLUXER_UNFURL_SHARD_MEMORY_LIMIT=256mb #FLUXER_ADMIN_MEMORY_LIMIT=256mb -# Meilisearch indexing memory. Keep it well under FLUXER_MEILISEARCH_MEMORY_LIMIT, -# which is the container ceiling the indexer shares with the search process. +# Meilisearch indexing memory. Keep it well under the container limit above. #FLUXER_MEILISEARCH_MAX_INDEXING_MEMORY=384mb -# SeaweedFS heap ceiling. Go collects against this value instead of against the -# container limit, which it cannot see, so without it an upload burst grows the -# heap past FLUXER_SEAWEEDFS_MEMORY_LIMIT and the kernel OOM-kills the container -# mid-upload (exit 137). Keep it near three quarters of that limit, and raise both -# together: the peak is the parts of one upload in flight at once, which is 25 MB -# times 20 for a 500 MB attachment. +# SeaweedFS heap ceiling. Go cannot see the container limit, so without this an +# upload burst gets the container OOM-killed. Keep it near three quarters of +# FLUXER_SEAWEEDFS_MEMORY_LIMIT and raise both together. #FLUXER_SEAWEEDFS_GOMEMLIMIT=1536MiB -# Node sizes its own heap from the container memory limit by default, at roughly -# 55 percent of it, which always leaves room for the buffers and stacks that live -# outside the heap. Leave these unset unless you have a reason to pin the value. -# Any value set here must stay well below the container limit above: a heap ceiling -# above the container limit makes the kernel OOM-kill the container (exit 137, no -# diagnostics) instead of Node reporting a JavaScript heap out of memory error. +# Node sizes its heap from the container limit by default. Leave these unset +# unless you need to pin it. A heap ceiling above the container limit gets the +# container OOM-killed instead of reporting a heap error. #FLUXER_API_NODE_HEAP_MB=1792 #FLUXER_WORKER_NODE_HEAP_MB=1792 -# Bundled Postgres tuning. Keep these consistent with FLUXER_POSTGRES_MEMORY_LIMIT: -# budget roughly shared_buffers + (server max_connections x 12 MB) + -# (3 x autovacuum_work_mem) + 300 MB for page cache and WAL. Note this is the -# server setting, distinct from the per-service FLUXER_POSTGRES_MAX_CONNECTIONS -# pool sizes used by the api, worker and shards. +# Bundled Postgres tuning. Keep it consistent with the memory limit above. This +# is the server setting, not the per-service pool sizes. #FLUXER_POSTGRES_SERVER_MAX_CONNECTIONS=150 #FLUXER_POSTGRES_SHARED_BUFFERS=512MB #FLUXER_POSTGRES_EFFECTIVE_CACHE_SIZE=2GB @@ -371,48 +277,30 @@ FLUXER_DISCOVERY_ENABLED=true #FLUXER_POSTGRES_MAINTENANCE_WORK_MEM=256MB #FLUXER_POSTGRES_AUTOVACUUM_WORK_MEM=128MB -# The bundled Valkey holds durable state as well as cache. The bulk message -# deletion queue and the account deletion queue are sorted sets with no expiry, -# and nothing else stores the first of the two. It therefore runs with an -# append-only file on a named volume and with noeviction, so an over-limit write -# fails loudly instead of silently deleting queued work. Distributed locks all -# carry a TTL and are not what the durability is for. Only change the policy if -# you have moved that durable state elsewhere. +# The bundled Valkey holds durable state as well as cache, so it runs with an +# append-only file and with noeviction, which fails an over-limit write instead +# of dropping queued work. Change the policy only if that state lives elsewhere. #FLUXER_VALKEY_MAXMEMORY=192mb #FLUXER_VALKEY_MAXMEMORY_POLICY=noeviction -# The gateway derives its BEAM scheduler count from the container CPU quota, -# clamped to this range. The floor matters: a single scheduler lets one blocking -# operation stall every websocket on the node. The ceiling stops a large host -# from starting far more schedulers than the container can actually use. +# The gateway derives its scheduler count from the CPU quota, clamped here. One +# scheduler lets a single blocking operation stall every websocket on the node. #FLUXER_ERLANG_SCHEDULERS_MIN=2 #FLUXER_ERLANG_SCHEDULERS_MAX=16 -# In-flight request ceiling for the services Compose forwards it to: the users -# and messages routers and their shards. Leave it unset and each service uses its -# built-in default. Set it and the one value replaces that default on all of -# them, so size it for the busiest. The built-in defaults are 192 for -# messages, 320 for snowflakes and 64 elsewhere, and they govern every service -# Compose does not forward this to. A router holds a slot for the whole round -# trip to its shard, so this is a ceiling on requests in flight at once and not a -# rate: too low a value does not slow requests down, it rejects them. The api -# turns that rejection into a 503 and logs "shard rejected the request because -# it is at its concurrency limit". +# In-flight request ceiling for the users and messages routers and their shards. +# One value replaces the built-in default on all of them, so size it for the +# busiest. Too low a value rejects requests rather than slowing them, and the api +# turns that into a 503. #FLUXER_SVC_MAX_CONCURRENT_REQUESTS=192 -# The api and the Rust services name their fixed Postgres statement shapes so the -# server can reuse their plans. Named prepared statements require a session that -# outlives the transaction, so set this to false if you put a transaction-pooling -# connection pooler such as PgBouncer in front of Postgres. One setting governs -# every service. The bundled compose talks to Postgres directly, where naming is -# a win and the default is correct. +# Named prepared statements need a session that outlives the transaction, so set +# this to false behind a transaction-pooling connection pooler. The bundled +# compose talks to Postgres directly, where the default is correct. #FLUXER_POSTGRES_PREPARED_STATEMENTS=true -# The api bounds how long a client may take to send a request. The header timeout -# covers the request line and headers only, while the request timeout covers the -# whole exchange, so a slow uploader is bounded by the second value and not by -# the first. Raise both if you front large uploads or serve clients on high -# latency links. The header timeout is clamped down to the request timeout, so -# raising it alone does nothing. Both are milliseconds, between 1000 and 3600000. +# How long a client may take to send a request. The header timeout covers the +# request line and headers, the request timeout the whole exchange, and the first +# is clamped down to the second. Milliseconds, 1000 to 3600000. #FLUXER_API_HEADERS_TIMEOUT_MS=30000 #FLUXER_API_REQUEST_TIMEOUT_MS=120000 diff --git a/fluxer_api/src/api/attachment/AttachmentUrls.ts b/fluxer_api/src/api/attachment/AttachmentUrls.ts index ae0c5ec65..a9cce1df3 100644 --- a/fluxer_api/src/api/attachment/AttachmentUrls.ts +++ b/fluxer_api/src/api/attachment/AttachmentUrls.ts @@ -51,10 +51,6 @@ export function signDataPackageAttachmentUrl(url: string, nowSecs?: number): str return options === null ? url : signDataPackageWithSecret(url, options); } -export function stripAttachmentSignature(url: string): string { - return stripSignature(url); -} - export function stripOwnAttachmentSignature(url: string): string { return attachmentStorageKeyFromUrl(url, Config.endpoints.media) === null ? url : stripSignature(url); } diff --git a/fluxer_api/src/api/channel/services/message/AttachmentProcessingService.ts b/fluxer_api/src/api/channel/services/message/AttachmentProcessingService.ts index 88b5bcfbe..802564909 100644 --- a/fluxer_api/src/api/channel/services/message/AttachmentProcessingService.ts +++ b/fluxer_api/src/api/channel/services/message/AttachmentProcessingService.ts @@ -1,7 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import fs from 'node:fs'; -import {createAttachmentID, createGuildID, createUserID, type UserID} from '@app/api/BrandedTypes'; +import {createAttachmentID, type UserID} from '@app/api/BrandedTypes'; import {Config} from '@app/api/Config'; import type {AttachmentToProcess} from '@app/api/channel/AttachmentDTOs'; import type {AttachmentUploadTraceRepository} from '@app/api/channel/repositories/message/AttachmentUploadTraceRepository'; @@ -11,7 +11,6 @@ import { makeAttachmentCdnKey, validateAttachmentIds, } from '@app/api/channel/services/message/MessageHelpers'; -import {scheduleUploadSegmentSignal} from '@app/api/channel/services/message/UploadSegmentSignal'; import type {MessageAttachment} from '@app/api/database/types/MessageTypes'; import {contentModerationService, type ModerationContext} from '@app/api/infrastructure/ContentModerationService'; import type { @@ -174,24 +173,6 @@ export class AttachmentProcessingService { } return result.attachment; }); - scheduleUploadSegmentSignal({ - userId: params.uploadUserId, - guildId: params.guild ? createGuildID(BigInt(params.guild.id)) : null, - guildOwnerId: params.guild ? createUserID(BigInt(params.guild.owner_id)) : null, - channelId: params.message.channelId, - messageId: params.message.id, - attachments: processedAttachments.map((attachment, index) => ({ - attachmentId: attachment.attachment_id, - uploadKey: results[index].copyOperation.sourceKey, - filename: attachment.filename, - contentType: attachment.content_type, - size: attachment.size, - duration: attachment.duration ?? null, - waveform: attachment.waveform ?? null, - sniffedContentType: results[index].sniffedContentType, - requestIp: bindingResults[index].bound?.request_ip ?? null, - })), - }); return {attachments: processedAttachments, hasVirusDetected: false}; } diff --git a/fluxer_api/src/api/channel/services/message/UploadSegmentSignal.ts b/fluxer_api/src/api/channel/services/message/UploadSegmentSignal.ts deleted file mode 100644 index b5b3b0bb4..000000000 --- a/fluxer_api/src/api/channel/services/message/UploadSegmentSignal.ts +++ /dev/null @@ -1,209 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import type {AttachmentID, ChannelID, GuildID, MessageID, UserID} from '@app/api/BrandedTypes'; -import {Logger} from '@app/api/Logger'; -import {getKVClient} from '@app/api/middleware/ServiceRegistry'; -import {snowflakeToDate} from '@fluxer/snowflake/src/Snowflake'; - -const WINDOW_MS = 600000; -const USER_THRESHOLD = 60; -const GUILD_THRESHOLD = 120; -const FRESH_GUILD_THRESHOLD = 20; -const FRESH_GUILD_MAX_AGE_MS = 604800000; -const SEGMENT_MAX_BYTES = 16 * 1024 * 1024; -const SEGMENT_MAX_DURATION_SECONDS = 30; - -const KEY_PREFIX = 'abuse:upload_segment:'; -const WINDOW_TTL_SECONDS = WINDOW_MS / 1000; -const MAX_IN_FLIGHT_SIGNALS = 32; -const SURFACE = 'message_attachment'; -const SIGNAL_MESSAGE = 'content_moderation.upload_segment_pattern'; -const FAILURE_MESSAGE = 'content_moderation.upload_segment_signal_failed'; -const DROPPED_MESSAGE = 'content_moderation.upload_segment_signal_dropped'; -const PLAYLIST_CONTENT_TYPES = new Set(['application/vnd.apple.mpegurl', 'application/x-mpegurl']); - -export type UploadSegmentScope = 'user' | 'guild' | 'fresh_guild'; - -export interface UploadSegmentAttachment { - attachmentId: AttachmentID; - uploadKey: string; - filename: string; - contentType: string; - size: bigint; - duration: number | null; - waveform: string | null; - sniffedContentType: string | null; - requestIp: string | null; -} - -export interface UploadSegmentSignalInput { - userId: UserID; - guildId: GuildID | null; - guildOwnerId: UserID | null; - channelId: ChannelID; - messageId: MessageID; - attachments: ReadonlyArray; -} - -interface CountedScope { - scope: UploadSegmentScope; - key: string; - threshold: number; -} - -function baseContentType(contentType: string): string { - const [base] = contentType.split(';'); - return (base ?? '').trim().toLowerCase(); -} - -export function isSegmentShapedAttachment(attachment: UploadSegmentAttachment): boolean { - if (attachment.waveform !== null) { - return false; - } - if (attachment.sniffedContentType !== null) { - return true; - } - const contentType = baseContentType(attachment.contentType); - const isSegmentType = - contentType.startsWith('video/') || contentType.startsWith('audio/') || PLAYLIST_CONTENT_TYPES.has(contentType); - if (!isSegmentType) { - return false; - } - if (attachment.size > BigInt(SEGMENT_MAX_BYTES)) { - return false; - } - return attachment.duration === null || attachment.duration <= SEGMENT_MAX_DURATION_SECONDS; -} - -export function isRungValue(count: number, threshold: number): boolean { - if (count < threshold || count % threshold !== 0) { - return false; - } - const multiple = count / threshold; - return (multiple & (multiple - 1)) === 0; -} - -function uploaderOwnsGuild(input: UploadSegmentSignalInput): boolean { - return input.guildOwnerId !== null && input.guildOwnerId === input.userId; -} - -function resolveScopes(input: UploadSegmentSignalInput, windowIndex: number, nowMs: number): Array { - const scopes: Array = [ - { - scope: 'user', - key: `${KEY_PREFIX}user:${input.userId.toString()}:${windowIndex}`, - threshold: USER_THRESHOLD, - }, - ]; - if (input.guildId === null) { - return scopes; - } - const guildAgeMs = nowMs - snowflakeToDate(input.guildId).getTime(); - const isFreshGuild = uploaderOwnsGuild(input) && guildAgeMs < FRESH_GUILD_MAX_AGE_MS; - scopes.push({ - scope: isFreshGuild ? 'fresh_guild' : 'guild', - key: `${KEY_PREFIX}guild:${input.guildId.toString()}:${windowIndex}`, - threshold: isFreshGuild ? FRESH_GUILD_THRESHOLD : GUILD_THRESHOLD, - }); - return scopes; -} - -function buildSignalFields(params: { - input: UploadSegmentSignalInput; - segments: ReadonlyArray; - windowIndex: number; - scope: CountedScope; - count: number; -}): Record { - const {input, segments, windowIndex, scope, count} = params; - const requestIps = new Set(); - for (const segment of segments) { - if (segment.requestIp !== null) { - requestIps.add(segment.requestIp); - } - } - return { - surface: SURFACE, - scope: scope.scope, - count, - threshold: scope.threshold, - windowMs: WINDOW_MS, - windowStartedAt: new Date(windowIndex * WINDOW_MS).toISOString(), - userId: input.userId.toString(), - userCreatedAt: snowflakeToDate(input.userId).toISOString(), - guildId: input.guildId === null ? null : input.guildId.toString(), - guildCreatedAt: input.guildId === null ? null : snowflakeToDate(input.guildId).toISOString(), - guildOwnerId: input.guildOwnerId === null ? null : input.guildOwnerId.toString(), - uploaderOwnsGuild: uploaderOwnsGuild(input), - channelId: input.channelId.toString(), - messageId: input.messageId.toString(), - attachmentIds: segments.map((segment) => segment.attachmentId.toString()), - uploadKeys: segments.map((segment) => segment.uploadKey), - requestIps: Array.from(requestIps), - filenames: segments.map((segment) => segment.filename), - contentTypes: segments.map((segment) => segment.contentType), - disguisedCount: segments.filter((segment) => segment.sniffedContentType !== null).length, - }; -} - -function scopeFields(input: UploadSegmentSignalInput): Record { - return { - surface: SURFACE, - userId: input.userId.toString(), - guildId: input.guildId === null ? null : input.guildId.toString(), - channelId: input.channelId.toString(), - messageId: input.messageId.toString(), - }; -} - -const inFlightSignals = new Set>(); - -export function scheduleUploadSegmentSignal(input: UploadSegmentSignalInput): void { - if (!input.attachments.some(isSegmentShapedAttachment)) { - return; - } - if (inFlightSignals.size >= MAX_IN_FLIGHT_SIGNALS) { - Logger.warn(scopeFields(input), DROPPED_MESSAGE); - return; - } - const pending = recordUploadSegmentSignal(input).finally(() => { - inFlightSignals.delete(pending); - }); - inFlightSignals.add(pending); -} - -export async function flushUploadSegmentSignals(): Promise { - while (inFlightSignals.size > 0) { - await Promise.all([...inFlightSignals]); - } -} - -export async function recordUploadSegmentSignal(input: UploadSegmentSignalInput): Promise { - try { - const segments = input.attachments.filter(isSegmentShapedAttachment); - if (segments.length === 0) { - return; - } - const nowMs = Date.now(); - const windowIndex = Math.floor(nowMs / WINDOW_MS); - const scopes = resolveScopes(input, windowIndex, nowMs); - const kv = getKVClient(); - const counts: Array = []; - for (const scope of scopes) { - const count = await kv.incr(scope.key); - counts.push(count); - if (count === 1) { - await kv.expire(scope.key, WINDOW_TTL_SECONDS); - } - } - for (const [index, scope] of scopes.entries()) { - const count = counts[index]; - if (!isRungValue(count, scope.threshold)) { - continue; - } - Logger.warn(buildSignalFields({input, segments, windowIndex, scope, count}), SIGNAL_MESSAGE); - } - } catch (error) { - Logger.warn({error, ...scopeFields(input)}, FAILURE_MESSAGE); - } -}