mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-07 19:22:14 +09:00
fix(api): drop the upload segment signal and dead exports (#2831)
This commit is contained in:
@@ -1,108 +1,64 @@
|
|||||||
# Every variable docker-compose.yml reads is named here, uncommented when it has
|
# Every variable docker-compose.yml reads, uncommented when it has no default and
|
||||||
# no default and commented with its default when it has one. A name absent from
|
# commented with its default when it has one. Compose expands top to bottom, so a
|
||||||
# this file reaches a service only through a Compose override. Compose expands
|
# line using ${...} must sit below every name it reads.
|
||||||
# top to bottom, so a line using ${...} must sit below every name it reads.
|
|
||||||
|
|
||||||
FLUXER_DOMAIN=chat.example.com
|
FLUXER_DOMAIN=chat.example.com
|
||||||
FLUXER_PUBLIC_SCHEME=https
|
FLUXER_PUBLIC_SCHEME=https
|
||||||
FLUXER_PUBLIC_PORT=443
|
FLUXER_PUBLIC_PORT=443
|
||||||
|
|
||||||
# The lines above are the address browsers use, and every advertised endpoint
|
# The address browsers use. FLUXER_HTTP_PORT and FLUXER_HTTPS_PORT below decide
|
||||||
# carries FLUXER_PUBLIC_PORT. They do not move what the host publishes.
|
# which host ports Fluxer binds.
|
||||||
# FLUXER_HTTP_PORT and FLUXER_HTTPS_PORT below do that, and a non-default port
|
|
||||||
# needs the matching one set as well. Complete recipes sit beside them.
|
|
||||||
|
|
||||||
# How browsers reach this instance.
|
# By default Fluxer binds 80 and 443 and gets its own certificate. Point DNS here.
|
||||||
#
|
# Behind your own reverse proxy, uncomment this instead: Fluxer then serves plain
|
||||||
# Default: Fluxer binds 80 and 443 and gets its own Let's Encrypt certificate.
|
# HTTP on 127.0.0.1:8080. Keep the scheme and port above describing the public
|
||||||
# Point DNS at this host.
|
# address, not this one.
|
||||||
#
|
|
||||||
# Behind your own reverse proxy (nginx, Traefik, HAProxy, Cloudflare Tunnel,
|
|
||||||
# another Caddy): uncomment COMPOSE_FILE below. Fluxer then serves plain HTTP on
|
|
||||||
# 127.0.0.1:8080 instead, and your proxy forwards everything to it. Keep
|
|
||||||
# FLUXER_PUBLIC_SCHEME and FLUXER_PUBLIC_PORT describing the PUBLIC address your
|
|
||||||
# proxy serves, not this local port.
|
|
||||||
#COMPOSE_FILE=docker-compose.yml:docker-compose.proxy.yml
|
#COMPOSE_FILE=docker-compose.yml:docker-compose.proxy.yml
|
||||||
|
|
||||||
# Where the plain-HTTP port binds when the proxy overlay is in use. Leave it on
|
# Where that plain-HTTP port binds. Use 0.0.0.0:8080 only when the proxy is on
|
||||||
# loopback when the proxy runs on this host. Use 0.0.0.0:8080 only when the proxy
|
# another machine, and firewall it to that machine.
|
||||||
# is on another machine, and firewall the port to that machine.
|
|
||||||
#FLUXER_EDGE_BIND=127.0.0.1:8080
|
#FLUXER_EDGE_BIND=127.0.0.1:8080
|
||||||
|
|
||||||
# Which upstream hops may set X-Forwarded-For. Fluxer rewrites the header from
|
# Which hops may set X-Forwarded-For. The default covers private and loopback
|
||||||
# this to the real client address, so IP bans, rate limits and abuse detection
|
# addresses. Set your proxy's address if it reaches Fluxer from a public IP.
|
||||||
# see the caller rather than the proxy. The default covers proxies on private or
|
|
||||||
# loopback addresses, which is every same-host setup. Set it to your proxy's
|
|
||||||
# address if it reaches Fluxer from a public IP.
|
|
||||||
#FLUXER_EDGE_TRUSTED_PROXIES=private_ranges
|
#FLUXER_EDGE_TRUSTED_PROXIES=private_ranges
|
||||||
|
|
||||||
# The origin browsers see, without a trailing slash. Leave it unset and each
|
# The origin browsers see, no trailing slash. Set it when browsers reach the
|
||||||
# service builds one from the three values at the top of this file. Set it and it
|
# instance on a host FLUXER_DOMAIN does not name, and it wins over the three
|
||||||
# wins: every service reads the host, the scheme and the port out of it and
|
# values above. Scheme, host and optional port only. It does not move the
|
||||||
# ignores those three names. Use it when browsers reach the instance on a host
|
# published ports.
|
||||||
# FLUXER_DOMAIN does not name. It has to be a bare origin, a scheme and a host
|
|
||||||
# and an optional port and nothing after them, or the services refuse to start.
|
|
||||||
# It does not move the edge listener or the published ports either, so set the
|
|
||||||
# publish below to the port written here.
|
|
||||||
#FLUXER_PUBLIC_ORIGIN=https://chat.example.com
|
#FLUXER_PUBLIC_ORIGIN=https://chat.example.com
|
||||||
|
|
||||||
# Overrides the address the edge listens on inside its container. Compose builds
|
# The address the edge listens on inside its container. Both proxy overlays set
|
||||||
# it from FLUXER_PUBLIC_SCHEME and FLUXER_DOMAIN with no port, and the edge keeps
|
# this themselves, so a value here is ignored under either. Include the scheme.
|
||||||
# its container ports at 80 and 443 whatever the public port is. Caddy matches a
|
|
||||||
# site by host and ignores the port in the Host header, so a request arriving on
|
|
||||||
# a non-default published port still lands on this site. Put a port in this value
|
|
||||||
# only if you also publish that same container port below, or nothing will be
|
|
||||||
# listening where the publish points. Honoured in the default mode only:
|
|
||||||
# docker-compose.proxy.yml sets the literal :8080 and tunnel.compose.yml the
|
|
||||||
# literal :80, and Compose lets the last file win, so a value here is discarded
|
|
||||||
# under either overlay with no warning. Set it for an unusual default-mode
|
|
||||||
# layout, such as serving several hostnames. Write the scheme into it: a bare
|
|
||||||
# hostname means automatic HTTPS on 443 whatever FLUXER_PUBLIC_SCHEME says.
|
|
||||||
#FLUXER_EDGE_SITE_ADDRESS=https://chat.example.com
|
#FLUXER_EDGE_SITE_ADDRESS=https://chat.example.com
|
||||||
|
|
||||||
# The old name for the value above, read only when FLUXER_EDGE_SITE_ADDRESS is
|
# The old name for the line above, read only when it is unset.
|
||||||
# unset, so an existing .env keeps the listener it already had.
|
|
||||||
#FLUXER_CADDY_SITE_ADDRESS=
|
#FLUXER_CADDY_SITE_ADDRESS=
|
||||||
|
|
||||||
# Host side of the edge's publishes, and the only names that decide which host
|
# Host ports. Container 80 handles the redirect and the certificate challenge,
|
||||||
# ports Fluxer binds. The container side is fixed. Container 80 carries the
|
# container 443 the TLS site. FLUXER_HTTPS_PORT moves TCP and UDP together, since
|
||||||
# HTTP to HTTPS redirect and the Let's Encrypt HTTP challenge under an https
|
# HTTP/3 needs both. Both accept a bind address. Give them different host ports.
|
||||||
# scheme, and the site itself under an http one. Container 443 carries the TLS
|
|
||||||
# site. FLUXER_HTTPS_PORT moves the TCP and the UDP publish together, because
|
|
||||||
# HTTP/3 needs both on the same port. Both take an optional bind address in front
|
|
||||||
# of the port, and 127.0.0.1 keeps the publish off every public interface. Give
|
|
||||||
# them different host ports: the same host port on both is two publishes of one
|
|
||||||
# port and the edge refuses to start.
|
|
||||||
#FLUXER_HTTP_PORT=80
|
#FLUXER_HTTP_PORT=80
|
||||||
#FLUXER_HTTPS_PORT=443
|
#FLUXER_HTTPS_PORT=443
|
||||||
#FLUXER_HTTP_PORT=127.0.0.1:80
|
#FLUXER_HTTP_PORT=127.0.0.1:80
|
||||||
#FLUXER_HTTPS_PORT=127.0.0.1:443
|
#FLUXER_HTTPS_PORT=127.0.0.1:443
|
||||||
|
|
||||||
# HTTPS on 8443, complete. Host 80 stays published and still answers the ACME
|
# HTTPS on 8443. Host 80 stays published for the certificate challenge, which
|
||||||
# challenge. Let's Encrypt only ever connects to the public 80 or 443, so the
|
# only ever arrives on public 80 or 443. Serve your own certificate if nothing
|
||||||
# certificate is issued if a router in front forwards public 80 to this host and
|
# forwards those.
|
||||||
# is not issued otherwise. Serve your own certificate from the Caddyfile when it
|
|
||||||
# cannot.
|
|
||||||
#FLUXER_PUBLIC_PORT=8443
|
#FLUXER_PUBLIC_PORT=8443
|
||||||
#FLUXER_HTTPS_PORT=8443
|
#FLUXER_HTTPS_PORT=8443
|
||||||
|
|
||||||
# Plain HTTP on 19080, complete. The port 80 publish moves to 19080, so nothing
|
# Plain HTTP on 19080. Nothing binds host 80, and the last line parks the idle
|
||||||
# binds host 80. Under an http scheme nothing listens on container 443, so the
|
# 443 publish on loopback.
|
||||||
# last line parks that publish on loopback for a host that wants 443 for
|
|
||||||
# something else. Drop it and 443 is published and idle, which is what earlier
|
|
||||||
# releases did.
|
|
||||||
#FLUXER_PUBLIC_SCHEME=http
|
#FLUXER_PUBLIC_SCHEME=http
|
||||||
#FLUXER_PUBLIC_PORT=19080
|
#FLUXER_PUBLIC_PORT=19080
|
||||||
#FLUXER_HTTP_PORT=19080
|
#FLUXER_HTTP_PORT=19080
|
||||||
#FLUXER_HTTPS_PORT=127.0.0.1:443
|
#FLUXER_HTTPS_PORT=127.0.0.1:443
|
||||||
|
|
||||||
# A tunnel or another proxy in front of the stack needs no HTTPS publish at all.
|
# A tunnel needs no HTTPS publish. tunnel.compose.yml ships beside this file and
|
||||||
# tunnel.compose.yml ships beside this file and replaces Caddy's published ports
|
# leaves one loopback HTTP publish. Needs Compose 2.24.4 or newer.
|
||||||
# with a single loopback HTTP publish, so nothing binds 443, and points the edge
|
|
||||||
# at plain HTTP on that publish so it stops redirecting to https. FLUXER_HTTP_PORT
|
|
||||||
# still moves that one publish. Set the line below and plain docker compose
|
|
||||||
# commands pick the file up, or add it to your own -f flags if you pass any. The
|
|
||||||
# file uses the !override tag, which needs Compose 2.24.4 or newer.
|
|
||||||
#COMPOSE_FILE=docker-compose.yml:tunnel.compose.yml
|
#COMPOSE_FILE=docker-compose.yml:tunnel.compose.yml
|
||||||
|
|
||||||
FLUXER_REGISTRY_OWNER=fluxerapp
|
FLUXER_REGISTRY_OWNER=fluxerapp
|
||||||
@@ -111,11 +67,8 @@ FLUXER_IMAGE_TAG=v1
|
|||||||
|
|
||||||
POSTGRES_PASSWORD=CHANGE_ME
|
POSTGRES_PASSWORD=CHANGE_ME
|
||||||
MEILI_MASTER_KEY=CHANGE_ME
|
MEILI_MASTER_KEY=CHANGE_ME
|
||||||
# The stack ships its own Postgres and its own object store, and points at both
|
# Set these to run Postgres or the object store outside the stack. Backing up a
|
||||||
# by service name. Set these to run either one outside the stack. Leave them
|
# store you moved out is yours to arrange, and an upgrade skips it.
|
||||||
# unset and the bundled services are used. Taking a service out of the stack
|
|
||||||
# means an upgrade skips the backup step that reaches into it, and backing that
|
|
||||||
# store up belongs to whoever runs it.
|
|
||||||
#FLUXER_POSTGRES_HOST=db.example.com
|
#FLUXER_POSTGRES_HOST=db.example.com
|
||||||
#FLUXER_POSTGRES_PORT=5432
|
#FLUXER_POSTGRES_PORT=5432
|
||||||
#FLUXER_POSTGRES_DATABASE=fluxer
|
#FLUXER_POSTGRES_DATABASE=fluxer
|
||||||
@@ -125,8 +78,7 @@ MEILI_MASTER_KEY=CHANGE_ME
|
|||||||
#FLUXER_S3_PUBLIC_ENDPOINT=https://cdn.example.com
|
#FLUXER_S3_PUBLIC_ENDPOINT=https://cdn.example.com
|
||||||
#FLUXER_S3_REGION=eu-central-1
|
#FLUXER_S3_REGION=eu-central-1
|
||||||
#FLUXER_S3_FORCE_PATH_STYLE=false
|
#FLUXER_S3_FORCE_PATH_STYLE=false
|
||||||
# Bucket names. The bundled object store creates whichever names these hold, so
|
# Bucket names. The bundled store creates these. An outside store needs them to
|
||||||
# the two stay in step. An object store outside the stack needs the buckets to
|
|
||||||
# exist already.
|
# exist already.
|
||||||
#FLUXER_S3_BUCKET_CDN=fluxer
|
#FLUXER_S3_BUCKET_CDN=fluxer
|
||||||
#FLUXER_S3_BUCKET_UPLOADS=fluxer-uploads
|
#FLUXER_S3_BUCKET_UPLOADS=fluxer-uploads
|
||||||
@@ -134,10 +86,8 @@ MEILI_MASTER_KEY=CHANGE_ME
|
|||||||
#FLUXER_S3_BUCKET_REPORTS=fluxer-reports
|
#FLUXER_S3_BUCKET_REPORTS=fluxer-reports
|
||||||
#FLUXER_S3_BUCKET_HARVESTS=fluxer-harvests
|
#FLUXER_S3_BUCKET_HARVESTS=fluxer-harvests
|
||||||
|
|
||||||
# The rest of the bundled services, pointed somewhere else the same way. Leave a
|
# The other bundled services, pointed elsewhere. Removing a service from the
|
||||||
# line unset and the service in the stack is used. Taking a service out of the
|
# stack belongs in an override file, since an upgrade replaces docker-compose.yml.
|
||||||
# stack goes in an override file listed in COMPOSE_FILE, because an upgrade
|
|
||||||
# replaces docker-compose.yml.
|
|
||||||
#FLUXER_KV_URL=redis://cache.example.com:6379/0
|
#FLUXER_KV_URL=redis://cache.example.com:6379/0
|
||||||
#FLUXER_NATS_URL=nats://mq.example.com:4222
|
#FLUXER_NATS_URL=nats://mq.example.com:4222
|
||||||
#FLUXER_NATS_JETSTREAM_URL=nats://mq.example.com:4222
|
#FLUXER_NATS_JETSTREAM_URL=nats://mq.example.com:4222
|
||||||
@@ -145,24 +95,21 @@ MEILI_MASTER_KEY=CHANGE_ME
|
|||||||
#FLUXER_SEARCH_URL=https://search.example.com
|
#FLUXER_SEARCH_URL=https://search.example.com
|
||||||
#FLUXER_LIVEKIT_INTERNAL_URL=http://livekit.example.com:7880
|
#FLUXER_LIVEKIT_INTERNAL_URL=http://livekit.example.com:7880
|
||||||
|
|
||||||
# Voice off. The livekit service still runs until an override file takes it out.
|
# Voice off. The livekit service still runs until an override removes it.
|
||||||
#FLUXER_LIVEKIT_ENABLED=false
|
#FLUXER_LIVEKIT_ENABLED=false
|
||||||
|
|
||||||
# Optional systems, each off unless the instance is configured for it.
|
# Optional systems, each off unless configured.
|
||||||
#FLUXER_SMS_ENABLED=false
|
#FLUXER_SMS_ENABLED=false
|
||||||
#FLUXER_STRIPE_ENABLED=false
|
#FLUXER_STRIPE_ENABLED=false
|
||||||
#FLUXER_NCMEC_ENABLED=false
|
#FLUXER_NCMEC_ENABLED=false
|
||||||
#FLUXER_CLAMAV_ENABLED=false
|
#FLUXER_CLAMAV_ENABLED=false
|
||||||
|
|
||||||
# The client address. Set the header name a proxy in front actually writes, and
|
# The client address. Name the header your proxy actually writes, and turn the
|
||||||
# turn the trust off when nothing sits in front, because a trusted header an
|
# trust off when nothing sits in front.
|
||||||
# attacker can set is a spoofed client address.
|
|
||||||
#FLUXER_CLIENT_IP_HEADER_NAME=cf-connecting-ip
|
#FLUXER_CLIENT_IP_HEADER_NAME=cf-connecting-ip
|
||||||
#FLUXER_TRUST_CLIENT_IP_HEADER=true
|
#FLUXER_TRUST_CLIENT_IP_HEADER=true
|
||||||
|
|
||||||
# How much the services write. trace, debug, info, warn, error or fatal. Every
|
# How much the services write. trace, debug, info, warn, error or fatal.
|
||||||
# service names the object storage endpoint and its addressing at info on start,
|
|
||||||
# so a bucket that answers 404 is visible without raising this.
|
|
||||||
#LOG_LEVEL=debug
|
#LOG_LEVEL=debug
|
||||||
|
|
||||||
FLUXER_S3_ACCESS_KEY=fluxer
|
FLUXER_S3_ACCESS_KEY=fluxer
|
||||||
@@ -177,62 +124,43 @@ FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64=CHANGE_ME
|
|||||||
FLUXER_ADMIN_SECRET_KEY_BASE=CHANGE_ME
|
FLUXER_ADMIN_SECRET_KEY_BASE=CHANGE_ME
|
||||||
FLUXER_ADMIN_OAUTH_CLIENT_SECRET=CHANGE_ME
|
FLUXER_ADMIN_OAUTH_CLIENT_SECRET=CHANGE_ME
|
||||||
|
|
||||||
# The token every service sends to NATS. The bundled NATS runs without
|
# The token every service sends to NATS. The bundled NATS needs none, so this
|
||||||
# authentication, so this stays empty unless a Compose override points the stack
|
# stays empty unless an override points at an external one.
|
||||||
# at an external NATS that requires a token. Compose forwards the name to every
|
|
||||||
# container that connects.
|
|
||||||
#FLUXER_NATS_AUTH_TOKEN=
|
#FLUXER_NATS_AUTH_TOKEN=
|
||||||
|
|
||||||
FLUXER_VAPID_PUBLIC_KEY=CHANGE_ME
|
FLUXER_VAPID_PUBLIC_KEY=CHANGE_ME
|
||||||
FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME
|
FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME
|
||||||
|
|
||||||
# The VAPID contact address defaults to admin@ followed by FLUXER_DOMAIN. Set it
|
# Defaults to admin@ followed by FLUXER_DOMAIN. Set it if that mailbox does not
|
||||||
# only if that mailbox does not exist.
|
# exist.
|
||||||
#[email protected]
|
#[email protected]
|
||||||
|
|
||||||
# Passkeys follow FLUXER_DOMAIN by default. Set these only if browsers reach the
|
# Passkeys follow FLUXER_DOMAIN. Set these only if browsers use another host.
|
||||||
# instance on a different host, and note that changing FLUXER_PASSKEY_RP_ID
|
# Changing the RP ID invalidates every passkey registered against the old value.
|
||||||
# invalidates every passkey already registered against the old value.
|
|
||||||
#FLUXER_PASSKEY_RP_ID=chat.example.com
|
#FLUXER_PASSKEY_RP_ID=chat.example.com
|
||||||
#FLUXER_PASSKEY_RP_NAME=Fluxer
|
#FLUXER_PASSKEY_RP_NAME=Fluxer
|
||||||
#FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS=https://chat.example.com
|
#FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS=https://chat.example.com
|
||||||
#FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS=http://chat.example.com:19080
|
#FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS=http://chat.example.com:19080
|
||||||
|
|
||||||
# Two optional media policies, both off unless you turn them on. Nothing below is
|
# Optional media policies, both off by default. See the operator docs.
|
||||||
# needed for a working instance, and an upgrade never adds any of it.
|
|
||||||
#
|
#
|
||||||
# The first limits which web origins may read media through CORS. A request with
|
# CORS limits which web origins may read media. A request with no Origin is
|
||||||
# no Origin header is always served, so direct links, image tags and native
|
# always served. Add https://web.fluxer.app if people use the hosted client.
|
||||||
# clients keep working. The allowlist defaults to the public origin above, which
|
|
||||||
# is where this instance serves its web app. The hosted web client and the
|
|
||||||
# desktop app run on https://web.fluxer.app, so add that origin, as in the second
|
|
||||||
# allowlist line, if people use them with this instance.
|
|
||||||
#
|
#
|
||||||
# The second makes an attachment read need a signed URL, which stops a copied
|
# Signatures make an attachment read need a signed URL, so a copied link stops
|
||||||
# link working forever elsewhere. Turning it on takes two settings: a secret, and
|
# working. Needs a secret from openssl rand -base64 32, first entry signs and
|
||||||
# the mode. Generate the secret with openssl rand -base64 32. It is a comma
|
# every entry verifies.
|
||||||
# separated list, the first entry signs and every entry verifies. To rotate, add
|
|
||||||
# the new secret second and run docker compose up -d, then move it first and run
|
|
||||||
# again. Remove the old secret no sooner than a day after that, because an
|
|
||||||
# ordinary URL it signed stays valid for up to a day. A data package URL inside a
|
|
||||||
# harvest export never expires, so removing a secret ends every data package URL
|
|
||||||
# it signed and those exports have to be rebuilt.
|
|
||||||
#
|
#
|
||||||
# Each mode is off, report or enforce on its own. Set a mode to report first to
|
# Each mode is off, report or enforce. Start at report. media-proxy reads these
|
||||||
# log what enforce would refuse while refusing nothing. media-proxy reads these
|
# at start, so apply with docker compose up -d media-proxy.
|
||||||
# at container start, so apply a change with docker compose up -d media-proxy.
|
|
||||||
# docker compose restart media-proxy keeps the old environment.
|
|
||||||
#FLUXER_MEDIA_PROXY_CORS_MODE=enforce
|
#FLUXER_MEDIA_PROXY_CORS_MODE=enforce
|
||||||
#FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS=https://chat.example.com
|
|
||||||
#FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS=https://chat.example.com,https://web.fluxer.app
|
#FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS=https://chat.example.com,https://web.fluxer.app
|
||||||
#FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64=
|
#FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64=
|
||||||
#FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE=enforce
|
#FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE=enforce
|
||||||
|
|
||||||
# Extra Content-Security-Policy sources, appended to the built-in ones. Set these
|
# Extra Content-Security-Policy sources, appended to the built-in ones. Set one
|
||||||
# only when a browser must reach an origin the defaults do not cover, such as a
|
# only when a browser must reach an origin the defaults do not cover. Separate
|
||||||
# voice server hosted on a domain other than FLUXER_DOMAIN. Separate several
|
# several with spaces or commas. The three values below are illustrations.
|
||||||
# sources with spaces or commas. Every one of them is empty by default, and the
|
|
||||||
# three carrying a value below are illustrations, not defaults.
|
|
||||||
#FLUXER_CSP_EXTRA_DEFAULT_SRC=
|
#FLUXER_CSP_EXTRA_DEFAULT_SRC=
|
||||||
#FLUXER_CSP_EXTRA_CONNECT_SRC=wss://livekit.example.com:7881
|
#FLUXER_CSP_EXTRA_CONNECT_SRC=wss://livekit.example.com:7881
|
||||||
#FLUXER_CSP_EXTRA_IMG_SRC=https://cdn.example.com
|
#FLUXER_CSP_EXTRA_IMG_SRC=https://cdn.example.com
|
||||||
@@ -244,39 +172,29 @@ FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME
|
|||||||
#FLUXER_CSP_EXTRA_WORKER_SRC=
|
#FLUXER_CSP_EXTRA_WORKER_SRC=
|
||||||
#FLUXER_CSP_EXTRA_MANIFEST_SRC=
|
#FLUXER_CSP_EXTRA_MANIFEST_SRC=
|
||||||
|
|
||||||
# One report-uri for Content-Security-Policy violation reports. Empty leaves the
|
# One report-uri for CSP violation reports. Empty leaves the directive off.
|
||||||
# directive off the header.
|
|
||||||
#FLUXER_CSP_REPORT_URI=
|
#FLUXER_CSP_REPORT_URI=
|
||||||
|
|
||||||
# Allow the SSO identity provider to resolve to a private or internal address.
|
# Let the SSO provider resolve to a private address. Off by default, so a
|
||||||
# Off by default: the API refuses to call non-public addresses so a misconfigured
|
# misconfigured provider URL cannot reach internal services. Turn it on only for
|
||||||
# provider URL cannot be used to reach internal services. Turn it on only when the
|
# a provider on your own network.
|
||||||
# provider genuinely lives on your own network, such as split-horizon DNS or a LAN
|
|
||||||
# identity provider, and only when you trust everyone who can configure SSO.
|
|
||||||
#FLUXER_SSO_ALLOW_PRIVATE_ADDRESSES=true
|
#FLUXER_SSO_ALLOW_PRIVATE_ADDRESSES=true
|
||||||
|
|
||||||
# Both reach LiveKit as LIVEKIT_KEYS and the webhook signing key, and the API as
|
# These reach both LiveKit and the api. Change them together.
|
||||||
# FLUXER_LIVEKIT_API_KEY and FLUXER_LIVEKIT_API_SECRET. Change them together.
|
|
||||||
LIVEKIT_API_KEY=fluxer
|
LIVEKIT_API_KEY=fluxer
|
||||||
LIVEKIT_API_SECRET=CHANGE_ME
|
LIVEKIT_API_SECRET=CHANGE_ME
|
||||||
|
|
||||||
# The URL browsers use for voice signalling. Compose builds it from
|
# The URL browsers use for voice signalling. Built from the public origin plus
|
||||||
# FLUXER_PUBLIC_ORIGIN, or from FLUXER_PUBLIC_SCHEME, FLUXER_DOMAIN and
|
# /livekit. Set it only when LiveKit is served from another host.
|
||||||
# FLUXER_PUBLIC_PORT, as that origin followed by /livekit. The client rewrites a
|
|
||||||
# leading http to ws itself. Set it only when LiveKit is served from another
|
|
||||||
# host.
|
|
||||||
#FLUXER_LIVEKIT_URL=
|
#FLUXER_LIVEKIT_URL=
|
||||||
|
|
||||||
# Media ports. LiveKit advertises these in ICE candidates, so the host must
|
# Media ports. LiveKit advertises these, so forward the same numbers.
|
||||||
# forward the same numbers.
|
|
||||||
#FLUXER_LIVEKIT_TCP_PORT=7881
|
#FLUXER_LIVEKIT_TCP_PORT=7881
|
||||||
#FLUXER_LIVEKIT_UDP_PORT=7882
|
#FLUXER_LIVEKIT_UDP_PORT=7882
|
||||||
|
|
||||||
# LiveKit finds the address browsers dial by asking a STUN server. A host that
|
# LiveKit finds its public address over STUN. A host that cannot reach one stops
|
||||||
# cannot reach one over UDP stops with "could not resolve external IP", and the
|
# with "could not resolve external IP", so set the address by hand instead, or
|
||||||
# address is then set by hand: put it in FLUXER_LIVEKIT_NODE_IP and set
|
# point STUN elsewhere.
|
||||||
# FLUXER_LIVEKIT_USE_EXTERNAL_IP to false. Point the STUN entries at another
|
|
||||||
# server to keep the lookup and leave Google out of it.
|
|
||||||
#FLUXER_LIVEKIT_USE_EXTERNAL_IP=false
|
#FLUXER_LIVEKIT_USE_EXTERNAL_IP=false
|
||||||
#FLUXER_LIVEKIT_NODE_IP=203.0.113.10
|
#FLUXER_LIVEKIT_NODE_IP=203.0.113.10
|
||||||
#FLUXER_LIVEKIT_STUN_PRIMARY=stun.l.google.com:19302
|
#FLUXER_LIVEKIT_STUN_PRIMARY=stun.l.google.com:19302
|
||||||
@@ -303,11 +221,9 @@ FLUXER_CAPTCHA_TURNSTILE_SITE_KEY=
|
|||||||
FLUXER_CAPTCHA_TURNSTILE_SECRET_KEY=
|
FLUXER_CAPTCHA_TURNSTILE_SECRET_KEY=
|
||||||
FLUXER_DISCOVERY_ENABLED=true
|
FLUXER_DISCOVERY_ENABLED=true
|
||||||
|
|
||||||
# Container memory. The limits sum to 18.25 GiB, which is a sum of ceilings and
|
# Container memory. These are ceilings, not allocations, and the defaults suit a
|
||||||
# not an allocation, so the defaults fit a host with 8 GB and are sized for 16 GB.
|
# 16 GB host. The reservations bias the kernel away from reclaiming from services
|
||||||
# The reservations are cgroup memory.low, which biases the kernel away from
|
# whose death takes the instance down. Lower the limits on a smaller host.
|
||||||
# reclaiming from services whose death takes the instance down. They reserve
|
|
||||||
# nothing. Lower the limits on a smaller host.
|
|
||||||
#FLUXER_CADDY_MEMORY_LIMIT=256mb
|
#FLUXER_CADDY_MEMORY_LIMIT=256mb
|
||||||
#FLUXER_POSTGRES_MEMORY_LIMIT=5gb
|
#FLUXER_POSTGRES_MEMORY_LIMIT=5gb
|
||||||
#FLUXER_POSTGRES_MEMORY_RESERVATION=3gb
|
#FLUXER_POSTGRES_MEMORY_RESERVATION=3gb
|
||||||
@@ -338,32 +254,22 @@ FLUXER_DISCOVERY_ENABLED=true
|
|||||||
#FLUXER_UNFURL_SHARD_MEMORY_LIMIT=256mb
|
#FLUXER_UNFURL_SHARD_MEMORY_LIMIT=256mb
|
||||||
#FLUXER_ADMIN_MEMORY_LIMIT=256mb
|
#FLUXER_ADMIN_MEMORY_LIMIT=256mb
|
||||||
|
|
||||||
# Meilisearch indexing memory. Keep it well under FLUXER_MEILISEARCH_MEMORY_LIMIT,
|
# Meilisearch indexing memory. Keep it well under the container limit above.
|
||||||
# which is the container ceiling the indexer shares with the search process.
|
|
||||||
#FLUXER_MEILISEARCH_MAX_INDEXING_MEMORY=384mb
|
#FLUXER_MEILISEARCH_MAX_INDEXING_MEMORY=384mb
|
||||||
|
|
||||||
# SeaweedFS heap ceiling. Go collects against this value instead of against the
|
# SeaweedFS heap ceiling. Go cannot see the container limit, so without this an
|
||||||
# container limit, which it cannot see, so without it an upload burst grows the
|
# upload burst gets the container OOM-killed. Keep it near three quarters of
|
||||||
# heap past FLUXER_SEAWEEDFS_MEMORY_LIMIT and the kernel OOM-kills the container
|
# FLUXER_SEAWEEDFS_MEMORY_LIMIT and raise both together.
|
||||||
# mid-upload (exit 137). Keep it near three quarters of that limit, and raise both
|
|
||||||
# together: the peak is the parts of one upload in flight at once, which is 25 MB
|
|
||||||
# times 20 for a 500 MB attachment.
|
|
||||||
#FLUXER_SEAWEEDFS_GOMEMLIMIT=1536MiB
|
#FLUXER_SEAWEEDFS_GOMEMLIMIT=1536MiB
|
||||||
|
|
||||||
# Node sizes its own heap from the container memory limit by default, at roughly
|
# Node sizes its heap from the container limit by default. Leave these unset
|
||||||
# 55 percent of it, which always leaves room for the buffers and stacks that live
|
# unless you need to pin it. A heap ceiling above the container limit gets the
|
||||||
# outside the heap. Leave these unset unless you have a reason to pin the value.
|
# container OOM-killed instead of reporting a heap error.
|
||||||
# Any value set here must stay well below the container limit above: a heap ceiling
|
|
||||||
# above the container limit makes the kernel OOM-kill the container (exit 137, no
|
|
||||||
# diagnostics) instead of Node reporting a JavaScript heap out of memory error.
|
|
||||||
#FLUXER_API_NODE_HEAP_MB=1792
|
#FLUXER_API_NODE_HEAP_MB=1792
|
||||||
#FLUXER_WORKER_NODE_HEAP_MB=1792
|
#FLUXER_WORKER_NODE_HEAP_MB=1792
|
||||||
|
|
||||||
# Bundled Postgres tuning. Keep these consistent with FLUXER_POSTGRES_MEMORY_LIMIT:
|
# Bundled Postgres tuning. Keep it consistent with the memory limit above. This
|
||||||
# budget roughly shared_buffers + (server max_connections x 12 MB) +
|
# is the server setting, not the per-service pool sizes.
|
||||||
# (3 x autovacuum_work_mem) + 300 MB for page cache and WAL. Note this is the
|
|
||||||
# server setting, distinct from the per-service FLUXER_POSTGRES_MAX_CONNECTIONS
|
|
||||||
# pool sizes used by the api, worker and shards.
|
|
||||||
#FLUXER_POSTGRES_SERVER_MAX_CONNECTIONS=150
|
#FLUXER_POSTGRES_SERVER_MAX_CONNECTIONS=150
|
||||||
#FLUXER_POSTGRES_SHARED_BUFFERS=512MB
|
#FLUXER_POSTGRES_SHARED_BUFFERS=512MB
|
||||||
#FLUXER_POSTGRES_EFFECTIVE_CACHE_SIZE=2GB
|
#FLUXER_POSTGRES_EFFECTIVE_CACHE_SIZE=2GB
|
||||||
@@ -371,48 +277,30 @@ FLUXER_DISCOVERY_ENABLED=true
|
|||||||
#FLUXER_POSTGRES_MAINTENANCE_WORK_MEM=256MB
|
#FLUXER_POSTGRES_MAINTENANCE_WORK_MEM=256MB
|
||||||
#FLUXER_POSTGRES_AUTOVACUUM_WORK_MEM=128MB
|
#FLUXER_POSTGRES_AUTOVACUUM_WORK_MEM=128MB
|
||||||
|
|
||||||
# The bundled Valkey holds durable state as well as cache. The bulk message
|
# The bundled Valkey holds durable state as well as cache, so it runs with an
|
||||||
# deletion queue and the account deletion queue are sorted sets with no expiry,
|
# append-only file and with noeviction, which fails an over-limit write instead
|
||||||
# and nothing else stores the first of the two. It therefore runs with an
|
# of dropping queued work. Change the policy only if that state lives elsewhere.
|
||||||
# append-only file on a named volume and with noeviction, so an over-limit write
|
|
||||||
# fails loudly instead of silently deleting queued work. Distributed locks all
|
|
||||||
# carry a TTL and are not what the durability is for. Only change the policy if
|
|
||||||
# you have moved that durable state elsewhere.
|
|
||||||
#FLUXER_VALKEY_MAXMEMORY=192mb
|
#FLUXER_VALKEY_MAXMEMORY=192mb
|
||||||
#FLUXER_VALKEY_MAXMEMORY_POLICY=noeviction
|
#FLUXER_VALKEY_MAXMEMORY_POLICY=noeviction
|
||||||
|
|
||||||
# The gateway derives its BEAM scheduler count from the container CPU quota,
|
# The gateway derives its scheduler count from the CPU quota, clamped here. One
|
||||||
# clamped to this range. The floor matters: a single scheduler lets one blocking
|
# scheduler lets a single blocking operation stall every websocket on the node.
|
||||||
# operation stall every websocket on the node. The ceiling stops a large host
|
|
||||||
# from starting far more schedulers than the container can actually use.
|
|
||||||
#FLUXER_ERLANG_SCHEDULERS_MIN=2
|
#FLUXER_ERLANG_SCHEDULERS_MIN=2
|
||||||
#FLUXER_ERLANG_SCHEDULERS_MAX=16
|
#FLUXER_ERLANG_SCHEDULERS_MAX=16
|
||||||
|
|
||||||
# In-flight request ceiling for the services Compose forwards it to: the users
|
# In-flight request ceiling for the users and messages routers and their shards.
|
||||||
# and messages routers and their shards. Leave it unset and each service uses its
|
# One value replaces the built-in default on all of them, so size it for the
|
||||||
# built-in default. Set it and the one value replaces that default on all of
|
# busiest. Too low a value rejects requests rather than slowing them, and the api
|
||||||
# them, so size it for the busiest. The built-in defaults are 192 for
|
# turns that into a 503.
|
||||||
# messages, 320 for snowflakes and 64 elsewhere, and they govern every service
|
|
||||||
# Compose does not forward this to. A router holds a slot for the whole round
|
|
||||||
# trip to its shard, so this is a ceiling on requests in flight at once and not a
|
|
||||||
# rate: too low a value does not slow requests down, it rejects them. The api
|
|
||||||
# turns that rejection into a 503 and logs "shard rejected the request because
|
|
||||||
# it is at its concurrency limit".
|
|
||||||
#FLUXER_SVC_MAX_CONCURRENT_REQUESTS=192
|
#FLUXER_SVC_MAX_CONCURRENT_REQUESTS=192
|
||||||
|
|
||||||
# The api and the Rust services name their fixed Postgres statement shapes so the
|
# Named prepared statements need a session that outlives the transaction, so set
|
||||||
# server can reuse their plans. Named prepared statements require a session that
|
# this to false behind a transaction-pooling connection pooler. The bundled
|
||||||
# outlives the transaction, so set this to false if you put a transaction-pooling
|
# compose talks to Postgres directly, where the default is correct.
|
||||||
# connection pooler such as PgBouncer in front of Postgres. One setting governs
|
|
||||||
# every service. The bundled compose talks to Postgres directly, where naming is
|
|
||||||
# a win and the default is correct.
|
|
||||||
#FLUXER_POSTGRES_PREPARED_STATEMENTS=true
|
#FLUXER_POSTGRES_PREPARED_STATEMENTS=true
|
||||||
|
|
||||||
# The api bounds how long a client may take to send a request. The header timeout
|
# How long a client may take to send a request. The header timeout covers the
|
||||||
# covers the request line and headers only, while the request timeout covers the
|
# request line and headers, the request timeout the whole exchange, and the first
|
||||||
# whole exchange, so a slow uploader is bounded by the second value and not by
|
# is clamped down to the second. Milliseconds, 1000 to 3600000.
|
||||||
# the first. Raise both if you front large uploads or serve clients on high
|
|
||||||
# latency links. The header timeout is clamped down to the request timeout, so
|
|
||||||
# raising it alone does nothing. Both are milliseconds, between 1000 and 3600000.
|
|
||||||
#FLUXER_API_HEADERS_TIMEOUT_MS=30000
|
#FLUXER_API_HEADERS_TIMEOUT_MS=30000
|
||||||
#FLUXER_API_REQUEST_TIMEOUT_MS=120000
|
#FLUXER_API_REQUEST_TIMEOUT_MS=120000
|
||||||
|
|||||||
@@ -51,10 +51,6 @@ export function signDataPackageAttachmentUrl(url: string, nowSecs?: number): str
|
|||||||
return options === null ? url : signDataPackageWithSecret(url, options);
|
return options === null ? url : signDataPackageWithSecret(url, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function stripAttachmentSignature(url: string): string {
|
|
||||||
return stripSignature(url);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function stripOwnAttachmentSignature(url: string): string {
|
export function stripOwnAttachmentSignature(url: string): string {
|
||||||
return attachmentStorageKeyFromUrl(url, Config.endpoints.media) === null ? url : stripSignature(url);
|
return attachmentStorageKeyFromUrl(url, Config.endpoints.media) === null ? url : stripSignature(url);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
import fs from 'node:fs';
|
import fs from 'node:fs';
|
||||||
import {createAttachmentID, createGuildID, createUserID, type UserID} from '@app/api/BrandedTypes';
|
import {createAttachmentID, type UserID} from '@app/api/BrandedTypes';
|
||||||
import {Config} from '@app/api/Config';
|
import {Config} from '@app/api/Config';
|
||||||
import type {AttachmentToProcess} from '@app/api/channel/AttachmentDTOs';
|
import type {AttachmentToProcess} from '@app/api/channel/AttachmentDTOs';
|
||||||
import type {AttachmentUploadTraceRepository} from '@app/api/channel/repositories/message/AttachmentUploadTraceRepository';
|
import type {AttachmentUploadTraceRepository} from '@app/api/channel/repositories/message/AttachmentUploadTraceRepository';
|
||||||
@@ -11,7 +11,6 @@ import {
|
|||||||
makeAttachmentCdnKey,
|
makeAttachmentCdnKey,
|
||||||
validateAttachmentIds,
|
validateAttachmentIds,
|
||||||
} from '@app/api/channel/services/message/MessageHelpers';
|
} from '@app/api/channel/services/message/MessageHelpers';
|
||||||
import {scheduleUploadSegmentSignal} from '@app/api/channel/services/message/UploadSegmentSignal';
|
|
||||||
import type {MessageAttachment} from '@app/api/database/types/MessageTypes';
|
import type {MessageAttachment} from '@app/api/database/types/MessageTypes';
|
||||||
import {contentModerationService, type ModerationContext} from '@app/api/infrastructure/ContentModerationService';
|
import {contentModerationService, type ModerationContext} from '@app/api/infrastructure/ContentModerationService';
|
||||||
import type {
|
import type {
|
||||||
@@ -174,24 +173,6 @@ export class AttachmentProcessingService {
|
|||||||
}
|
}
|
||||||
return result.attachment;
|
return result.attachment;
|
||||||
});
|
});
|
||||||
scheduleUploadSegmentSignal({
|
|
||||||
userId: params.uploadUserId,
|
|
||||||
guildId: params.guild ? createGuildID(BigInt(params.guild.id)) : null,
|
|
||||||
guildOwnerId: params.guild ? createUserID(BigInt(params.guild.owner_id)) : null,
|
|
||||||
channelId: params.message.channelId,
|
|
||||||
messageId: params.message.id,
|
|
||||||
attachments: processedAttachments.map((attachment, index) => ({
|
|
||||||
attachmentId: attachment.attachment_id,
|
|
||||||
uploadKey: results[index].copyOperation.sourceKey,
|
|
||||||
filename: attachment.filename,
|
|
||||||
contentType: attachment.content_type,
|
|
||||||
size: attachment.size,
|
|
||||||
duration: attachment.duration ?? null,
|
|
||||||
waveform: attachment.waveform ?? null,
|
|
||||||
sniffedContentType: results[index].sniffedContentType,
|
|
||||||
requestIp: bindingResults[index].bound?.request_ip ?? null,
|
|
||||||
})),
|
|
||||||
});
|
|
||||||
return {attachments: processedAttachments, hasVirusDetected: false};
|
return {attachments: processedAttachments, hasVirusDetected: false};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,209 +0,0 @@
|
|||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
import type {AttachmentID, ChannelID, GuildID, MessageID, UserID} from '@app/api/BrandedTypes';
|
|
||||||
import {Logger} from '@app/api/Logger';
|
|
||||||
import {getKVClient} from '@app/api/middleware/ServiceRegistry';
|
|
||||||
import {snowflakeToDate} from '@fluxer/snowflake/src/Snowflake';
|
|
||||||
|
|
||||||
const WINDOW_MS = 600000;
|
|
||||||
const USER_THRESHOLD = 60;
|
|
||||||
const GUILD_THRESHOLD = 120;
|
|
||||||
const FRESH_GUILD_THRESHOLD = 20;
|
|
||||||
const FRESH_GUILD_MAX_AGE_MS = 604800000;
|
|
||||||
const SEGMENT_MAX_BYTES = 16 * 1024 * 1024;
|
|
||||||
const SEGMENT_MAX_DURATION_SECONDS = 30;
|
|
||||||
|
|
||||||
const KEY_PREFIX = 'abuse:upload_segment:';
|
|
||||||
const WINDOW_TTL_SECONDS = WINDOW_MS / 1000;
|
|
||||||
const MAX_IN_FLIGHT_SIGNALS = 32;
|
|
||||||
const SURFACE = 'message_attachment';
|
|
||||||
const SIGNAL_MESSAGE = 'content_moderation.upload_segment_pattern';
|
|
||||||
const FAILURE_MESSAGE = 'content_moderation.upload_segment_signal_failed';
|
|
||||||
const DROPPED_MESSAGE = 'content_moderation.upload_segment_signal_dropped';
|
|
||||||
const PLAYLIST_CONTENT_TYPES = new Set(['application/vnd.apple.mpegurl', 'application/x-mpegurl']);
|
|
||||||
|
|
||||||
export type UploadSegmentScope = 'user' | 'guild' | 'fresh_guild';
|
|
||||||
|
|
||||||
export interface UploadSegmentAttachment {
|
|
||||||
attachmentId: AttachmentID;
|
|
||||||
uploadKey: string;
|
|
||||||
filename: string;
|
|
||||||
contentType: string;
|
|
||||||
size: bigint;
|
|
||||||
duration: number | null;
|
|
||||||
waveform: string | null;
|
|
||||||
sniffedContentType: string | null;
|
|
||||||
requestIp: string | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface UploadSegmentSignalInput {
|
|
||||||
userId: UserID;
|
|
||||||
guildId: GuildID | null;
|
|
||||||
guildOwnerId: UserID | null;
|
|
||||||
channelId: ChannelID;
|
|
||||||
messageId: MessageID;
|
|
||||||
attachments: ReadonlyArray<UploadSegmentAttachment>;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface CountedScope {
|
|
||||||
scope: UploadSegmentScope;
|
|
||||||
key: string;
|
|
||||||
threshold: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
function baseContentType(contentType: string): string {
|
|
||||||
const [base] = contentType.split(';');
|
|
||||||
return (base ?? '').trim().toLowerCase();
|
|
||||||
}
|
|
||||||
|
|
||||||
export function isSegmentShapedAttachment(attachment: UploadSegmentAttachment): boolean {
|
|
||||||
if (attachment.waveform !== null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (attachment.sniffedContentType !== null) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
const contentType = baseContentType(attachment.contentType);
|
|
||||||
const isSegmentType =
|
|
||||||
contentType.startsWith('video/') || contentType.startsWith('audio/') || PLAYLIST_CONTENT_TYPES.has(contentType);
|
|
||||||
if (!isSegmentType) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (attachment.size > BigInt(SEGMENT_MAX_BYTES)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return attachment.duration === null || attachment.duration <= SEGMENT_MAX_DURATION_SECONDS;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function isRungValue(count: number, threshold: number): boolean {
|
|
||||||
if (count < threshold || count % threshold !== 0) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
const multiple = count / threshold;
|
|
||||||
return (multiple & (multiple - 1)) === 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
function uploaderOwnsGuild(input: UploadSegmentSignalInput): boolean {
|
|
||||||
return input.guildOwnerId !== null && input.guildOwnerId === input.userId;
|
|
||||||
}
|
|
||||||
|
|
||||||
function resolveScopes(input: UploadSegmentSignalInput, windowIndex: number, nowMs: number): Array<CountedScope> {
|
|
||||||
const scopes: Array<CountedScope> = [
|
|
||||||
{
|
|
||||||
scope: 'user',
|
|
||||||
key: `${KEY_PREFIX}user:${input.userId.toString()}:${windowIndex}`,
|
|
||||||
threshold: USER_THRESHOLD,
|
|
||||||
},
|
|
||||||
];
|
|
||||||
if (input.guildId === null) {
|
|
||||||
return scopes;
|
|
||||||
}
|
|
||||||
const guildAgeMs = nowMs - snowflakeToDate(input.guildId).getTime();
|
|
||||||
const isFreshGuild = uploaderOwnsGuild(input) && guildAgeMs < FRESH_GUILD_MAX_AGE_MS;
|
|
||||||
scopes.push({
|
|
||||||
scope: isFreshGuild ? 'fresh_guild' : 'guild',
|
|
||||||
key: `${KEY_PREFIX}guild:${input.guildId.toString()}:${windowIndex}`,
|
|
||||||
threshold: isFreshGuild ? FRESH_GUILD_THRESHOLD : GUILD_THRESHOLD,
|
|
||||||
});
|
|
||||||
return scopes;
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildSignalFields(params: {
|
|
||||||
input: UploadSegmentSignalInput;
|
|
||||||
segments: ReadonlyArray<UploadSegmentAttachment>;
|
|
||||||
windowIndex: number;
|
|
||||||
scope: CountedScope;
|
|
||||||
count: number;
|
|
||||||
}): Record<string, unknown> {
|
|
||||||
const {input, segments, windowIndex, scope, count} = params;
|
|
||||||
const requestIps = new Set<string>();
|
|
||||||
for (const segment of segments) {
|
|
||||||
if (segment.requestIp !== null) {
|
|
||||||
requestIps.add(segment.requestIp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
surface: SURFACE,
|
|
||||||
scope: scope.scope,
|
|
||||||
count,
|
|
||||||
threshold: scope.threshold,
|
|
||||||
windowMs: WINDOW_MS,
|
|
||||||
windowStartedAt: new Date(windowIndex * WINDOW_MS).toISOString(),
|
|
||||||
userId: input.userId.toString(),
|
|
||||||
userCreatedAt: snowflakeToDate(input.userId).toISOString(),
|
|
||||||
guildId: input.guildId === null ? null : input.guildId.toString(),
|
|
||||||
guildCreatedAt: input.guildId === null ? null : snowflakeToDate(input.guildId).toISOString(),
|
|
||||||
guildOwnerId: input.guildOwnerId === null ? null : input.guildOwnerId.toString(),
|
|
||||||
uploaderOwnsGuild: uploaderOwnsGuild(input),
|
|
||||||
channelId: input.channelId.toString(),
|
|
||||||
messageId: input.messageId.toString(),
|
|
||||||
attachmentIds: segments.map((segment) => segment.attachmentId.toString()),
|
|
||||||
uploadKeys: segments.map((segment) => segment.uploadKey),
|
|
||||||
requestIps: Array.from(requestIps),
|
|
||||||
filenames: segments.map((segment) => segment.filename),
|
|
||||||
contentTypes: segments.map((segment) => segment.contentType),
|
|
||||||
disguisedCount: segments.filter((segment) => segment.sniffedContentType !== null).length,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function scopeFields(input: UploadSegmentSignalInput): Record<string, unknown> {
|
|
||||||
return {
|
|
||||||
surface: SURFACE,
|
|
||||||
userId: input.userId.toString(),
|
|
||||||
guildId: input.guildId === null ? null : input.guildId.toString(),
|
|
||||||
channelId: input.channelId.toString(),
|
|
||||||
messageId: input.messageId.toString(),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const inFlightSignals = new Set<Promise<void>>();
|
|
||||||
|
|
||||||
export function scheduleUploadSegmentSignal(input: UploadSegmentSignalInput): void {
|
|
||||||
if (!input.attachments.some(isSegmentShapedAttachment)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (inFlightSignals.size >= MAX_IN_FLIGHT_SIGNALS) {
|
|
||||||
Logger.warn(scopeFields(input), DROPPED_MESSAGE);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const pending = recordUploadSegmentSignal(input).finally(() => {
|
|
||||||
inFlightSignals.delete(pending);
|
|
||||||
});
|
|
||||||
inFlightSignals.add(pending);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function flushUploadSegmentSignals(): Promise<void> {
|
|
||||||
while (inFlightSignals.size > 0) {
|
|
||||||
await Promise.all([...inFlightSignals]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function recordUploadSegmentSignal(input: UploadSegmentSignalInput): Promise<void> {
|
|
||||||
try {
|
|
||||||
const segments = input.attachments.filter(isSegmentShapedAttachment);
|
|
||||||
if (segments.length === 0) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const nowMs = Date.now();
|
|
||||||
const windowIndex = Math.floor(nowMs / WINDOW_MS);
|
|
||||||
const scopes = resolveScopes(input, windowIndex, nowMs);
|
|
||||||
const kv = getKVClient();
|
|
||||||
const counts: Array<number> = [];
|
|
||||||
for (const scope of scopes) {
|
|
||||||
const count = await kv.incr(scope.key);
|
|
||||||
counts.push(count);
|
|
||||||
if (count === 1) {
|
|
||||||
await kv.expire(scope.key, WINDOW_TTL_SECONDS);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (const [index, scope] of scopes.entries()) {
|
|
||||||
const count = counts[index];
|
|
||||||
if (!isRungValue(count, scope.threshold)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
Logger.warn(buildSignalFields({input, segments, windowIndex, scope, count}), SIGNAL_MESSAGE);
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
Logger.warn({error, ...scopeFields(input)}, FAILURE_MESSAGE);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user