diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index faacb9163..0ab67b416 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -23,7 +23,6 @@ services: FLUXER_S3_PUBLIC_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}" FLUXER_LIVEKIT_URL: "ws://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/livekit" FLUXER_LIVEKIT_INTERNAL_URL: "http://livekit:7880" - FLUXER_LIVEKIT_WEBHOOK_URL: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/api/webhooks/livekit" FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media" FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media" FLUXER_GATEWAY_STATIC_CDN_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}" diff --git a/Cargo.lock b/Cargo.lock index 9f0a27f98..bb31045e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1904,7 +1904,6 @@ dependencies = [ "thiserror", "tokio", "tracing", - "tracing-subscriber", "url", ] @@ -2067,6 +2066,7 @@ dependencies = [ "thiserror", "time", "tracing", + "tracing-subscriber", "url", "urlencoding", ] diff --git a/config/env/development.env b/config/env/development.env index f4948cd56..175931913 100644 --- a/config/env/development.env +++ b/config/env/development.env @@ -34,7 +34,6 @@ FLUXER_KV_URL=redis://valkey:6379/0 FLUXER_NATS_URL=nats://nats:4222 FLUXER_NATS_JETSTREAM_URL=nats://nats:4222 FLUXER_INTERNAL_API_ENDPOINT=http://127.0.0.1:8080 -FLUXER_INTERNAL_GATEWAY_ENDPOINT=http://127.0.0.1:8771 FLUXER_INTERNAL_MEDIA_PROXY_ENDPOINT=http://127.0.0.1:8082 FLUXER_MEDIA_PROXY_ENDPOINT=http://127.0.0.1:8082 FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT=http://localhost:8088/media @@ -42,7 +41,6 @@ FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT=http://localhost:8088/media FLUXER_SVC_NATS_URL=nats://nats:4222 FLUXER_SVC_SHARD_COUNT=1 FLUXER_SVC_CACHE_TTL_MS=30000 -FLUXER_SVC_CACHE_HARD_TTL_MS=600000 FLUXER_S3_ENDPOINT=http://127.0.0.1:8333 FLUXER_S3_PUBLIC_ENDPOINT=http://localhost:8088 @@ -61,7 +59,6 @@ FLUXER_LIVEKIT_URL=ws://localhost:8088/livekit FLUXER_LIVEKIT_INTERNAL_URL=http://localhost:7880 FLUXER_LIVEKIT_API_KEY=devkey FLUXER_LIVEKIT_API_SECRET=fluxer-livekit-development-secret -FLUXER_LIVEKIT_WEBHOOK_URL=http://localhost:8088/api/webhooks/livekit FLUXER_LIVEKIT_DEFAULT_REGION={"id":"local","name":"Local","emoji":"LC","latitude":59.3293,"longitude":18.0686} FLUXER_API_PORT=8080 @@ -128,6 +125,3 @@ PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT=http://localhost:8088/api FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64=Zmx1eGVyLWRldi11cGxvYWQtcmVsYXktc2VjcmV0LTAwMDA= FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64=Zmx1eGVyLWRldi1hdHRhY2htZW50LXVybC1zZWNyZXQ= AWS_EC2_METADATA_DISABLED=true -AWS_ACCESS_KEY_ID=fluxer -AWS_SECRET_ACCESS_KEY=fluxer-secret -AWS_DEFAULT_REGION=us-east-1 diff --git a/deploy/self-hosting/.env.example b/deploy/self-hosting/.env.example index 44f127296..e3d4d24d7 100644 --- a/deploy/self-hosting/.env.example +++ b/deploy/self-hosting/.env.example @@ -1,6 +1,8 @@ -# Every variable docker-compose.yml reads, uncommented when it has no default and -# commented with its default when it has one. Compose expands top to bottom, so a -# line using ${...} must sit below every name it reads. +# Every variable docker-compose.yml reads. A value an install must set is +# uncommented. A commented line shows the default, or an example where its comment +# says so, and nothing after = means the service decides. An empty value keeps the +# default too. Compose expands top to bottom, so a line using ${...} must sit below +# every name it reads. FLUXER_DOMAIN=chat.example.com FLUXER_PUBLIC_SCHEME=https @@ -68,7 +70,9 @@ FLUXER_IMAGE_TAG=v1 POSTGRES_PASSWORD=CHANGE_ME MEILI_MASTER_KEY=CHANGE_ME # Set these to run Postgres or the object store outside the stack. Backing up a -# store you moved out is yours to arrange, and an upgrade skips it. +# store you moved out is yours to arrange. An upgrade dumps the bundled postgres +# service and skips the dump only when the stack defines none. The values below +# are examples. #FLUXER_POSTGRES_HOST=db.example.com #FLUXER_POSTGRES_PORT=5432 #FLUXER_POSTGRES_DATABASE=fluxer @@ -78,6 +82,7 @@ MEILI_MASTER_KEY=CHANGE_ME #FLUXER_S3_PUBLIC_ENDPOINT=https://cdn.example.com #FLUXER_S3_REGION=eu-central-1 #FLUXER_S3_FORCE_PATH_STYLE=false + # Bucket names. The bundled store creates these. An outside store needs them to # exist already. #FLUXER_S3_BUCKET_CDN=fluxer @@ -90,34 +95,89 @@ MEILI_MASTER_KEY=CHANGE_ME # Needs Compose 2.24.4 or newer. #COMPOSE_FILE=docker-compose.yml:external-object-store.compose.yml +# A full connection URL wins over the host, port and database above. The URL is an +# example. The CA is the PEM text of the certificate, with \n for line breaks. +#FLUXER_POSTGRES_URL=postgres://fluxer:secret@db.example.com:5432/fluxer +#FLUXER_POSTGRES_SSL_CA= +# The Postgres table that holds the key-value store. +#FLUXER_POSTGRES_KV_TABLE=fluxer_kv +# media-proxy reads through these when the store serves reads from another +# address or bucket. +#FLUXER_S3_READ_ENDPOINT= +#FLUXER_S3_READ_BUCKET= +#FLUXER_S3_READ_BUCKET_STYLE= +# A temporary S3 session token, read by media-proxy only. +#FLUXER_S3_SESSION_TOKEN= +# The bundled store refuses unsigned reads. Set false only for a public-read bucket. +#FLUXER_S3_READ_SIGNED=true + # The other bundled services, pointed elsewhere. Removing a service from the # stack belongs in an override file, since an upgrade replaces docker-compose.yml. +# The URLs below are examples. #FLUXER_KV_URL=redis://cache.example.com:6379/0 #FLUXER_NATS_URL=nats://mq.example.com:4222 #FLUXER_NATS_JETSTREAM_URL=nats://mq.example.com:4222 #FLUXER_SVC_NATS_URL=nats://mq.example.com:4222 #FLUXER_SEARCH_URL=https://search.example.com #FLUXER_LIVEKIT_INTERNAL_URL=http://livekit.example.com:7880 +# How the stack talks to those services. +#FLUXER_KV_MODE=standalone +#FLUXER_SEARCH_ENGINE=meilisearch +#FLUXER_SEARCH_USERNAME= +#FLUXER_SEARCH_PASSWORD= +#FLUXER_SEARCH_TLS_REJECT_UNAUTHORIZED=true # Voice off. The livekit service still runs until an override removes it. #FLUXER_LIVEKIT_ENABLED=false # Optional systems, each off unless configured. #FLUXER_STRIPE_ENABLED=false -#FLUXER_NCMEC_ENABLED=false -#FLUXER_CLAMAV_ENABLED=false +#FLUXER_STRIPE_SECRET_KEY= +#FLUXER_STRIPE_WEBHOOK_SECRET= +# Stripe prices as one JSON object. The admin dashboard can set them instead. +#FLUXER_STRIPE_PRICES={} +#FLUXER_STRIPE_LEGACY_PRICES={} +#FLUXER_API_DONATION_PROXY_KEY= +#FLUXER_VISIONARIES_GUILD_ID= +#FLUXER_VISIONARIES_GUILD_VISIONARY_ROLE_ID= + +# NCMEC CyberTipline reporting, off by default. All four values are required +# once it is on. The values below are examples. +#FLUXER_NCMEC_ENABLED=true +#FLUXER_NCMEC_BASE_URL=https://report.cybertip.org/ispws +#FLUXER_NCMEC_USERNAME= +#FLUXER_NCMEC_PASSWORD= +#FLUXER_NCMEC_REPORTER_EMAIL=trust@example.com + +# Upload virus scanning, off by default. No ClamAV container ships, so point +# this at your own. The values below are examples. +#FLUXER_CLAMAV_ENABLED=true +#FLUXER_CLAMAV_HOST=clamav +#FLUXER_CLAMAV_PORT=3310 +#FLUXER_CLAMAV_FAIL_OPEN=false # Outside lookups, off unless turned on. The breached password check asks # api.pwnedpasswords.com. -#FLUXER_BREACHED_PASSWORD_CHECK_ENABLED=true +#FLUXER_BREACHED_PASSWORD_CHECK_ENABLED=false +#FLUXER_BLOCKLIST_FEEDS_ENABLED=false +#FLUXER_IPINFO_API_KEY= +# A local path, or an s3:// URL read with the S3 credentials of this file. +#FLUXER_GEOIP_DB_PATH= -# The client address. Name the header your proxy actually writes, and turn the -# trust off when nothing sits in front. -#FLUXER_CLIENT_IP_HEADER_NAME=cf-connecting-ip +# The client address. The edge sets X-Forwarded-For on every hop, so keep the +# trust on and the default header. Turning the trust off makes the api refuse +# every request outside its exempt routes with a 403. +#FLUXER_CLIENT_IP_HEADER_NAME=x-forwarded-for #FLUXER_TRUST_CLIENT_IP_HEADER=true -# How much the services write. trace, debug, info, warn, error or fatal. -#LOG_LEVEL=debug +# How much the services write. LOG_LEVEL covers the api and worker and takes trace, +# debug, info, warn, error or fatal. RUST_LOG covers the Rust services and takes +# an EnvFilter such as debug. The gateway takes an Erlang level such as notice, +# and LOGGER_LEVEL beats FLUXER_GATEWAY_LOGGER_LEVEL. +#LOG_LEVEL=info +#RUST_LOG=info +#FLUXER_GATEWAY_LOGGER_LEVEL=info +#LOGGER_LEVEL= FLUXER_S3_ACCESS_KEY=fluxer FLUXER_S3_SECRET_KEY=CHANGE_ME @@ -142,7 +202,7 @@ FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME # exist. #FLUXER_VAPID_EMAIL=admin@chat.example.com -# Passkeys follow FLUXER_DOMAIN. Set these only if browsers use another host. +# The passkey RP ID defaults to FLUXER_DOMAIN, whatever FLUXER_PUBLIC_ORIGIN says. # Changing the RP ID invalidates every passkey registered against the old value. #FLUXER_PASSKEY_RP_ID=chat.example.com #FLUXER_PASSKEY_RP_NAME=Fluxer @@ -153,6 +213,29 @@ FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME #FLUXER_PUSH_SERVICE_QUEUE_CAPACITY=10000 # Provider requests the push container sends at once, 1 to 65536. #FLUXER_PUSH_SERVICE_SEND_CONCURRENCY=256 +# The push container's provider addresses and relay hosts. +#FLUXER_PUSH_SERVICE_APNS_BASE_URL= +#FLUXER_PUSH_SERVICE_FCM_BASE_URL=https://fcm.googleapis.com +#FLUXER_PUSH_SERVICE_MANAGED_RELAY_HOSTS=push.fluxer.com +#FLUXER_PUSH_SERVICE_OWN_RELAY_HOSTS= +#FLUXER_PUSH_SERVICE_RELAY_CONSENT_ACCEPTED=false + +# Direct mobile push through your own APNs and FCM credentials, off by default. +#FLUXER_PUSH_APNS_ENABLED=false +#FLUXER_PUSH_APNS_TEAM_ID= +#FLUXER_PUSH_APNS_KEY_ID= +#FLUXER_PUSH_APNS_PRIVATE_KEY= +#FLUXER_PUSH_APNS_PRIVATE_KEY_PATH= +#FLUXER_PUSH_APNS_APPS= +#FLUXER_PUSH_APNS_DEFAULT_ENVIRONMENT=production +#FLUXER_PUSH_FCM_ENABLED=false +#FLUXER_PUSH_FCM_PROJECT_ID= +#FLUXER_PUSH_FCM_CLIENT_EMAIL= +#FLUXER_PUSH_FCM_PRIVATE_KEY= +#FLUXER_PUSH_FCM_PRIVATE_KEY_PATH= +#FLUXER_PUSH_FCM_SERVICE_ACCOUNT_JSON_PATH= +#FLUXER_PUSH_FCM_TOKEN_URI=https://oauth2.googleapis.com/token +#FLUXER_PUSH_FCM_APPS= # Optional media policies, both off by default. See the operator docs. @@ -164,8 +247,9 @@ FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME # working. Needs a secret from openssl rand -base64 32, first entry signs and # every entry verifies. # -# Each mode is off, report or enforce. Start at report. media-proxy reads these -# at start, so apply with docker compose up -d media-proxy. +# Each mode is off, report or enforce, and off is the default. Start at report. +# media-proxy reads these at start, so apply with docker compose up -d +# media-proxy. The values below are examples. #FLUXER_MEDIA_PROXY_CORS_MODE=enforce #FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS=https://chat.example.com,https://web.fluxer.app #FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64= @@ -190,7 +274,7 @@ FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME # Let the SSO provider resolve to a private address. Off by default, so a # misconfigured provider URL cannot reach internal services. Turn it on only for -# a provider on your own network. +# a provider on your own network. The value below is an example. #FLUXER_SSO_ALLOW_PRIVATE_ADDRESSES=true # These reach both LiveKit and the api. Change them together. @@ -207,13 +291,20 @@ LIVEKIT_API_SECRET=CHANGE_ME # LiveKit finds its public address over STUN. A host that cannot reach one stops # with "could not resolve external IP", so set the address by hand instead, or -# point STUN elsewhere. +# point STUN elsewhere. The values below are examples. #FLUXER_LIVEKIT_USE_EXTERNAL_IP=false #FLUXER_LIVEKIT_NODE_IP=203.0.113.10 #FLUXER_LIVEKIT_STUN_PRIMARY=stun.l.google.com:19302 #FLUXER_LIVEKIT_STUN_SECONDARY=stun1.l.google.com:19302 +# The voice region users see, and how much LiveKit logs. +#FLUXER_LIVEKIT_DEFAULT_REGION={"id":"default","name":"Default","emoji":"🌍","latitude":0,"longitude":0} +#FLUXER_LIVEKIT_LOG_LEVEL=info + FLUXER_KLIPY_API_KEY= +#FLUXER_YOUTUBE_API_KEY= +# Hosts the api never unfurls, comma separated. +#FLUXER_API_UNFURL_IGNORED_HOSTS= FLUXER_EMAIL_ENABLED=false FLUXER_EMAIL_PROVIDER=none @@ -225,8 +316,93 @@ FLUXER_EMAIL_SMTP_PORT=587 FLUXER_EMAIL_SMTP_USERNAME= FLUXER_EMAIL_SMTP_PASSWORD= FLUXER_EMAIL_SMTP_SECURE=true +#FLUXER_EMAIL_WEBHOOK_SECRET= FLUXER_DISCOVERY_ENABLED=true +#FLUXER_DISCOVERY_MIN_MEMBER_COUNT=1 + +# Instance identity and account policy. +#FLUXER_APP_PRODUCT_NAME=Fluxer +#FLUXER_APP_ICON_URL= +#FLUXER_APP_SYMBOL_URL= +#FLUXER_APP_LOGO_URL= +#FLUXER_APP_WORDMARK_URL= +#FLUXER_APP_FAVICON_URL= +#FLUXER_APP_THEME_COLOR= +#FLUXER_APP_STATUS_PAGE_URL= +#FLUXER_APP_STATUS_PAGE_INCIDENT_HISTORY_URL= +#FLUXER_INSTANCE_SETUP_CONFIGURED=false +#FLUXER_AUTO_JOIN_INVITE_CODE= +#FLUXER_DELETION_GRACE_PERIOD_HOURS=336 + +# Sign in with Bluesky, off unless turned on. +#FLUXER_AUTH_BLUESKY_ENABLED=false +#FLUXER_AUTH_BLUESKY_CLIENT_NAME=Fluxer +#FLUXER_AUTH_BLUESKY_CLIENT_URI= +#FLUXER_AUTH_BLUESKY_LOGO_URI= +#FLUXER_AUTH_BLUESKY_TOS_URI= +#FLUXER_AUTH_BLUESKY_POLICY_URI= +#FLUXER_AUTH_BLUESKY_KEYS= + +# Public addresses. Each follows the public origin unless set here. +#FLUXER_API_ENDPOINT= +#FLUXER_API_CLIENT_ENDPOINT= +#FLUXER_APP_ENDPOINT= +#FLUXER_GATEWAY_ENDPOINT= +#FLUXER_MEDIA_ENDPOINT= +#FLUXER_STATIC_CDN_ENDPOINT= +#FLUXER_ADMIN_ENDPOINT= +#FLUXER_MARKETING_ENDPOINT= +#FLUXER_INVITE_ENDPOINT= +#FLUXER_GIFT_ENDPOINT= +#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT= +#PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT= +# These follow FLUXER_STATIC_CDN_ENDPOINT first, then the public origin. +#FLUXER_GATEWAY_STATIC_CDN_ENDPOINT= +#FLUXER_UNFURL_STATIC_CDN_ENDPOINT= +# These follow FLUXER_MEDIA_ENDPOINT first, then the public origin. +#FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT= +#FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT= + +# Extra hosts for static assets, invites, gifts and the web app. Empty by default. +#FLUXER_STATIC_CDN_DOMAIN= +#FLUXER_INVITE_DOMAIN= +#FLUXER_GIFT_DOMAIN= +#FLUXER_APP_ORIGIN_ALIASES= + +# The path the admin panel is served under. The edge and the admin service read +# it. The api follows it through the default FLUXER_ADMIN_ENDPOINT, and not when +# FLUXER_ADMIN_ENDPOINT is set. Write it with a leading slash and no trailing +# slash. +#FLUXER_ADMIN_BASE_PATH=/admin + +# The compression the edge offers, as Caddy encode arguments. +#FLUXER_EDGE_ENCODE=zstd gzip + +# Images of the bundled services, for a mirror or another tag. A new Postgres +# major needs a dump and restore, as the upgrade guide describes. +#FLUXER_CADDY_IMAGE=caddy:2.11-alpine +#FLUXER_POSTGRES_IMAGE=postgres:16-alpine +#FLUXER_VALKEY_IMAGE=valkey/valkey:9.1-alpine +#FLUXER_NATS_IMAGE=nats:2.14-alpine +#FLUXER_MEILISEARCH_IMAGE=getmeili/meilisearch:v1.53 +#FLUXER_SEAWEEDFS_IMAGE=chrislusf/seaweedfs:4.47 +#FLUXER_LIVEKIT_IMAGE=livekit/livekit-server:v1.12.0 + +# Restart policy for every long-running service. +#FLUXER_RESTART_POLICY=unless-stopped + +# Health checks. Raise the retries or start periods on a slow host. +#FLUXER_HEALTHCHECK_INTERVAL=10s +#FLUXER_HEALTHCHECK_TIMEOUT=5s +#FLUXER_HEALTHCHECK_RETRIES=10 +#FLUXER_APP_HEALTHCHECK_RETRIES=30 +#FLUXER_APP_HEALTHCHECK_START_PERIOD=90s +#FLUXER_SVC_HEALTHCHECK_START_PERIOD=60s +#FLUXER_WORKER_HEALTHCHECK_RETRIES=3 +#FLUXER_SEAWEEDFS_HEALTHCHECK_RETRIES=20 +#FLUXER_SEAWEEDFS_HEALTHCHECK_START_PERIOD=60s +#FLUXER_SEAWEEDFS_INIT_ATTEMPTS=60 # Container memory. These are ceilings, not allocations, and the defaults suit a # 16 GB host. The reservations bias the kernel away from reclaiming from services @@ -264,18 +440,31 @@ FLUXER_DISCOVERY_ENABLED=true # Meilisearch indexing memory. Keep it well under the container limit above. #FLUXER_MEILISEARCH_MAX_INDEXING_MEMORY=384mb +#FLUXER_MEILISEARCH_ENV=production +#FLUXER_MEILISEARCH_NO_ANALYTICS=true # SeaweedFS heap ceiling. Go cannot see the container limit, so without this an # upload burst gets the container OOM-killed. Keep it near three quarters of # FLUXER_SEAWEEDFS_MEMORY_LIMIT and raise both together. #FLUXER_SEAWEEDFS_GOMEMLIMIT=1536MiB +#FLUXER_SEAWEEDFS_TELEMETRY=false # Node sizes its heap from the container limit by default. Leave these unset # unless you need to pin it. A heap ceiling above the container limit gets the -# container OOM-killed instead of reporting a heap error. +# container OOM-killed instead of reporting a heap error. The values below are +# examples. #FLUXER_API_NODE_HEAP_MB=1792 #FLUXER_WORKER_NODE_HEAP_MB=1792 +# Extra Node flags for api and worker, appended to NODE_OPTIONS. Empty by +# default. The value below is an example. +#FLUXER_API_NODE_OPTIONS=--heapsnapshot-near-heap-limit=1 +#FLUXER_WORKER_NODE_OPTIONS=--heapsnapshot-near-heap-limit=1 + +# Extra CA certificates api and worker trust, as a PEM bundle path inside the +# container. The default is the image's system bundle. +#FLUXER_NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt + # Bundled Postgres tuning. Keep it consistent with the memory limit above. This # is the server setting, not the per-service pool sizes. #FLUXER_POSTGRES_SERVER_MAX_CONNECTIONS=150 @@ -285,23 +474,81 @@ FLUXER_DISCOVERY_ENABLED=true #FLUXER_POSTGRES_MAINTENANCE_WORK_MEM=256MB #FLUXER_POSTGRES_AUTOVACUUM_WORK_MEM=128MB #FLUXER_POSTGRES_SHM_SIZE=1gb +#FLUXER_POSTGRES_RANDOM_PAGE_COST=1.1 +#FLUXER_POSTGRES_EFFECTIVE_IO_CONCURRENCY=200 +#FLUXER_POSTGRES_DEFAULT_STATISTICS_TARGET=200 +#FLUXER_POSTGRES_JIT=off +#FLUXER_POSTGRES_MIN_WAL_SIZE=512MB +#FLUXER_POSTGRES_MAX_WAL_SIZE=2GB +#FLUXER_POSTGRES_CHECKPOINT_COMPLETION_TARGET=0.9 +#FLUXER_POSTGRES_WAL_BUFFERS=16MB +#FLUXER_POSTGRES_WAL_COMPRESSION=zstd +#FLUXER_POSTGRES_BGWRITER_DELAY=50ms +#FLUXER_POSTGRES_BGWRITER_LRU_MAXPAGES=1000 +#FLUXER_POSTGRES_AUTOVACUUM_VACUUM_SCALE_FACTOR=0.05 +#FLUXER_POSTGRES_AUTOVACUUM_ANALYZE_SCALE_FACTOR=0.02 +#FLUXER_POSTGRES_AUTOVACUUM_VACUUM_COST_LIMIT=2000 +#FLUXER_POSTGRES_TRACK_IO_TIMING=on +#FLUXER_POSTGRES_SHARED_PRELOAD_LIBRARIES=pg_stat_statements + +# Postgres pool size of each service that opens a pool. +#FLUXER_API_POSTGRES_MAX_CONNECTIONS=25 +#FLUXER_WORKER_POSTGRES_MAX_CONNECTIONS=25 +#FLUXER_USERS_SHARD_POSTGRES_MAX_CONNECTIONS=20 +#FLUXER_MESSAGES_SHARD_POSTGRES_MAX_CONNECTIONS=20 # The bundled Valkey holds durable state as well as cache, so it runs with an # append-only file and with noeviction, which fails an over-limit write instead # of dropping queued work. Change the policy only if that state lives elsewhere. #FLUXER_VALKEY_MAXMEMORY=192mb #FLUXER_VALKEY_MAXMEMORY_POLICY=noeviction +#FLUXER_VALKEY_APPENDFSYNC=everysec # The gateway derives its scheduler count from the CPU quota, clamped here. One # scheduler lets a single blocking operation stall every websocket on the node. #FLUXER_ERLANG_SCHEDULERS_MIN=2 #FLUXER_ERLANG_SCHEDULERS_MAX=16 +# A fixed scheduler count skips the clamp. Dirty CPU schedulers default to two +# thirds of it. +#FLUXER_ERLANG_SCHEDULERS= +#FLUXER_ERLANG_DIRTY_CPU_SCHEDULERS= -# In-flight request ceiling for the users and messages routers and their shards. -# One value replaces the built-in default on all of them, so size it for the -# busiest. Too low a value rejects requests rather than slowing them, and the api -# turns that into a 503. -#FLUXER_SVC_MAX_CONCURRENT_REQUESTS=192 +# Gateway push and RPC tuning. +#FLUXER_GATEWAY_PUSH_ENABLED=true +#FLUXER_GATEWAY_PUSH_ENROLLED_CLEAR_NOTIFICATIONS_ENABLED=true +#FLUXER_GATEWAY_PUSH_OUTBOX_REQUEST_TIMEOUT_MS=100000 +#FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_ENTRIES=128 +#FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_BYTES=1048576 +#FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY=512 +#FLUXER_GATEWAY_NATS_RPC_MAX_HANDLERS=512 +#FLUXER_GATEWAY_HTTP_FAILURE_THRESHOLD=6 +#FLUXER_GATEWAY_HTTP_RECOVERY_TIMEOUT_MS=15000 + +# In-flight request ceiling for every svc router and shard. Unset, each keeps its +# own default: 192 for messages, 320 for snowflakes and 64 for the rest. One value +# replaces all of them, so size it for the busiest. Too low a value rejects +# requests rather than slowing them, and the api turns that into a 503. The value +# below is an example. +#FLUXER_SVC_MAX_CONCURRENT_REQUESTS=320 + +# svc caches, and how the api calls the svc services over NATS. +#FLUXER_SVC_CACHE_MAX_ENTRIES=100000 +#FLUXER_SVC_CACHE_TTL_MS=30000 +#FLUXER_GIFS_SHARD_CACHE_MAX_BYTES=536870912 +#FLUXER_GIF_SERVICE_NATS_CLIENT_NAME=fluxer-api-gifs +#FLUXER_GIF_SERVICE_TIMEOUT_MS=12000 +#FLUXER_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS=3000 +#FLUXER_USERS_SERVICE_NATS_CLIENT_NAME=fluxer-api-users +#FLUXER_USERS_SERVICE_TIMEOUT_MS=6000 +#FLUXER_USERS_SERVICE_INFLIGHT_MAX_ENTRIES=10000 +#FLUXER_SNOWFLAKE_SERVICE_NATS_CLIENT_NAME=fluxer-api-snowflakes +#FLUXER_SNOWFLAKE_SERVICE_BATCH_SIZE=128 +#FLUXER_SNOWFLAKE_SERVICE_LOW_WATERMARK= +#FLUXER_SNOWFLAKE_SERVICE_MAX_BUFFER_AGE_MS=5000 +#FLUXER_SNOWFLAKE_SERVICE_REQUEST_TIMEOUT_MS=6000 + +# Worker concurrency per lane, as a JSON object keyed by lane. +#FLUXER_API_WORKER_LANE_CONCURRENCY_OVERRIDES= # Named prepared statements need a session that outlives the transaction, so set # this to false behind a transaction-pooling connection pooler. The bundled @@ -313,3 +560,51 @@ FLUXER_DISCOVERY_ENABLED=true # is clamped down to the second. Milliseconds, 1000 to 3600000. #FLUXER_API_HEADERS_TIMEOUT_MS=30000 #FLUXER_API_REQUEST_TIMEOUT_MS=120000 + +# api request limits and IP bans. A refresh interval of 0 stops the periodic +# ban reload. +#FLUXER_API_MAX_INFLIGHT_REQUESTS=512 +#FLUXER_API_IP_BAN_EXEMPT_IPS= +#FLUXER_IP_BAN_REFRESH_INTERVAL_MS=300000 + +# Uploads and data exports. Presigned exports link to FLUXER_S3_PUBLIC_ENDPOINT, +# so turn them on only once browsers can reach it. +#FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED=true +#FLUXER_API_PRESIGNED_HARVEST_DOWNLOADS_ENABLED=false +#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_MAX_BODY_BYTES=524288000 +#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_TOKEN_TTL_SECS=900 +#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_KEEP_DIRECT_COUNTRIES= +#FLUXER_API_STORAGE_CHANGE_FEED_ENABLED=false +#FLUXER_API_STORAGE_CHANGE_FEED_STREAM=STORAGE_CHANGES +#FLUXER_API_STORAGE_CHANGE_FEED_SKIP_BUCKETS= +#FLUXER_CACHE_PURGE_ADAPTER=none +#FLUXER_CACHE_PURGE_HTTP_ENDPOINT= +#FLUXER_CACHE_PURGE_HTTP_TOKEN= +#FLUXER_CACHE_PURGE_HTTP_TIMEOUT_MS=10000 + +# media-proxy limits and timeouts. +#FLUXER_MEDIA_PROXY_READ_ONLY=false +#FLUXER_MEDIA_PROXY_NSFW_THRESHOLD=0.85 +#FLUXER_NSFW_SERVICE_ENDPOINT= +#FLUXER_MEDIA_PROXY_MAX_NATIVE_TRANSFORMS= +#FLUXER_MEDIA_PROXY_WORKER_QUEUE_CAPACITY= +#FLUXER_MEDIA_PROXY_MAX_ENCODE_DURATION_MS=30000 +#FLUXER_MEDIA_PROXY_MAX_ENCODE_FRAMES=20000 +#FLUXER_MEDIA_PROXY_TRANSFORM_TIMEOUT_MS=15000 +#FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_BYTES=268435456 +#FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_MAX_ENTRY_BYTES=67108864 +#FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_TTL_MS=120000 +#FLUXER_MEDIA_PROXY_SOCKET_IO_TIMEOUT_MS=30000 +#FLUXER_MEDIA_PROXY_SHUTDOWN_GRACE_MS=30000 +#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_S3_TIMEOUT_MS=900000 +#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_BYTES=33554432 +#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_TOTAL_BYTES=536870912 +#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_DIR= +#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_CHUNK_BYTES=1048576 +#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_MAX_TOTAL_BYTES=8589934592 + +# app-proxy discovery refresh, index upstream and manifest scope. +#DISCOVERY_REFRESH_INTERVAL_MS=60000 +#FLUXER_APP_PROXY_INDEX_UPSTREAM_URL= +#FLUXER_APP_PROXY_SAME_ORIGIN_HOSTS= +#FLUXER_APP_PROXY_MANIFEST_SCOPE_EXTENSIONS= diff --git a/deploy/self-hosting/Caddyfile b/deploy/self-hosting/Caddyfile index 11080f5c6..b0dcbf2a6 100644 --- a/deploy/self-hosting/Caddyfile +++ b/deploy/self-hosting/Caddyfile @@ -6,7 +6,7 @@ } {$FLUXER_EDGE_SITE_ADDRESS} { - encode zstd gzip + encode {$FLUXER_EDGE_ENCODE:zstd gzip} handle /_health { respond "OK" 200 @@ -33,12 +33,12 @@ reverse_proxy livekit:7880 } - handle /admin { + handle {$FLUXER_ADMIN_BASE_PATH:/admin} { rewrite * / reverse_proxy admin:8080 } - handle_path /admin/* { + handle_path {$FLUXER_ADMIN_BASE_PATH:/admin}/* { reverse_proxy admin:8080 } diff --git a/deploy/self-hosting/docker-compose.yml b/deploy/self-hosting/docker-compose.yml index 6bbdcbfe0..dd82a43df 100644 --- a/deploy/self-hosting/docker-compose.yml +++ b/deploy/self-hosting/docker-compose.yml @@ -3,39 +3,82 @@ name: fluxer x-fluxer-postgres-env: &fluxer-postgres-env FLUXER_DATABASE_BACKEND: postgres FLUXER_POSTGRES_HOST: ${FLUXER_POSTGRES_HOST:-postgres} - FLUXER_POSTGRES_PORT: "${FLUXER_POSTGRES_PORT:-5432}" - FLUXER_POSTGRES_DATABASE: ${FLUXER_POSTGRES_DATABASE:-fluxer} - FLUXER_POSTGRES_USERNAME: ${FLUXER_POSTGRES_USERNAME:-fluxer} + FLUXER_POSTGRES_PORT: ${FLUXER_POSTGRES_PORT:-} + FLUXER_POSTGRES_DATABASE: ${FLUXER_POSTGRES_DATABASE:-} + FLUXER_POSTGRES_USERNAME: ${FLUXER_POSTGRES_USERNAME:-} FLUXER_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env} - FLUXER_POSTGRES_SSL: "${FLUXER_POSTGRES_SSL:-false}" - FLUXER_POSTGRES_PREPARED_STATEMENTS: ${FLUXER_POSTGRES_PREPARED_STATEMENTS:-true} + FLUXER_POSTGRES_URL: ${FLUXER_POSTGRES_URL:-} + FLUXER_POSTGRES_SSL: ${FLUXER_POSTGRES_SSL:-} + FLUXER_POSTGRES_SSL_CA: ${FLUXER_POSTGRES_SSL_CA:-} + FLUXER_POSTGRES_PREPARED_STATEMENTS: ${FLUXER_POSTGRES_PREPARED_STATEMENTS:-} + FLUXER_POSTGRES_KV_TABLE: ${FLUXER_POSTGRES_KV_TABLE:-} x-fluxer-env: &fluxer-env <<: *fluxer-postgres-env FLUXER_ENV: production - NODE_ENV: production - LOG_LEVEL: ${LOG_LEVEL:-info} + LOG_LEVEL: ${LOG_LEVEL:-} + RUST_LOG: ${RUST_LOG:-} FLUXER_SELF_HOSTED: "true" FLUXER_BASE_DOMAIN: ${FLUXER_DOMAIN:?set FLUXER_DOMAIN in .env} FLUXER_PUBLIC_SCHEME: ${FLUXER_PUBLIC_SCHEME:-https} FLUXER_PUBLIC_PORT: ${FLUXER_PUBLIC_PORT:-443} FLUXER_PUBLIC_ORIGIN: ${FLUXER_PUBLIC_ORIGIN:-} FLUXER_TRUST_CLIENT_IP_HEADER: "${FLUXER_TRUST_CLIENT_IP_HEADER:-true}" - FLUXER_CLIENT_IP_HEADER_NAME: ${FLUXER_CLIENT_IP_HEADER_NAME:-x-forwarded-for} - FLUXER_API_HEADERS_TIMEOUT_MS: ${FLUXER_API_HEADERS_TIMEOUT_MS:-30000} - FLUXER_API_REQUEST_TIMEOUT_MS: ${FLUXER_API_REQUEST_TIMEOUT_MS:-120000} - FLUXER_BREACHED_PASSWORD_CHECK_ENABLED: "${FLUXER_BREACHED_PASSWORD_CHECK_ENABLED:-false}" + FLUXER_CLIENT_IP_HEADER_NAME: ${FLUXER_CLIENT_IP_HEADER_NAME:-} + FLUXER_API_HEADERS_TIMEOUT_MS: ${FLUXER_API_HEADERS_TIMEOUT_MS:-} + FLUXER_API_REQUEST_TIMEOUT_MS: ${FLUXER_API_REQUEST_TIMEOUT_MS:-} + FLUXER_API_MAX_INFLIGHT_REQUESTS: ${FLUXER_API_MAX_INFLIGHT_REQUESTS:-} + FLUXER_API_IP_BAN_EXEMPT_IPS: ${FLUXER_API_IP_BAN_EXEMPT_IPS:-} + FLUXER_IP_BAN_REFRESH_INTERVAL_MS: ${FLUXER_IP_BAN_REFRESH_INTERVAL_MS:-} + FLUXER_APP_ORIGIN_ALIASES: ${FLUXER_APP_ORIGIN_ALIASES:-} + FLUXER_BREACHED_PASSWORD_CHECK_ENABLED: ${FLUXER_BREACHED_PASSWORD_CHECK_ENABLED:-} + FLUXER_BLOCKLIST_FEEDS_ENABLED: ${FLUXER_BLOCKLIST_FEEDS_ENABLED:-} + FLUXER_IPINFO_API_KEY: ${FLUXER_IPINFO_API_KEY:-} + FLUXER_GEOIP_DB_PATH: ${FLUXER_GEOIP_DB_PATH:-} + + FLUXER_API_ENDPOINT: ${FLUXER_API_ENDPOINT:-} + FLUXER_API_CLIENT_ENDPOINT: ${FLUXER_API_CLIENT_ENDPOINT:-} + FLUXER_APP_ENDPOINT: ${FLUXER_APP_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}} + FLUXER_GATEWAY_ENDPOINT: ${FLUXER_GATEWAY_ENDPOINT:-} + FLUXER_MEDIA_ENDPOINT: ${FLUXER_MEDIA_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media} + FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT:-${FLUXER_MEDIA_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media}} + FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_STATIC_CDN_ENDPOINT:-} + FLUXER_ADMIN_ENDPOINT: ${FLUXER_ADMIN_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}${FLUXER_ADMIN_BASE_PATH:-/admin}} + FLUXER_MARKETING_ENDPOINT: ${FLUXER_MARKETING_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}} + FLUXER_INVITE_ENDPOINT: ${FLUXER_INVITE_ENDPOINT:-} + FLUXER_GIFT_ENDPOINT: ${FLUXER_GIFT_ENDPOINT:-} + FLUXER_STATIC_CDN_DOMAIN: ${FLUXER_STATIC_CDN_DOMAIN:-} + FLUXER_INVITE_DOMAIN: ${FLUXER_INVITE_DOMAIN:-} + FLUXER_GIFT_DOMAIN: ${FLUXER_GIFT_DOMAIN:-} FLUXER_KV_URL: ${FLUXER_KV_URL:-redis://valkey:6379/0} + FLUXER_KV_MODE: ${FLUXER_KV_MODE:-} FLUXER_NATS_URL: ${FLUXER_NATS_URL:-nats://nats:4222} FLUXER_NATS_JETSTREAM_URL: ${FLUXER_NATS_JETSTREAM_URL:-${FLUXER_NATS_URL:-nats://nats:4222}} FLUXER_NATS_AUTH_TOKEN: ${FLUXER_NATS_AUTH_TOKEN:-} FLUXER_SVC_NATS_URL: ${FLUXER_SVC_NATS_URL:-${FLUXER_NATS_URL:-nats://nats:4222}} FLUXER_SVC_SHARD_COUNT: "1" + FLUXER_SVC_MAX_CONCURRENT_REQUESTS: ${FLUXER_SVC_MAX_CONCURRENT_REQUESTS:-} + FLUXER_SVC_CACHE_MAX_ENTRIES: ${FLUXER_SVC_CACHE_MAX_ENTRIES:-} + FLUXER_SVC_CACHE_TTL_MS: ${FLUXER_SVC_CACHE_TTL_MS:-} + FLUXER_GIF_SERVICE_NATS_CLIENT_NAME: ${FLUXER_GIF_SERVICE_NATS_CLIENT_NAME:-} + FLUXER_GIF_SERVICE_TIMEOUT_MS: ${FLUXER_GIF_SERVICE_TIMEOUT_MS:-} + FLUXER_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS: ${FLUXER_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS:-} + FLUXER_USERS_SERVICE_NATS_CLIENT_NAME: ${FLUXER_USERS_SERVICE_NATS_CLIENT_NAME:-} + FLUXER_USERS_SERVICE_TIMEOUT_MS: ${FLUXER_USERS_SERVICE_TIMEOUT_MS:-} + FLUXER_USERS_SERVICE_INFLIGHT_MAX_ENTRIES: ${FLUXER_USERS_SERVICE_INFLIGHT_MAX_ENTRIES:-} + FLUXER_SNOWFLAKE_SERVICE_NATS_CLIENT_NAME: ${FLUXER_SNOWFLAKE_SERVICE_NATS_CLIENT_NAME:-} + FLUXER_SNOWFLAKE_SERVICE_BATCH_SIZE: ${FLUXER_SNOWFLAKE_SERVICE_BATCH_SIZE:-} + FLUXER_SNOWFLAKE_SERVICE_LOW_WATERMARK: ${FLUXER_SNOWFLAKE_SERVICE_LOW_WATERMARK:-} + FLUXER_SNOWFLAKE_SERVICE_MAX_BUFFER_AGE_MS: ${FLUXER_SNOWFLAKE_SERVICE_MAX_BUFFER_AGE_MS:-} + FLUXER_SNOWFLAKE_SERVICE_REQUEST_TIMEOUT_MS: ${FLUXER_SNOWFLAKE_SERVICE_REQUEST_TIMEOUT_MS:-} - FLUXER_SEARCH_ENGINE: meilisearch + FLUXER_SEARCH_ENGINE: ${FLUXER_SEARCH_ENGINE:-meilisearch} FLUXER_SEARCH_URL: ${FLUXER_SEARCH_URL:-http://meilisearch:7700} FLUXER_SEARCH_API_KEY: ${MEILI_MASTER_KEY:?set MEILI_MASTER_KEY in .env} + FLUXER_SEARCH_USERNAME: ${FLUXER_SEARCH_USERNAME:-} + FLUXER_SEARCH_PASSWORD: ${FLUXER_SEARCH_PASSWORD:-} + FLUXER_SEARCH_TLS_REJECT_UNAUTHORIZED: ${FLUXER_SEARCH_TLS_REJECT_UNAUTHORIZED:-} FLUXER_S3_ENDPOINT: ${FLUXER_S3_ENDPOINT:-http://seaweedfs:8333} FLUXER_S3_PUBLIC_ENDPOINT: ${FLUXER_S3_PUBLIC_ENDPOINT:-${FLUXER_S3_ENDPOINT:-http://seaweedfs:8333}} @@ -47,45 +90,96 @@ x-fluxer-env: &fluxer-env FLUXER_S3_BUCKET_UPLOADS: ${FLUXER_S3_BUCKET_UPLOADS:-fluxer-uploads} FLUXER_S3_BUCKET_REPORTS: ${FLUXER_S3_BUCKET_REPORTS:-fluxer-reports} FLUXER_S3_BUCKET_HARVESTS: ${FLUXER_S3_BUCKET_HARVESTS:-fluxer-harvests} - AWS_ACCESS_KEY_ID: ${FLUXER_S3_ACCESS_KEY:?set FLUXER_S3_ACCESS_KEY in .env} - AWS_SECRET_ACCESS_KEY: ${FLUXER_S3_SECRET_KEY:?set FLUXER_S3_SECRET_KEY in .env} - AWS_DEFAULT_REGION: ${FLUXER_S3_REGION:-us-east-1} - AWS_EC2_METADATA_DISABLED: "true" + FLUXER_API_PRESIGNED_HARVEST_DOWNLOADS_ENABLED: "${FLUXER_API_PRESIGNED_HARVEST_DOWNLOADS_ENABLED:-false}" + FLUXER_API_STORAGE_CHANGE_FEED_ENABLED: ${FLUXER_API_STORAGE_CHANGE_FEED_ENABLED:-} + FLUXER_API_STORAGE_CHANGE_FEED_STREAM: ${FLUXER_API_STORAGE_CHANGE_FEED_STREAM:-} + FLUXER_API_STORAGE_CHANGE_FEED_SKIP_BUCKETS: ${FLUXER_API_STORAGE_CHANGE_FEED_SKIP_BUCKETS:-} + FLUXER_CACHE_PURGE_ADAPTER: ${FLUXER_CACHE_PURGE_ADAPTER:-} + FLUXER_CACHE_PURGE_HTTP_ENDPOINT: ${FLUXER_CACHE_PURGE_HTTP_ENDPOINT:-} + FLUXER_CACHE_PURGE_HTTP_TOKEN: ${FLUXER_CACHE_PURGE_HTTP_TOKEN:-} + FLUXER_CACHE_PURGE_HTTP_TIMEOUT_MS: ${FLUXER_CACHE_PURGE_HTTP_TIMEOUT_MS:-} FLUXER_LIVEKIT_ENABLED: "${FLUXER_LIVEKIT_ENABLED:-true}" FLUXER_LIVEKIT_API_KEY: ${LIVEKIT_API_KEY:?set LIVEKIT_API_KEY in .env} FLUXER_LIVEKIT_API_SECRET: ${LIVEKIT_API_SECRET:?set LIVEKIT_API_SECRET in .env} FLUXER_LIVEKIT_INTERNAL_URL: ${FLUXER_LIVEKIT_INTERNAL_URL:-http://livekit:7880} - FLUXER_LIVEKIT_WEBHOOK_URL: http://api:8080/webhooks/livekit - FLUXER_LIVEKIT_DEFAULT_REGION: '{"id":"default","name":"Default","emoji":"🌍","latitude":0,"longitude":0}' + FLUXER_LIVEKIT_DEFAULT_REGION: '${FLUXER_LIVEKIT_DEFAULT_REGION:-{"id":"default","name":"Default","emoji":"🌍","latitude":0,"longitude":0}}' FLUXER_LIVEKIT_URL: ${FLUXER_LIVEKIT_URL:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}:${FLUXER_PUBLIC_PORT:-443}}/livekit} FLUXER_KLIPY_API_KEY: ${FLUXER_KLIPY_API_KEY:-} + FLUXER_YOUTUBE_API_KEY: ${FLUXER_YOUTUBE_API_KEY:-} + FLUXER_API_UNFURL_IGNORED_HOSTS: ${FLUXER_API_UNFURL_IGNORED_HOSTS:-} - FLUXER_EMAIL_ENABLED: ${FLUXER_EMAIL_ENABLED:-false} - FLUXER_EMAIL_PROVIDER: ${FLUXER_EMAIL_PROVIDER:-none} + FLUXER_EMAIL_ENABLED: ${FLUXER_EMAIL_ENABLED:-} + FLUXER_EMAIL_PROVIDER: ${FLUXER_EMAIL_PROVIDER:-} FLUXER_EMAIL_FROM_EMAIL: ${FLUXER_EMAIL_FROM_EMAIL:-noreply@localhost} - FLUXER_EMAIL_FROM_NAME: ${FLUXER_EMAIL_FROM_NAME:-Fluxer} + FLUXER_EMAIL_FROM_NAME: ${FLUXER_EMAIL_FROM_NAME:-} FLUXER_EMAIL_APP_BASE_URL: ${FLUXER_EMAIL_APP_BASE_URL:-} + FLUXER_EMAIL_WEBHOOK_SECRET: ${FLUXER_EMAIL_WEBHOOK_SECRET:-} FLUXER_EMAIL_SMTP_HOST: ${FLUXER_EMAIL_SMTP_HOST:-} - FLUXER_EMAIL_SMTP_PORT: ${FLUXER_EMAIL_SMTP_PORT:-587} + FLUXER_EMAIL_SMTP_PORT: ${FLUXER_EMAIL_SMTP_PORT:-} FLUXER_EMAIL_SMTP_USERNAME: ${FLUXER_EMAIL_SMTP_USERNAME:-} FLUXER_EMAIL_SMTP_PASSWORD: ${FLUXER_EMAIL_SMTP_PASSWORD:-} - FLUXER_EMAIL_SMTP_SECURE: ${FLUXER_EMAIL_SMTP_SECURE:-true} + FLUXER_EMAIL_SMTP_SECURE: ${FLUXER_EMAIL_SMTP_SECURE:-} - FLUXER_STRIPE_ENABLED: "${FLUXER_STRIPE_ENABLED:-false}" - FLUXER_NCMEC_ENABLED: "${FLUXER_NCMEC_ENABLED:-false}" - FLUXER_CLAMAV_ENABLED: "${FLUXER_CLAMAV_ENABLED:-false}" - FLUXER_DISCOVERY_ENABLED: ${FLUXER_DISCOVERY_ENABLED:-true} + FLUXER_STRIPE_ENABLED: ${FLUXER_STRIPE_ENABLED:-} + FLUXER_STRIPE_SECRET_KEY: ${FLUXER_STRIPE_SECRET_KEY:-} + FLUXER_STRIPE_WEBHOOK_SECRET: ${FLUXER_STRIPE_WEBHOOK_SECRET:-} + FLUXER_STRIPE_PRICES: ${FLUXER_STRIPE_PRICES:-} + FLUXER_STRIPE_LEGACY_PRICES: ${FLUXER_STRIPE_LEGACY_PRICES:-} + FLUXER_API_DONATION_PROXY_KEY: ${FLUXER_API_DONATION_PROXY_KEY:-} + FLUXER_VISIONARIES_GUILD_ID: ${FLUXER_VISIONARIES_GUILD_ID:-} + FLUXER_VISIONARIES_GUILD_VISIONARY_ROLE_ID: ${FLUXER_VISIONARIES_GUILD_VISIONARY_ROLE_ID:-} + + FLUXER_NCMEC_ENABLED: ${FLUXER_NCMEC_ENABLED:-} + FLUXER_NCMEC_BASE_URL: ${FLUXER_NCMEC_BASE_URL:-} + FLUXER_NCMEC_USERNAME: ${FLUXER_NCMEC_USERNAME:-} + FLUXER_NCMEC_PASSWORD: ${FLUXER_NCMEC_PASSWORD:-} + FLUXER_NCMEC_REPORTER_EMAIL: ${FLUXER_NCMEC_REPORTER_EMAIL:-} + FLUXER_CLAMAV_ENABLED: ${FLUXER_CLAMAV_ENABLED:-} + FLUXER_CLAMAV_HOST: ${FLUXER_CLAMAV_HOST:-} + FLUXER_CLAMAV_PORT: ${FLUXER_CLAMAV_PORT:-} + FLUXER_CLAMAV_FAIL_OPEN: ${FLUXER_CLAMAV_FAIL_OPEN:-} + + FLUXER_APP_PRODUCT_NAME: ${FLUXER_APP_PRODUCT_NAME:-} + FLUXER_APP_ICON_URL: ${FLUXER_APP_ICON_URL:-} + FLUXER_APP_SYMBOL_URL: ${FLUXER_APP_SYMBOL_URL:-} + FLUXER_APP_LOGO_URL: ${FLUXER_APP_LOGO_URL:-} + FLUXER_APP_WORDMARK_URL: ${FLUXER_APP_WORDMARK_URL:-} + FLUXER_APP_FAVICON_URL: ${FLUXER_APP_FAVICON_URL:-} + FLUXER_APP_THEME_COLOR: ${FLUXER_APP_THEME_COLOR:-} + FLUXER_APP_STATUS_PAGE_URL: ${FLUXER_APP_STATUS_PAGE_URL:-} + FLUXER_APP_STATUS_PAGE_INCIDENT_HISTORY_URL: ${FLUXER_APP_STATUS_PAGE_INCIDENT_HISTORY_URL:-} + FLUXER_INSTANCE_SETUP_CONFIGURED: ${FLUXER_INSTANCE_SETUP_CONFIGURED:-} + FLUXER_AUTO_JOIN_INVITE_CODE: ${FLUXER_AUTO_JOIN_INVITE_CODE:-} + FLUXER_DISCOVERY_ENABLED: ${FLUXER_DISCOVERY_ENABLED:-} + FLUXER_DISCOVERY_MIN_MEMBER_COUNT: ${FLUXER_DISCOVERY_MIN_MEMBER_COUNT:-} + FLUXER_DELETION_GRACE_PERIOD_HOURS: ${FLUXER_DELETION_GRACE_PERIOD_HOURS:-} + FLUXER_API_WORKER_LANE_CONCURRENCY_OVERRIDES: ${FLUXER_API_WORKER_LANE_CONCURRENCY_OVERRIDES:-} + + FLUXER_AUTH_BLUESKY_ENABLED: ${FLUXER_AUTH_BLUESKY_ENABLED:-} + FLUXER_AUTH_BLUESKY_CLIENT_NAME: ${FLUXER_AUTH_BLUESKY_CLIENT_NAME:-} + FLUXER_AUTH_BLUESKY_CLIENT_URI: ${FLUXER_AUTH_BLUESKY_CLIENT_URI:-} + FLUXER_AUTH_BLUESKY_LOGO_URI: ${FLUXER_AUTH_BLUESKY_LOGO_URI:-} + FLUXER_AUTH_BLUESKY_TOS_URI: ${FLUXER_AUTH_BLUESKY_TOS_URI:-} + FLUXER_AUTH_BLUESKY_POLICY_URI: ${FLUXER_AUTH_BLUESKY_POLICY_URI:-} + FLUXER_AUTH_BLUESKY_KEYS: ${FLUXER_AUTH_BLUESKY_KEYS:-} + + FLUXER_PUSH_APNS_ENABLED: ${FLUXER_PUSH_APNS_ENABLED:-} + FLUXER_PUSH_APNS_TEAM_ID: ${FLUXER_PUSH_APNS_TEAM_ID:-} + FLUXER_PUSH_APNS_KEY_ID: ${FLUXER_PUSH_APNS_KEY_ID:-} + FLUXER_PUSH_APNS_PRIVATE_KEY: ${FLUXER_PUSH_APNS_PRIVATE_KEY:-} + FLUXER_PUSH_APNS_PRIVATE_KEY_PATH: ${FLUXER_PUSH_APNS_PRIVATE_KEY_PATH:-} + FLUXER_PUSH_APNS_APPS: ${FLUXER_PUSH_APNS_APPS:-} FLUXER_SUDO_MODE_SECRET: ${FLUXER_SUDO_MODE_SECRET:?set FLUXER_SUDO_MODE_SECRET in .env} FLUXER_CONNECTION_INITIATION_SECRET: ${FLUXER_CONNECTION_INITIATION_SECRET:?set FLUXER_CONNECTION_INITIATION_SECRET in .env} - FLUXER_SSO_ALLOW_PRIVATE_ADDRESSES: ${FLUXER_SSO_ALLOW_PRIVATE_ADDRESSES:-false} + FLUXER_SSO_ALLOW_PRIVATE_ADDRESSES: ${FLUXER_SSO_ALLOW_PRIVATE_ADDRESSES:-} FLUXER_VAPID_PUBLIC_KEY: ${FLUXER_VAPID_PUBLIC_KEY:?set FLUXER_VAPID_PUBLIC_KEY in .env} FLUXER_VAPID_PRIVATE_KEY: ${FLUXER_VAPID_PRIVATE_KEY:?set FLUXER_VAPID_PRIVATE_KEY in .env} FLUXER_VAPID_EMAIL: ${FLUXER_VAPID_EMAIL:-admin@${FLUXER_DOMAIN}} FLUXER_PASSKEY_RP_ID: ${FLUXER_PASSKEY_RP_ID:-${FLUXER_DOMAIN}} - FLUXER_PASSKEY_RP_NAME: ${FLUXER_PASSKEY_RP_NAME:-Fluxer} + FLUXER_PASSKEY_RP_NAME: ${FLUXER_PASSKEY_RP_NAME:-} FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS: ${FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}} FLUXER_GATEWAY_RPC_AUTH_TOKEN: ${FLUXER_GATEWAY_RPC_AUTH_TOKEN:?set FLUXER_GATEWAY_RPC_AUTH_TOKEN in .env} FLUXER_MEDIA_PROXY_SECRET_KEY: ${FLUXER_MEDIA_PROXY_SECRET_KEY:?set FLUXER_MEDIA_PROXY_SECRET_KEY in .env} @@ -95,34 +189,36 @@ x-fluxer-env: &fluxer-env FLUXER_ADMIN_OAUTH_CLIENT_SECRET: ${FLUXER_ADMIN_OAUTH_CLIENT_SECRET:?set FLUXER_ADMIN_OAUTH_CLIENT_SECRET in .env} FLUXER_INTERNAL_API_ENDPOINT: http://api:8080 - FLUXER_INTERNAL_GATEWAY_ENDPOINT: http://gateway:8080 FLUXER_INTERNAL_MEDIA_PROXY_ENDPOINT: http://media-proxy:8080 - FLUXER_MARKETING_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}} - FLUXER_MEDIA_PROXY_ENDPOINT: http://media-proxy:8080 - FLUXER_MEDIA_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media - FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media + FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media} + FLUXER_MEDIA_PROXY_UPLOAD_RELAY_MAX_BODY_BYTES: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_MAX_BODY_BYTES:-} + FLUXER_MEDIA_PROXY_UPLOAD_RELAY_TOKEN_TTL_SECS: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_TOKEN_TTL_SECS:-} + FLUXER_MEDIA_PROXY_UPLOAD_RELAY_KEEP_DIRECT_COUNTRIES: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_KEEP_DIRECT_COUNTRIES:-} x-fluxer-service: &fluxer-service - restart: unless-stopped + restart: ${FLUXER_RESTART_POLICY:-unless-stopped} networks: [fluxer] -x-fluxer-svc-healthcheck: &fluxer-svc-healthcheck - test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/8090 && printf 'GET /_health HTTP/1.0\\r\\n\\r\\n' >&3 && head -n 1 <&3 | grep -q ' 200 '"] - interval: 10s - timeout: 5s - retries: 30 - start_period: 60s +x-fluxer-app-healthcheck: &fluxer-app-healthcheck + interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s} + timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s} + retries: ${FLUXER_APP_HEALTHCHECK_RETRIES:-30} + start_period: ${FLUXER_APP_HEALTHCHECK_START_PERIOD:-90s} start_interval: 1s +x-fluxer-svc-healthcheck: &fluxer-svc-healthcheck + <<: *fluxer-app-healthcheck + start_period: ${FLUXER_SVC_HEALTHCHECK_START_PERIOD:-60s} + test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/8090 && printf 'GET /_health HTTP/1.0\\r\\n\\r\\n' >&3 && head -n 1 <&3 | grep -q ' 200 '"] + services: edge: - image: caddy:2.11-alpine + <<: *fluxer-service + image: ${FLUXER_CADDY_IMAGE:-caddy:2.11-alpine} deploy: resources: limits: memory: ${FLUXER_CADDY_MEMORY_LIMIT:-256mb} - restart: unless-stopped - networks: [fluxer] ports: - "${FLUXER_HTTP_PORT:-80}:80" - "${FLUXER_HTTPS_PORT:-443}:443" @@ -130,15 +226,17 @@ services: environment: FLUXER_EDGE_SITE_ADDRESS: ${FLUXER_EDGE_SITE_ADDRESS:-${FLUXER_CADDY_SITE_ADDRESS:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN:?set FLUXER_DOMAIN in .env}}} FLUXER_EDGE_TRUSTED_PROXIES: ${FLUXER_EDGE_TRUSTED_PROXIES:-private_ranges} + FLUXER_EDGE_ENCODE: ${FLUXER_EDGE_ENCODE:-zstd gzip} + FLUXER_ADMIN_BASE_PATH: ${FLUXER_ADMIN_BASE_PATH:-/admin} volumes: - ./Caddyfile:/etc/caddy/Caddyfile:ro - edge-data:/data - edge-config:/config healthcheck: test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:2019/config/"] - interval: 10s - timeout: 5s - retries: 10 + interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s} + timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s} + retries: ${FLUXER_HEALTHCHECK_RETRIES:-10} depends_on: api: {condition: service_started} gateway: {condition: service_healthy} @@ -147,15 +245,14 @@ services: admin: {condition: service_started} postgres: - image: postgres:16-alpine + <<: *fluxer-service + image: ${FLUXER_POSTGRES_IMAGE:-postgres:16-alpine} deploy: resources: limits: memory: ${FLUXER_POSTGRES_MEMORY_LIMIT:-5gb} reservations: memory: ${FLUXER_POSTGRES_MEMORY_RESERVATION:-3gb} - restart: unless-stopped - networks: [fluxer] command: > postgres -c max_connections=${FLUXER_POSTGRES_SERVER_MAX_CONNECTIONS:-150} @@ -164,82 +261,79 @@ services: -c work_mem=${FLUXER_POSTGRES_WORK_MEM:-8MB} -c maintenance_work_mem=${FLUXER_POSTGRES_MAINTENANCE_WORK_MEM:-256MB} -c autovacuum_work_mem=${FLUXER_POSTGRES_AUTOVACUUM_WORK_MEM:-128MB} - -c random_page_cost=1.1 - -c effective_io_concurrency=200 - -c default_statistics_target=200 - -c jit=off - -c min_wal_size=512MB - -c max_wal_size=2GB - -c checkpoint_completion_target=0.9 - -c wal_buffers=16MB - -c wal_compression=zstd - -c bgwriter_delay=50ms - -c bgwriter_lru_maxpages=1000 - -c autovacuum_vacuum_scale_factor=0.05 - -c autovacuum_analyze_scale_factor=0.02 - -c autovacuum_vacuum_cost_limit=2000 - -c track_io_timing=on - -c shared_preload_libraries=pg_stat_statements + -c random_page_cost=${FLUXER_POSTGRES_RANDOM_PAGE_COST:-1.1} + -c effective_io_concurrency=${FLUXER_POSTGRES_EFFECTIVE_IO_CONCURRENCY:-200} + -c default_statistics_target=${FLUXER_POSTGRES_DEFAULT_STATISTICS_TARGET:-200} + -c jit=${FLUXER_POSTGRES_JIT:-off} + -c min_wal_size=${FLUXER_POSTGRES_MIN_WAL_SIZE:-512MB} + -c max_wal_size=${FLUXER_POSTGRES_MAX_WAL_SIZE:-2GB} + -c checkpoint_completion_target=${FLUXER_POSTGRES_CHECKPOINT_COMPLETION_TARGET:-0.9} + -c wal_buffers=${FLUXER_POSTGRES_WAL_BUFFERS:-16MB} + -c wal_compression=${FLUXER_POSTGRES_WAL_COMPRESSION:-zstd} + -c bgwriter_delay=${FLUXER_POSTGRES_BGWRITER_DELAY:-50ms} + -c bgwriter_lru_maxpages=${FLUXER_POSTGRES_BGWRITER_LRU_MAXPAGES:-1000} + -c autovacuum_vacuum_scale_factor=${FLUXER_POSTGRES_AUTOVACUUM_VACUUM_SCALE_FACTOR:-0.05} + -c autovacuum_analyze_scale_factor=${FLUXER_POSTGRES_AUTOVACUUM_ANALYZE_SCALE_FACTOR:-0.02} + -c autovacuum_vacuum_cost_limit=${FLUXER_POSTGRES_AUTOVACUUM_VACUUM_COST_LIMIT:-2000} + -c track_io_timing=${FLUXER_POSTGRES_TRACK_IO_TIMING:-on} + -c shared_preload_libraries=${FLUXER_POSTGRES_SHARED_PRELOAD_LIBRARIES:-pg_stat_statements} shm_size: ${FLUXER_POSTGRES_SHM_SIZE:-1gb} environment: - POSTGRES_DB: fluxer - POSTGRES_USER: fluxer + POSTGRES_DB: ${FLUXER_POSTGRES_DATABASE:-fluxer} + POSTGRES_USER: ${FLUXER_POSTGRES_USERNAME:-fluxer} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env} volumes: - postgres-data:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -U fluxer -d fluxer"] - interval: 10s - timeout: 5s - retries: 10 + test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""] + interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s} + timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s} + retries: ${FLUXER_HEALTHCHECK_RETRIES:-10} valkey: - image: valkey/valkey:9.1-alpine + <<: *fluxer-service + image: ${FLUXER_VALKEY_IMAGE:-valkey/valkey:9.1-alpine} deploy: resources: limits: memory: ${FLUXER_VALKEY_MEMORY_LIMIT:-256mb} - restart: unless-stopped - networks: [fluxer] - command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec", "--dir", "/data", + command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "${FLUXER_VALKEY_APPENDFSYNC:-everysec}", "--dir", "/data", "--maxmemory", "${FLUXER_VALKEY_MAXMEMORY:-192mb}", "--maxmemory-policy", "${FLUXER_VALKEY_MAXMEMORY_POLICY:-noeviction}"] volumes: - valkey-data:/data healthcheck: test: ["CMD", "valkey-cli", "ping"] - interval: 10s - timeout: 5s - retries: 10 + interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s} + timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s} + retries: ${FLUXER_HEALTHCHECK_RETRIES:-10} nats: - image: nats:2.14-alpine + <<: *fluxer-service + image: ${FLUXER_NATS_IMAGE:-nats:2.14-alpine} deploy: resources: limits: memory: ${FLUXER_NATS_MEMORY_LIMIT:-256mb} - restart: unless-stopped - networks: [fluxer] command: ["-js", "-sd", "/data", "-m", "8222"] volumes: - nats-data:/data healthcheck: test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8222/healthz"] - interval: 10s - timeout: 5s - retries: 10 + interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s} + timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s} + retries: ${FLUXER_HEALTHCHECK_RETRIES:-10} meilisearch: - image: getmeili/meilisearch:v1.53 + <<: *fluxer-service + image: ${FLUXER_MEILISEARCH_IMAGE:-getmeili/meilisearch:v1.53} deploy: resources: limits: memory: ${FLUXER_MEILISEARCH_MEMORY_LIMIT:-768mb} - restart: unless-stopped - networks: [fluxer] environment: - MEILI_ENV: production - MEILI_NO_ANALYTICS: "true" + MEILI_ENV: ${FLUXER_MEILISEARCH_ENV:-production} + MEILI_NO_ANALYTICS: "${FLUXER_MEILISEARCH_NO_ANALYTICS:-true}" MEILI_UPGRADE_DB: "true" MEILI_MAX_INDEXING_MEMORY: ${FLUXER_MEILISEARCH_MAX_INDEXING_MEMORY:-384mb} MEILI_MASTER_KEY: ${MEILI_MASTER_KEY:?set MEILI_MASTER_KEY in .env} @@ -247,32 +341,31 @@ services: - meilisearch-data:/meili_data healthcheck: test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:7700/health"] - interval: 10s - timeout: 5s - retries: 10 + interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s} + timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s} + retries: ${FLUXER_HEALTHCHECK_RETRIES:-10} seaweedfs: - image: chrislusf/seaweedfs:4.47 + <<: *fluxer-service + image: ${FLUXER_SEAWEEDFS_IMAGE:-chrislusf/seaweedfs:4.47} deploy: resources: limits: memory: ${FLUXER_SEAWEEDFS_MEMORY_LIMIT:-2gb} - restart: unless-stopped - networks: [fluxer] environment: GOMEMLIMIT: ${FLUXER_SEAWEEDFS_GOMEMLIMIT:-1536MiB} - command: ["server", "-s3", "-dir=/data", "-master.telemetry=false"] + command: ["server", "-s3", "-dir=/data", "-master.telemetry=${FLUXER_SEAWEEDFS_TELEMETRY:-false}"] volumes: - seaweedfs-data:/data healthcheck: test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8333/healthz"] - interval: 10s - timeout: 5s - retries: 20 - start_period: 60s + interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s} + timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s} + retries: ${FLUXER_SEAWEEDFS_HEALTHCHECK_RETRIES:-20} + start_period: ${FLUXER_SEAWEEDFS_HEALTHCHECK_START_PERIOD:-60s} seaweedfs-init: - image: chrislusf/seaweedfs:4.47 + image: ${FLUXER_SEAWEEDFS_IMAGE:-chrislusf/seaweedfs:4.47} deploy: resources: limits: @@ -288,13 +381,14 @@ services: FLUXER_S3_BUCKET_UPLOADS: ${FLUXER_S3_BUCKET_UPLOADS:-fluxer-uploads} FLUXER_S3_BUCKET_REPORTS: ${FLUXER_S3_BUCKET_REPORTS:-fluxer-reports} FLUXER_S3_BUCKET_HARVESTS: ${FLUXER_S3_BUCKET_HARVESTS:-fluxer-harvests} + FLUXER_SEAWEEDFS_INIT_ATTEMPTS: ${FLUXER_SEAWEEDFS_INIT_ATTEMPTS:-60} entrypoint: - /bin/sh - -c - > buckets="$$FLUXER_S3_BUCKET_CDN $$FLUXER_S3_BUCKET_UPLOADS $$FLUXER_S3_BUCKET_REPORTS $$FLUXER_S3_BUCKET_HARVESTS"; missing="$$buckets"; - for attempt in $$(seq 1 60); do + for attempt in $$(seq 1 $$FLUXER_SEAWEEDFS_INIT_ATTEMPTS); do if ! nc -z seaweedfs 9333 2>/dev/null; then sleep 2; continue; @@ -321,18 +415,17 @@ services: exit 1; livekit: - image: livekit/livekit-server:v1.12.0 + <<: *fluxer-service + image: ${FLUXER_LIVEKIT_IMAGE:-livekit/livekit-server:v1.12.0} deploy: resources: limits: memory: ${FLUXER_LIVEKIT_MEMORY_LIMIT:-512mb} - restart: unless-stopped - networks: [fluxer] environment: LIVEKIT_KEYS: "${LIVEKIT_API_KEY:?set LIVEKIT_API_KEY in .env}: ${LIVEKIT_API_SECRET:?set LIVEKIT_API_SECRET in .env}" LIVEKIT_CONFIG: | port: 7880 - log_level: info + log_level: ${FLUXER_LIVEKIT_LOG_LEVEL:-info} rtc: tcp_port: ${FLUXER_LIVEKIT_TCP_PORT:-7881} udp_port: ${FLUXER_LIVEKIT_UDP_PORT:-7882} @@ -350,9 +443,9 @@ services: - "${FLUXER_LIVEKIT_UDP_PORT:-7882}:${FLUXER_LIVEKIT_UDP_PORT:-7882}/udp" healthcheck: test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:7880/"] - interval: 10s - timeout: 5s - retries: 10 + interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s} + timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s} + retries: ${FLUXER_HEALTHCHECK_RETRIES:-10} api: <<: *fluxer-service @@ -366,16 +459,13 @@ services: environment: <<: *fluxer-env FLUXER_API_PORT: "8080" - NODE_OPTIONS: --enable-source-maps${FLUXER_API_NODE_HEAP_MB:+ --max-old-space-size=$FLUXER_API_NODE_HEAP_MB} - FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED: "true" - FLUXER_POSTGRES_MAX_CONNECTIONS: "25" + NODE_OPTIONS: --enable-source-maps${FLUXER_API_NODE_HEAP_MB:+ --max-old-space-size=$FLUXER_API_NODE_HEAP_MB}${FLUXER_API_NODE_OPTIONS:+ $FLUXER_API_NODE_OPTIONS} + NODE_EXTRA_CA_CERTS: ${FLUXER_NODE_EXTRA_CA_CERTS:-/etc/ssl/certs/ca-certificates.crt} + FLUXER_POSTGRES_MAX_CONNECTIONS: "${FLUXER_API_POSTGRES_MAX_CONNECTIONS:-25}" + FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED: "${FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED:-true}" healthcheck: test: ["CMD-SHELL", "node -e \"fetch('http://127.0.0.1:8080/_health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))\""] - interval: 10s - timeout: 5s - retries: 30 - start_period: 90s - start_interval: 1s + <<: *fluxer-app-healthcheck depends_on: postgres: {condition: service_healthy} valkey: {condition: service_healthy} @@ -400,21 +490,18 @@ services: memory: ${FLUXER_WORKER_MEMORY_LIMIT:-2560mb} reservations: memory: ${FLUXER_WORKER_MEMORY_RESERVATION:-1gb} - working_dir: /usr/src/app/fluxer_api - command: ["sh", "-c", "if [ -f dist/WorkerEntrypoint.js ]; then exec node dist/WorkerEntrypoint.js; else exec ./node_modules/.bin/tsx src/WorkerEntrypoint.ts; fi"] + command: ["node", "dist/WorkerEntrypoint.js"] environment: <<: *fluxer-env - NODE_OPTIONS: --enable-source-maps${FLUXER_WORKER_NODE_HEAP_MB:+ --max-old-space-size=$FLUXER_WORKER_NODE_HEAP_MB} + NODE_OPTIONS: --enable-source-maps${FLUXER_WORKER_NODE_HEAP_MB:+ --max-old-space-size=$FLUXER_WORKER_NODE_HEAP_MB}${FLUXER_WORKER_NODE_OPTIONS:+ $FLUXER_WORKER_NODE_OPTIONS} + NODE_EXTRA_CA_CERTS: ${FLUXER_NODE_EXTRA_CA_CERTS:-/etc/ssl/certs/ca-certificates.crt} FLUXER_API_WORKER_MODE: all_lanes FLUXER_API_WORKER_ENABLE_CRON_SCHEDULER: "true" - FLUXER_POSTGRES_MAX_CONNECTIONS: "25" + FLUXER_POSTGRES_MAX_CONNECTIONS: "${FLUXER_WORKER_POSTGRES_MAX_CONNECTIONS:-25}" healthcheck: test: ["CMD", "node", "-e", "const age=Date.now()-require('node:fs').statSync('/tmp/fluxer-worker-heartbeat').mtimeMs;if(age>30000){console.error('worker heartbeat is '+Math.round(age)+'ms old');process.exit(1)}"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 90s - start_interval: 1s + <<: *fluxer-app-healthcheck + retries: ${FLUXER_WORKER_HEALTHCHECK_RETRIES:-3} depends_on: postgres: {condition: service_healthy} valkey: {condition: service_healthy} @@ -436,18 +523,30 @@ services: environment: <<: *fluxer-env FLUXER_GATEWAY_PORT: "8080" - FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media - FLUXER_GATEWAY_STATIC_CDN_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}} - FLUXER_GATEWAY_LOGGER_LEVEL: info + FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: ${FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT:-${FLUXER_MEDIA_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media}} + FLUXER_GATEWAY_STATIC_CDN_ENDPOINT: ${FLUXER_GATEWAY_STATIC_CDN_ENDPOINT:-${FLUXER_STATIC_CDN_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}}} + FLUXER_GATEWAY_LOGGER_LEVEL: ${FLUXER_GATEWAY_LOGGER_LEVEL:-} + LOGGER_LEVEL: ${LOGGER_LEVEL:-} + FLUXER_GATEWAY_PUSH_ENABLED: ${FLUXER_GATEWAY_PUSH_ENABLED:-} + FLUXER_GATEWAY_PUSH_ENROLLED_CLEAR_NOTIFICATIONS_ENABLED: ${FLUXER_GATEWAY_PUSH_ENROLLED_CLEAR_NOTIFICATIONS_ENABLED:-} + FLUXER_GATEWAY_PUSH_OUTBOX_REQUEST_TIMEOUT_MS: ${FLUXER_GATEWAY_PUSH_OUTBOX_REQUEST_TIMEOUT_MS:-} + FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_ENTRIES: ${FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_ENTRIES:-} + FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_BYTES: ${FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_BYTES:-} + FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY: ${FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY:-} + FLUXER_GATEWAY_NATS_RPC_MAX_HANDLERS: ${FLUXER_GATEWAY_NATS_RPC_MAX_HANDLERS:-} + FLUXER_GATEWAY_HTTP_FAILURE_THRESHOLD: ${FLUXER_GATEWAY_HTTP_FAILURE_THRESHOLD:-} + FLUXER_GATEWAY_HTTP_RECOVERY_TIMEOUT_MS: ${FLUXER_GATEWAY_HTTP_RECOVERY_TIMEOUT_MS:-} FLUXER_ERLANG_COOKIE: ${FLUXER_ERLANG_COOKIE:?set FLUXER_ERLANG_COOKIE in .env} - FLUXER_ERLANG_SCHEDULERS_MIN: "${FLUXER_ERLANG_SCHEDULERS_MIN:-2}" - FLUXER_ERLANG_SCHEDULERS_MAX: "${FLUXER_ERLANG_SCHEDULERS_MAX:-16}" + FLUXER_ERLANG_SCHEDULERS: ${FLUXER_ERLANG_SCHEDULERS:-} + FLUXER_ERLANG_SCHEDULERS_MIN: ${FLUXER_ERLANG_SCHEDULERS_MIN:-} + FLUXER_ERLANG_SCHEDULERS_MAX: ${FLUXER_ERLANG_SCHEDULERS_MAX:-} + FLUXER_ERLANG_DIRTY_CPU_SCHEDULERS: ${FLUXER_ERLANG_DIRTY_CPU_SCHEDULERS:-} healthcheck: test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:8080/_health/ready"] - interval: 10s - timeout: 5s - retries: 30 - start_period: 90s + interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s} + timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s} + retries: ${FLUXER_APP_HEALTHCHECK_RETRIES:-30} + start_period: ${FLUXER_APP_HEALTHCHECK_START_PERIOD:-90s} depends_on: nats: {condition: service_healthy} valkey: {condition: service_healthy} @@ -461,15 +560,36 @@ services: memory: ${FLUXER_MEDIA_PROXY_MEMORY_LIMIT:-512mb} environment: <<: *fluxer-env - FLUXER_MEDIA_PROXY_HOST: 0.0.0.0 FLUXER_MEDIA_PROXY_PORT: "8080" FLUXER_MEDIA_PROXY_MODE: upload FLUXER_MEDIA_PROXY_STORAGE_BACKEND: s3 - FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media - FLUXER_MEDIA_PROXY_CORS_MODE: ${FLUXER_MEDIA_PROXY_CORS_MODE:-off} + FLUXER_MEDIA_PROXY_CORS_MODE: ${FLUXER_MEDIA_PROXY_CORS_MODE:-} FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS: ${FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}:${FLUXER_PUBLIC_PORT:-443}}} - FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE: ${FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE:-off} - FLUXER_S3_READ_SIGNED: "true" + FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE: ${FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE:-} + FLUXER_MEDIA_PROXY_READ_ONLY: ${FLUXER_MEDIA_PROXY_READ_ONLY:-} + FLUXER_MEDIA_PROXY_NSFW_THRESHOLD: ${FLUXER_MEDIA_PROXY_NSFW_THRESHOLD:-} + FLUXER_NSFW_SERVICE_ENDPOINT: ${FLUXER_NSFW_SERVICE_ENDPOINT:-} + FLUXER_MEDIA_PROXY_MAX_NATIVE_TRANSFORMS: ${FLUXER_MEDIA_PROXY_MAX_NATIVE_TRANSFORMS:-} + FLUXER_MEDIA_PROXY_WORKER_QUEUE_CAPACITY: ${FLUXER_MEDIA_PROXY_WORKER_QUEUE_CAPACITY:-} + FLUXER_MEDIA_PROXY_MAX_ENCODE_DURATION_MS: ${FLUXER_MEDIA_PROXY_MAX_ENCODE_DURATION_MS:-} + FLUXER_MEDIA_PROXY_MAX_ENCODE_FRAMES: ${FLUXER_MEDIA_PROXY_MAX_ENCODE_FRAMES:-} + FLUXER_MEDIA_PROXY_TRANSFORM_TIMEOUT_MS: ${FLUXER_MEDIA_PROXY_TRANSFORM_TIMEOUT_MS:-} + FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_BYTES: ${FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_BYTES:-} + FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_MAX_ENTRY_BYTES: ${FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_MAX_ENTRY_BYTES:-} + FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_TTL_MS: ${FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_TTL_MS:-} + FLUXER_MEDIA_PROXY_SOCKET_IO_TIMEOUT_MS: ${FLUXER_MEDIA_PROXY_SOCKET_IO_TIMEOUT_MS:-} + FLUXER_MEDIA_PROXY_SHUTDOWN_GRACE_MS: ${FLUXER_MEDIA_PROXY_SHUTDOWN_GRACE_MS:-} + FLUXER_MEDIA_PROXY_UPLOAD_RELAY_S3_TIMEOUT_MS: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_S3_TIMEOUT_MS:-} + FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_BYTES: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_BYTES:-} + FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_TOTAL_BYTES: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_TOTAL_BYTES:-} + FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_DIR: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_DIR:-} + FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_CHUNK_BYTES: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_CHUNK_BYTES:-} + FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_MAX_TOTAL_BYTES: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_MAX_TOTAL_BYTES:-} + FLUXER_S3_SESSION_TOKEN: ${FLUXER_S3_SESSION_TOKEN:-} + FLUXER_S3_READ_ENDPOINT: ${FLUXER_S3_READ_ENDPOINT:-} + FLUXER_S3_READ_BUCKET: ${FLUXER_S3_READ_BUCKET:-} + FLUXER_S3_READ_BUCKET_STYLE: ${FLUXER_S3_READ_BUCKET_STYLE:-} + FLUXER_S3_READ_SIGNED: "${FLUXER_S3_READ_SIGNED:-true}" depends_on: seaweedfs-init: {condition: service_completed_successfully} nats: {condition: service_healthy} @@ -483,17 +603,26 @@ services: memory: ${FLUXER_PUSH_MEMORY_LIMIT:-256mb} environment: <<: *fluxer-env - FLUXER_PUSH_SERVICE_HOST: 0.0.0.0 - FLUXER_PUSH_SERVICE_PORT: "8126" - FLUXER_PUSH_SERVICE_QUEUE_CAPACITY: "${FLUXER_PUSH_SERVICE_QUEUE_CAPACITY:-}" - FLUXER_PUSH_SERVICE_SEND_CONCURRENCY: "${FLUXER_PUSH_SERVICE_SEND_CONCURRENCY:-}" + FLUXER_PUSH_SERVICE_QUEUE_CAPACITY: ${FLUXER_PUSH_SERVICE_QUEUE_CAPACITY:-} + FLUXER_PUSH_SERVICE_SEND_CONCURRENCY: ${FLUXER_PUSH_SERVICE_SEND_CONCURRENCY:-} + FLUXER_PUSH_SERVICE_APNS_BASE_URL: ${FLUXER_PUSH_SERVICE_APNS_BASE_URL:-} + FLUXER_PUSH_SERVICE_FCM_BASE_URL: ${FLUXER_PUSH_SERVICE_FCM_BASE_URL:-} + FLUXER_PUSH_SERVICE_MANAGED_RELAY_HOSTS: ${FLUXER_PUSH_SERVICE_MANAGED_RELAY_HOSTS:-} + FLUXER_PUSH_SERVICE_OWN_RELAY_HOSTS: ${FLUXER_PUSH_SERVICE_OWN_RELAY_HOSTS:-} + FLUXER_PUSH_SERVICE_RELAY_CONSENT_ACCEPTED: ${FLUXER_PUSH_SERVICE_RELAY_CONSENT_ACCEPTED:-} + FLUXER_PUSH_APNS_DEFAULT_ENVIRONMENT: ${FLUXER_PUSH_APNS_DEFAULT_ENVIRONMENT:-} + FLUXER_PUSH_FCM_ENABLED: ${FLUXER_PUSH_FCM_ENABLED:-} + FLUXER_PUSH_FCM_PROJECT_ID: ${FLUXER_PUSH_FCM_PROJECT_ID:-} + FLUXER_PUSH_FCM_CLIENT_EMAIL: ${FLUXER_PUSH_FCM_CLIENT_EMAIL:-} + FLUXER_PUSH_FCM_PRIVATE_KEY: ${FLUXER_PUSH_FCM_PRIVATE_KEY:-} + FLUXER_PUSH_FCM_PRIVATE_KEY_PATH: ${FLUXER_PUSH_FCM_PRIVATE_KEY_PATH:-} + FLUXER_PUSH_FCM_SERVICE_ACCOUNT_JSON_PATH: ${FLUXER_PUSH_FCM_SERVICE_ACCOUNT_JSON_PATH:-} + FLUXER_PUSH_FCM_TOKEN_URI: ${FLUXER_PUSH_FCM_TOKEN_URI:-} + FLUXER_PUSH_FCM_APPS: ${FLUXER_PUSH_FCM_APPS:-} healthcheck: test: ["CMD", "/usr/local/bin/fluxer-push", "healthcheck"] - interval: 10s - timeout: 5s - retries: 30 - start_period: 60s - start_interval: 1s + <<: *fluxer-app-healthcheck + start_period: ${FLUXER_SVC_HEALTHCHECK_START_PERIOD:-60s} depends_on: nats: {condition: service_healthy} api: {condition: service_healthy} @@ -507,9 +636,9 @@ services: memory: ${FLUXER_STATIC_PROXY_MEMORY_LIMIT:-256mb} healthcheck: test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/avatars/0.png"] - interval: 10s - timeout: 5s - retries: 10 + interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s} + timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s} + retries: ${FLUXER_HEALTHCHECK_RETRIES:-10} app-proxy: <<: *fluxer-service @@ -519,15 +648,29 @@ services: limits: memory: ${FLUXER_APP_PROXY_MEMORY_LIMIT:-256mb} environment: - FLUXER_APP_PROXY_HOST: 0.0.0.0 + RUST_LOG: ${RUST_LOG:-} FLUXER_APP_PROXY_PORT: "8080" FLUXER_BASE_DOMAIN: ${FLUXER_DOMAIN:?set FLUXER_DOMAIN in .env} FLUXER_PUBLIC_SCHEME: ${FLUXER_PUBLIC_SCHEME:-https} FLUXER_PUBLIC_PORT: ${FLUXER_PUBLIC_PORT:-443} FLUXER_PUBLIC_ORIGIN: ${FLUXER_PUBLIC_ORIGIN:-} + FLUXER_TRUST_CLIENT_IP_HEADER: "${FLUXER_TRUST_CLIENT_IP_HEADER:-true}" + FLUXER_CLIENT_IP_HEADER_NAME: ${FLUXER_CLIENT_IP_HEADER_NAME:-} + FLUXER_GEOIP_DB_PATH: ${FLUXER_GEOIP_DB_PATH:-} + FLUXER_S3_ENDPOINT: ${FLUXER_S3_ENDPOINT:-http://seaweedfs:8333} + FLUXER_S3_PUBLIC_ENDPOINT: ${FLUXER_S3_PUBLIC_ENDPOINT:-} + FLUXER_S3_REGION: ${FLUXER_S3_REGION:-us-east-1} + FLUXER_S3_ACCESS_KEY_ID: ${FLUXER_S3_ACCESS_KEY:?set FLUXER_S3_ACCESS_KEY in .env} + FLUXER_S3_SECRET_ACCESS_KEY: ${FLUXER_S3_SECRET_KEY:?set FLUXER_S3_SECRET_KEY in .env} + FLUXER_S3_BUCKET_UPLOADS: ${FLUXER_S3_BUCKET_UPLOADS:-} + FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_STATIC_CDN_ENDPOINT:-} DISCOVERY_UPSTREAM_URL: http://edge:8088/.well-known/fluxer + DISCOVERY_REFRESH_INTERVAL_MS: ${DISCOVERY_REFRESH_INTERVAL_MS:-} PUBLIC_BOOTSTRAP_API_ENDPOINT: /api - PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/api + PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT: ${PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/api} + FLUXER_APP_PROXY_INDEX_UPSTREAM_URL: ${FLUXER_APP_PROXY_INDEX_UPSTREAM_URL:-} + FLUXER_APP_PROXY_SAME_ORIGIN_HOSTS: ${FLUXER_APP_PROXY_SAME_ORIGIN_HOSTS:-} + FLUXER_APP_PROXY_MANIFEST_SCOPE_EXTENSIONS: ${FLUXER_APP_PROXY_MANIFEST_SCOPE_EXTENSIONS:-} FLUXER_CSP_EXTRA_DEFAULT_SRC: ${FLUXER_CSP_EXTRA_DEFAULT_SRC:-} FLUXER_CSP_EXTRA_CONNECT_SRC: ${FLUXER_CSP_EXTRA_CONNECT_SRC:-} FLUXER_CSP_EXTRA_IMG_SRC: ${FLUXER_CSP_EXTRA_IMG_SRC:-} @@ -585,7 +728,6 @@ services: <<: *fluxer-env FLUXER_SVC_NAME: users FLUXER_SVC_MODE: router - FLUXER_SVC_MAX_CONCURRENT_REQUESTS: "${FLUXER_SVC_MAX_CONCURRENT_REQUESTS:-}" healthcheck: *fluxer-svc-healthcheck depends_on: nats: {condition: service_healthy} @@ -602,8 +744,7 @@ services: FLUXER_SVC_NAME: users FLUXER_SVC_MODE: shard FLUXER_SVC_SHARD_ID: "0" - FLUXER_POSTGRES_MAX_CONNECTIONS: "20" - FLUXER_SVC_MAX_CONCURRENT_REQUESTS: "${FLUXER_SVC_MAX_CONCURRENT_REQUESTS:-}" + FLUXER_POSTGRES_MAX_CONNECTIONS: "${FLUXER_USERS_SHARD_POSTGRES_MAX_CONNECTIONS:-20}" healthcheck: *fluxer-svc-healthcheck depends_on: nats: {condition: service_healthy} @@ -620,7 +761,6 @@ services: <<: *fluxer-env FLUXER_SVC_NAME: gifs FLUXER_SVC_MODE: router - FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media healthcheck: *fluxer-svc-healthcheck depends_on: nats: {condition: service_healthy} @@ -637,7 +777,7 @@ services: FLUXER_SVC_NAME: gifs FLUXER_SVC_MODE: shard FLUXER_SVC_SHARD_ID: "0" - FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media + FLUXER_GIFS_SHARD_CACHE_MAX_BYTES: ${FLUXER_GIFS_SHARD_CACHE_MAX_BYTES:-} healthcheck: *fluxer-svc-healthcheck depends_on: nats: {condition: service_healthy} @@ -653,7 +793,6 @@ services: <<: *fluxer-env FLUXER_SVC_NAME: messages FLUXER_SVC_MODE: router - FLUXER_SVC_MAX_CONCURRENT_REQUESTS: "${FLUXER_SVC_MAX_CONCURRENT_REQUESTS:-}" healthcheck: *fluxer-svc-healthcheck depends_on: nats: {condition: service_healthy} @@ -670,8 +809,7 @@ services: FLUXER_SVC_NAME: messages FLUXER_SVC_MODE: shard FLUXER_SVC_SHARD_ID: "0" - FLUXER_POSTGRES_MAX_CONNECTIONS: "20" - FLUXER_SVC_MAX_CONCURRENT_REQUESTS: "${FLUXER_SVC_MAX_CONCURRENT_REQUESTS:-}" + FLUXER_POSTGRES_MAX_CONNECTIONS: "${FLUXER_MESSAGES_SHARD_POSTGRES_MAX_CONNECTIONS:-20}" healthcheck: *fluxer-svc-healthcheck depends_on: nats: {condition: service_healthy} @@ -688,8 +826,6 @@ services: <<: *fluxer-env FLUXER_SVC_NAME: unfurl FLUXER_SVC_MODE: router - FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media - FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}} healthcheck: *fluxer-svc-healthcheck depends_on: nats: {condition: service_healthy} @@ -706,8 +842,9 @@ services: FLUXER_SVC_NAME: unfurl FLUXER_SVC_MODE: shard FLUXER_SVC_SHARD_ID: "0" - FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media - FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}} + FLUXER_MEDIA_PROXY_ENDPOINT: http://media-proxy:8080 + FLUXER_UNFURL_STATIC_CDN_ENDPOINT: ${FLUXER_UNFURL_STATIC_CDN_ENDPOINT:-} + FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_STATIC_CDN_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}} healthcheck: *fluxer-svc-healthcheck depends_on: nats: {condition: service_healthy} @@ -721,22 +858,14 @@ services: memory: ${FLUXER_ADMIN_MEMORY_LIMIT:-256mb} environment: <<: *fluxer-env - FLUXER_ADMIN_HOST: 0.0.0.0 FLUXER_ADMIN_PORT: "8080" - FLUXER_ADMIN_BASE_PATH: /admin + FLUXER_ADMIN_BASE_PATH: ${FLUXER_ADMIN_BASE_PATH:-/admin} FLUXER_API_ENDPOINT: http://api:8080 - FLUXER_ADMIN_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/admin - FLUXER_APP_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}} - FLUXER_MEDIA_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media - FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}} - FLUXER_ADMIN_OAUTH_REDIRECT_URI: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/admin/oauth2_callback + FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_STATIC_CDN_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}} healthcheck: test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/8080 && printf 'GET /_health HTTP/1.0\\r\\n\\r\\n' >&3 && head -n 1 <&3 | grep -q ' 200 '"] - interval: 10s - timeout: 5s - retries: 30 - start_period: 60s - start_interval: 1s + <<: *fluxer-app-healthcheck + start_period: ${FLUXER_SVC_HEALTHCHECK_START_PERIOD:-60s} depends_on: api: {condition: service_healthy} diff --git a/fluxer_admin/Cargo.toml b/fluxer_admin/Cargo.toml index 539f9ea96..d64e254a0 100644 --- a/fluxer_admin/Cargo.toml +++ b/fluxer_admin/Cargo.toml @@ -26,7 +26,7 @@ tokio = { version = "1.53.1", features = ["macros", "net", "rt-multi-thread", "s tower = { version = "0.5.3", features = ["util"] } tower-http = { version = "0.7.1", features = ["compression-gzip", "trace"] } tracing = "0.1.44" -tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } +tracing-subscriber = "0.3.23" url = "2.5" urlencoding = "2.1.3" progenitor-client = { version = "0.15.0", default-features = false } diff --git a/fluxer_admin/Dockerfile b/fluxer_admin/Dockerfile index c83ef17c2..03204a8c1 100644 --- a/fluxer_admin/Dockerfile +++ b/fluxer_admin/Dockerfile @@ -2,7 +2,6 @@ FROM rust:1-trixie AS builder -ARG BUILD_VERSION="" ARG TARGETARCH WORKDIR /usr/src/app @@ -45,8 +44,6 @@ RUN printf '%s\n' \ 'strip = "symbols"' \ > Cargo.toml -ENV FLUXER_BUILD_VERSION="${BUILD_VERSION}" - RUN cargo build --release -p fluxer_admin \ && cp target/release/fluxer_admin /usr/local/bin/fluxer-admin diff --git a/fluxer_admin/src/config/mod.rs b/fluxer_admin/src/config/mod.rs index 5291bc20d..df47aeeaf 100644 --- a/fluxer_admin/src/config/mod.rs +++ b/fluxer_admin/src/config/mod.rs @@ -1,7 +1,9 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -use fluxer_common::config::normalize_public_endpoint_from_env; -use std::env; +use fluxer_common::config::{ + normalize_base_path, normalize_public_endpoint_from_env, read_bool_env, read_env, + read_first_env, trim_trailing_slash, +}; const DEFAULT_ADMIN_OAUTH_CLIENT_ID: &str = "1234567890123456789"; @@ -17,12 +19,10 @@ pub struct AdminConfig { pub static_cdn_endpoint: String, pub admin_endpoint: String, pub web_app_endpoint: String, - pub kv_url: String, pub oauth_client_id: String, pub oauth_client_secret: String, pub oauth_redirect_uri: String, pub build_version: String, - pub release_channel: String, pub self_hosted: bool, pub proxy: ProxyConfig, } @@ -47,8 +47,8 @@ impl AdminConfig { "FLUXER_ADMIN_ENDPOINT", "https://admin.fluxer.app", ))); - let oauth_redirect_uri = normalize_public_endpoint_from_env(&read_env_preferred( - &["FLUXER_ADMIN_OAUTH_REDIRECT_URI"], + let oauth_redirect_uri = normalize_public_endpoint_from_env(&read_env( + "FLUXER_ADMIN_OAUTH_REDIRECT_URI", &format!("{admin_endpoint}/oauth2_callback"), )); let secret_key_base = read_env("FLUXER_ADMIN_SECRET_KEY_BASE", ""); @@ -82,38 +82,22 @@ impl AdminConfig { "FLUXER_APP_ENDPOINT", "https://app.fluxer.app", ))), - kv_url: read_env("FLUXER_KV_URL", ""), oauth_client_id: read_env( "FLUXER_ADMIN_OAUTH_CLIENT_ID", DEFAULT_ADMIN_OAUTH_CLIENT_ID, ), oauth_client_secret: read_env("FLUXER_ADMIN_OAUTH_CLIENT_SECRET", ""), oauth_redirect_uri, - build_version: read_env_preferred( + build_version: read_first_env( &["BUILD_VERSION", "FLUXER_BUILD_VERSION"], env!("CARGO_PKG_VERSION"), ), - release_channel: read_env_preferred( - &["RELEASE_CHANNEL", "FLUXER_RELEASE_CHANNEL"], - "stable", - ), - self_hosted: read_bool_env(&["FLUXER_SELF_HOSTED"], false), + self_hosted: read_bool_env("FLUXER_SELF_HOSTED", false), proxy: ProxyConfig { - trust_client_ip_header: read_bool_env( - &["FLUXER_TRUST_CLIENT_IP_HEADER", "TRUST_CLIENT_IP_HEADER"], - false, - ), - client_ip_header_name: read_env_preferred( - &[ - "FLUXER_CLIENT_IP_HEADER_NAME", - "FLUXER_CLIENT_IP_HEADER", - "CLIENT_IP_HEADER_NAME", - "CLIENT_IP_HEADER", - ], - "x-forwarded-for", - ) - .trim() - .to_ascii_lowercase(), + trust_client_ip_header: read_bool_env("FLUXER_TRUST_CLIENT_IP_HEADER", false), + client_ip_header_name: read_env("FLUXER_CLIENT_IP_HEADER_NAME", "x-forwarded-for") + .trim() + .to_ascii_lowercase(), }, }) } @@ -146,55 +130,21 @@ impl RuntimeEnv { } } -pub fn normalize_base_path(value: &str) -> String { - let trimmed = value.trim().trim_matches('/'); - if trimmed.is_empty() { - String::new() - } else { - format!("/{trimmed}") - } -} - -pub fn trim_trailing_slash(value: &str) -> String { - value.trim_end_matches('/').to_owned() -} - -pub(crate) fn read_env(name: &str, fallback: &str) -> String { - env::var(name).unwrap_or_else(|_| fallback.to_owned()) -} - -pub(crate) fn read_env_preferred(names: &[&str], fallback: &str) -> String { - names - .iter() - .find_map(|name| env::var(name).ok().filter(|value| !value.trim().is_empty())) - .unwrap_or_else(|| fallback.to_owned()) -} - -pub(crate) fn read_bool_env(names: &[&str], fallback: bool) -> bool { - let Some(value) = names.iter().find_map(|name| env::var(name).ok()) else { - return fallback; - }; - matches!( - value.trim().to_ascii_lowercase().as_str(), - "1" | "true" | "yes" | "on" - ) -} - #[cfg(test)] mod tests { use super::*; + use std::env; use std::sync::Mutex; static ENV_LOCK: Mutex<()> = Mutex::new(()); - const MANAGED_ENV: [&str; 11] = [ + const MANAGED_ENV: [&str; 10] = [ "FLUXER_ENV", "FLUXER_ADMIN_HOST", "FLUXER_ADMIN_PORT", "FLUXER_ADMIN_ENDPOINT", "FLUXER_ADMIN_OAUTH_CLIENT_ID", "FLUXER_ADMIN_OAUTH_REDIRECT_URI", - "FLUXER_MASTER_CONFIG", "FLUXER_APP_ENDPOINT", "FLUXER_MEDIA_ENDPOINT", "FLUXER_STATIC_CDN_ENDPOINT", @@ -291,12 +241,10 @@ mod tests { admin_endpoint: String::new(), web_app_endpoint: String::new(), - kv_url: String::new(), oauth_client_id: String::new(), oauth_client_secret: String::new(), oauth_redirect_uri: String::new(), build_version: String::new(), - release_channel: String::new(), self_hosted: false, proxy: ProxyConfig { trust_client_ip_header: false, @@ -321,12 +269,10 @@ mod tests { admin_endpoint: String::new(), web_app_endpoint: String::new(), - kv_url: String::new(), oauth_client_id: String::new(), oauth_client_secret: String::new(), oauth_redirect_uri: String::new(), build_version: String::new(), - release_channel: String::new(), self_hosted: false, proxy: ProxyConfig { trust_client_ip_header: false, diff --git a/fluxer_admin/src/main.rs b/fluxer_admin/src/main.rs index 675d6e927..881ef9373 100644 --- a/fluxer_admin/src/main.rs +++ b/fluxer_admin/src/main.rs @@ -8,9 +8,7 @@ use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; #[tokio::main] async fn main() -> anyhow::Result<()> { tracing_subscriber::registry() - .with( - tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()), - ) + .with(fluxer_common::config::env_filter("info")) .with(tracing_subscriber::fmt::layer()) .init(); diff --git a/fluxer_admin/src/middleware/csrf.rs b/fluxer_admin/src/middleware/csrf.rs index 61f15cd29..fc938cdfb 100644 --- a/fluxer_admin/src/middleware/csrf.rs +++ b/fluxer_admin/src/middleware/csrf.rs @@ -215,12 +215,10 @@ mod tests { static_cdn_endpoint: String::new(), admin_endpoint: admin_endpoint.to_owned(), web_app_endpoint: String::new(), - kv_url: String::new(), oauth_client_id: String::new(), oauth_client_secret: String::new(), oauth_redirect_uri: String::new(), build_version: "test".to_owned(), - release_channel: String::new(), self_hosted: false, proxy: ProxyConfig { trust_client_ip_header: false, diff --git a/fluxer_admin/tests/csrf_legacy_cookie.rs b/fluxer_admin/tests/csrf_legacy_cookie.rs index 0af0fb5ec..4aadfacdc 100644 --- a/fluxer_admin/tests/csrf_legacy_cookie.rs +++ b/fluxer_admin/tests/csrf_legacy_cookie.rs @@ -95,12 +95,10 @@ fn production_config(api_endpoint: String) -> AdminConfig { static_cdn_endpoint: "https://static.example.test".to_owned(), admin_endpoint: ADMIN_ORIGIN.to_owned(), web_app_endpoint: "https://app.example.test".to_owned(), - kv_url: String::new(), oauth_client_id: "admin-client".to_owned(), oauth_client_secret: "admin-secret".to_owned(), oauth_redirect_uri: "https://admin.example.test/callback".to_owned(), build_version: "test".to_owned(), - release_channel: "test".to_owned(), self_hosted: false, proxy: ProxyConfig { trust_client_ip_header: false, diff --git a/fluxer_admin/tests/htmx_acceptance.rs b/fluxer_admin/tests/htmx_acceptance.rs index b279f0be2..0dae89051 100644 --- a/fluxer_admin/tests/htmx_acceptance.rs +++ b/fluxer_admin/tests/htmx_acceptance.rs @@ -1298,12 +1298,10 @@ fn test_config(api_endpoint: String) -> AdminConfig { static_cdn_endpoint: "https://static.example.test".to_owned(), admin_endpoint: "https://admin.example.test".to_owned(), web_app_endpoint: "https://app.example.test".to_owned(), - kv_url: String::new(), oauth_client_id: "admin-client".to_owned(), oauth_client_secret: "admin-secret".to_owned(), oauth_redirect_uri: "https://admin.example.test/callback".to_owned(), build_version: "test".to_owned(), - release_channel: "test".to_owned(), self_hosted: false, proxy: ProxyConfig { trust_client_ip_header: false, diff --git a/fluxer_admin/tests/parity/rust_server.rs b/fluxer_admin/tests/parity/rust_server.rs index 142d85f0d..006c22e46 100644 --- a/fluxer_admin/tests/parity/rust_server.rs +++ b/fluxer_admin/tests/parity/rust_server.rs @@ -55,12 +55,10 @@ fn admin_config(port: u16, api_endpoint: &str, admin_endpoint: &str) -> AdminCon static_cdn_endpoint: "https://static.example.test".to_owned(), admin_endpoint: admin_endpoint.to_owned(), web_app_endpoint: "http://127.0.0.1:8088".to_owned(), - kv_url: "redis://127.0.0.1:6379/0".to_owned(), oauth_client_id: "1234567890123456789".to_owned(), oauth_client_secret: "test-admin-oauth-secret".to_owned(), oauth_redirect_uri: format!("{admin_endpoint}/oauth2_callback"), build_version: "parity".to_owned(), - release_channel: "parity".to_owned(), self_hosted: false, proxy: ProxyConfig { trust_client_ip_header: false, diff --git a/fluxer_admin/tests/voice_restriction_writes.rs b/fluxer_admin/tests/voice_restriction_writes.rs index 06958d0a3..e4ad107b7 100644 --- a/fluxer_admin/tests/voice_restriction_writes.rs +++ b/fluxer_admin/tests/voice_restriction_writes.rs @@ -326,12 +326,10 @@ fn test_config(api_endpoint: String) -> AdminConfig { static_cdn_endpoint: "https://static.example.test".to_owned(), admin_endpoint: "https://admin.example.test".to_owned(), web_app_endpoint: "https://app.example.test".to_owned(), - kv_url: String::new(), oauth_client_id: "admin-client".to_owned(), oauth_client_secret: "admin-secret".to_owned(), oauth_redirect_uri: "https://admin.example.test/callback".to_owned(), build_version: "test".to_owned(), - release_channel: "test".to_owned(), self_hosted: false, proxy: ProxyConfig { trust_client_ip_header: false, diff --git a/fluxer_api/pkgs/kv_client/src/KVClient.ts b/fluxer_api/pkgs/kv_client/src/KVClient.ts index f77d5fad4..632795e2c 100644 --- a/fluxer_api/pkgs/kv_client/src/KVClient.ts +++ b/fluxer_api/pkgs/kv_client/src/KVClient.ts @@ -302,10 +302,8 @@ export class KVClient implements IKVProvider { } private createClusterClient(clusterConfig: ResolvedKVClientConfig): Cluster { - const {nodes, redisOptions} = resolveKVClusterConnection(clusterConfig.url, clusterConfig.clusterNodes); - const natMap = clusterConfig.clusterNatMap; - const hasNatMap = Object.keys(natMap).length > 0; - return new Cluster(nodes, { + const {node, redisOptions} = resolveKVClusterConnection(clusterConfig.url); + return new Cluster([node], { clusterRetryStrategy: createRetryStrategy(), redisOptions: { ...redisOptions, @@ -315,7 +313,6 @@ export class KVClient implements IKVProvider { protocol: 2, }, scaleReads: 'master', - ...(hasNatMap ? {natMap} : {}), }); } @@ -591,7 +588,6 @@ export class KVClient implements IKVProvider { return new KVSubscription({ url: this.url, mode: this.config.mode, - clusterNodes: this.config.clusterNodes, timeoutMs: this.timeoutMs, logger: this.logger, }); diff --git a/fluxer_api/pkgs/kv_client/src/KVClientConfig.ts b/fluxer_api/pkgs/kv_client/src/KVClientConfig.ts index b1a6a3bd9..8eaa2bbee 100644 --- a/fluxer_api/pkgs/kv_client/src/KVClientConfig.ts +++ b/fluxer_api/pkgs/kv_client/src/KVClientConfig.ts @@ -9,16 +9,9 @@ export interface IKVLogger { export type KVClientMode = 'standalone' | 'cluster'; -export interface KVClusterNode { - host: string; - port: number; -} - export interface KVClientConfig { url: string; mode?: KVClientMode; - clusterNodes?: Array; - clusterNatMap?: Record; timeoutMs?: number; logger?: IKVLogger; } @@ -26,8 +19,6 @@ export interface KVClientConfig { export interface ResolvedKVClientConfig { url: string; mode: KVClientMode; - clusterNodes: Array; - clusterNatMap: Record; timeoutMs: number; logger: IKVLogger; } @@ -42,8 +33,6 @@ export function resolveKVClientConfig(config: KVClientConfig | string): Resolved return { url: normalizeUrl(options.url), mode: options.mode ?? 'standalone', - clusterNodes: options.clusterNodes ?? [], - clusterNatMap: options.clusterNatMap ?? {}, timeoutMs: options.timeoutMs ?? DEFAULT_KV_TIMEOUT_MS, logger: options.logger ?? noopLogger, }; diff --git a/fluxer_api/pkgs/kv_client/src/KVClusterConnection.ts b/fluxer_api/pkgs/kv_client/src/KVClusterConnection.ts index 49c3cba3f..61eedcaab 100644 --- a/fluxer_api/pkgs/kv_client/src/KVClusterConnection.ts +++ b/fluxer_api/pkgs/kv_client/src/KVClusterConnection.ts @@ -1,13 +1,12 @@ import {domainToASCII} from 'node:url'; -import type {KVClusterNode} from '@pkgs/kv_client/src/KVClientConfig'; import type {RedisOptions} from 'ioredis'; interface KVClusterConnection { - nodes: Array; + node: {host: string; port: number}; redisOptions: RedisOptions; } -export function resolveKVClusterConnection(url: string, nodes: ReadonlyArray): KVClusterConnection { +export function resolveKVClusterConnection(url: string): KVClusterConnection { const normalizedUrl = url.trim(); for (let index = 0; index < normalizedUrl.length; index++) { const code = normalizedUrl.charCodeAt(index); @@ -46,17 +45,8 @@ export function resolveKVClusterConnection(url: string, nodes: ReadonlyArray 0 ? [...nodes] : [{host, port: Number(parsed.port || '6379')}]; - for (const node of resolvedNodes) { - if (node.host.trim().length === 0) { - throw new Error('KV cluster node must include a host'); - } - if (!Number.isInteger(node.port) || node.port < 1 || node.port > 65535) { - throw new Error('KV cluster node port must be an integer between 1 and 65535'); - } - } return { - nodes: resolvedNodes, + node: {host, port: Number(parsed.port || '6379')}, redisOptions, }; } diff --git a/fluxer_api/pkgs/kv_client/src/KVSubscription.ts b/fluxer_api/pkgs/kv_client/src/KVSubscription.ts index f481b623f..4f15b7670 100644 --- a/fluxer_api/pkgs/kv_client/src/KVSubscription.ts +++ b/fluxer_api/pkgs/kv_client/src/KVSubscription.ts @@ -1,14 +1,13 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import type {IKVSubscription} from '@pkgs/kv_client/src/IKVProvider'; -import type {IKVLogger, KVClientMode, KVClusterNode} from '@pkgs/kv_client/src/KVClientConfig'; +import type {IKVLogger, KVClientMode} from '@pkgs/kv_client/src/KVClientConfig'; import {resolveKVClusterConnection} from '@pkgs/kv_client/src/KVClusterConnection'; import Redis, {type RedisOptions} from 'ioredis'; interface KVSubscriptionConfig { url: string; mode?: KVClientMode; - clusterNodes?: Array; timeoutMs: number; logger: IKVLogger; } @@ -21,7 +20,6 @@ interface KVSubscriptionConnect { export class KVSubscription implements IKVSubscription { private readonly url: string; private readonly mode: KVClientMode; - private readonly clusterNodes: Array; private readonly timeoutMs: number; private readonly logger: IKVLogger; private readonly desiredChannels = new Set(); @@ -34,7 +32,6 @@ export class KVSubscription implements IKVSubscription { constructor(config: KVSubscriptionConfig) { this.url = config.url; this.mode = config.mode ?? 'standalone'; - this.clusterNodes = config.clusterNodes ?? []; this.timeoutMs = config.timeoutMs; this.logger = config.logger; } @@ -75,9 +72,9 @@ export class KVSubscription implements IKVSubscription { protocol: 2, retryStrategy: createRetryStrategy(), }; - const connection = this.mode === 'cluster' ? resolveKVClusterConnection(this.url, this.clusterNodes) : null; + const connection = this.mode === 'cluster' ? resolveKVClusterConnection(this.url) : null; const client = connection - ? new Redis({...connection.redisOptions, ...connection.nodes[0], db: 0, ...options}) + ? new Redis({...connection.redisOptions, ...connection.node, db: 0, ...options}) : new Redis(this.url, options); client.on('message', (channel: string, message: string) => { if (this.client !== client || this.closing !== null) { diff --git a/fluxer_api/pkgs/mime_utils/package.json b/fluxer_api/pkgs/mime_utils/package.json index 3513ccb64..984e2dd17 100644 --- a/fluxer_api/pkgs/mime_utils/package.json +++ b/fluxer_api/pkgs/mime_utils/package.json @@ -2,10 +2,7 @@ "name": "@pkgs/mime_utils", "version": "0.0.0", "type": "module", - "main": "./src/index.ts", - "types": "./src/index.ts", "exports": { - ".": "./src/index.ts", "./src/*": "./src/*", "./*": "./*" }, diff --git a/fluxer_api/pkgs/mime_utils/src/index.ts b/fluxer_api/pkgs/mime_utils/src/index.ts deleted file mode 100644 index ecabbc9b7..000000000 --- a/fluxer_api/pkgs/mime_utils/src/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -export { - getContentTypeFromFilename, - isSupportedMediaContentType, - normalizeContentType, -} from '@pkgs/mime_utils/src/ContentTypeUtils'; diff --git a/fluxer_api/pkgs/worker/package.json b/fluxer_api/pkgs/worker/package.json index 78e90fafe..312bcf8aa 100644 --- a/fluxer_api/pkgs/worker/package.json +++ b/fluxer_api/pkgs/worker/package.json @@ -4,10 +4,6 @@ "private": true, "type": "module", "exports": { - "./WorkerFactory": { - "import": "./src/runtime/WorkerFactory.ts", - "types": "./src/runtime/WorkerFactory.ts" - }, "./WorkerContext": { "import": "./src/context/WorkerContext.ts", "types": "./src/context/WorkerContext.ts" @@ -24,39 +20,13 @@ "import": "./src/contracts/WorkerTypes.ts", "types": "./src/contracts/WorkerTypes.ts" }, - "./IQueueProvider": { - "import": "./src/providers/IQueueProvider.ts", - "types": "./src/providers/IQueueProvider.ts" - }, - "./HttpWorkerQueue": { - "import": "./src/providers/HttpWorkerQueue.ts", - "types": "./src/providers/HttpWorkerQueue.ts" - }, - "./QueueProviderFactory": { - "import": "./src/providers/QueueProviderFactory.ts", - "types": "./src/providers/QueueProviderFactory.ts" - }, - "./WorkerRunner": { - "import": "./src/runtime/WorkerRunner.ts", - "types": "./src/runtime/WorkerRunner.ts" - }, - "./WorkerService": { - "import": "./src/services/WorkerService.ts", - "types": "./src/services/WorkerService.ts" - }, - "./WorkerTaskRegistry": { - "import": "./src/runtime/WorkerTaskRegistry.ts", - "types": "./src/runtime/WorkerTaskRegistry.ts" - }, "./*": "./*" }, "scripts": { "typecheck": "tsc --noEmit" }, "dependencies": { - "@fluxer/constants": "workspace:*", - "@fluxer/logger": "workspace:*", - "itty-time": "catalog:" + "@fluxer/logger": "workspace:*" }, "devDependencies": { "@types/node": "catalog:", diff --git a/fluxer_api/pkgs/worker/src/contracts/WorkerTypes.ts b/fluxer_api/pkgs/worker/src/contracts/WorkerTypes.ts index 378a53417..74764ff45 100644 --- a/fluxer_api/pkgs/worker/src/contracts/WorkerTypes.ts +++ b/fluxer_api/pkgs/worker/src/contracts/WorkerTypes.ts @@ -2,56 +2,6 @@ export type WorkerJobPayload = Record; -export interface WorkerRuntimeConfig { - workerId?: string | undefined; - concurrency?: number | undefined; - taskTypes?: Array | undefined; -} - -export interface WorkerQueueConfig { - queueBaseUrl: string; - requestTimeoutMs?: number | undefined; -} - -export interface WorkerConfig extends WorkerRuntimeConfig, WorkerQueueConfig {} - -export interface TracingInterface { - withSpan( - options: { - name: string; - attributes?: Record; - }, - fn: () => Promise, - ): Promise; - addSpanEvent(name: string, attributes?: Record): void; - setSpanAttributes(attributes: Record): void; -} - -export interface QueueJob { - id: string; - task_type: string; - payload: WorkerJobPayload; - priority: number; - run_at: string; - created_at: string; - attempts: number; - max_attempts: number; - error?: string | null; - deduplication_id?: string | null; -} - -export interface LeasedQueueJob { - receipt: string; - visibility_deadline: string; - job: QueueJob; -} - -export interface EnqueueOptions { - runAt?: Date | undefined; - maxAttempts?: number | undefined; - priority?: number | undefined; -} - export interface WorkerJobOptions { queueName?: string | undefined; runAt?: Date | undefined; diff --git a/fluxer_api/pkgs/worker/src/providers/HttpWorkerQueue.ts b/fluxer_api/pkgs/worker/src/providers/HttpWorkerQueue.ts deleted file mode 100644 index d8e078f74..000000000 --- a/fluxer_api/pkgs/worker/src/providers/HttpWorkerQueue.ts +++ /dev/null @@ -1,234 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {DEFAULT_HTTP_WORKER_TIMEOUT_MS} from '@fluxer/constants/src/Timeouts'; -import type { - EnqueueOptions, - LeasedQueueJob, - TracingInterface, - WorkerJobPayload, -} from '@pkgs/worker/src/contracts/WorkerTypes'; -import type {IQueueProvider} from '@pkgs/worker/src/providers/IQueueProvider'; - -export class HttpWorkerQueue implements IQueueProvider { - private readonly baseUrl: string; - private readonly timeoutMs: number; - private readonly tracing: TracingInterface | undefined; - - constructor(options: { - baseUrl: string; - timeoutMs?: number | undefined; - tracing?: TracingInterface | undefined; - }) { - this.baseUrl = options.baseUrl; - this.timeoutMs = options.timeoutMs ?? DEFAULT_HTTP_WORKER_TIMEOUT_MS; - this.tracing = options.tracing; - } - - private async withResponse( - input: string | URL, - init: RequestInit, - consume: (response: Response) => Promise, - ): Promise { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), this.timeoutMs); - try { - const response = await fetch(input, { - ...init, - signal: controller.signal, - }); - return await consume(response); - } finally { - clearTimeout(timeoutId); - controller.abort(); - } - } - - private async requireSuccess(response: Response, action: string): Promise { - if (response.ok) { - return; - } - const text = await response.text(); - throw new Error(`Failed to ${action}: ${response.status} ${text}`); - } - - private async withOptionalSpan( - options: { - name: string; - attributes?: Record; - }, - fn: () => Promise, - ): Promise { - if (this.tracing) { - return this.tracing.withSpan(options, fn); - } - return fn(); - } - - private addSpanEvent(name: string, attributes?: Record): void { - if (this.tracing) { - this.tracing.addSpanEvent(name, attributes); - } - } - - private setSpanAttributes(attributes: Record): void { - if (this.tracing) { - this.tracing.setSpanAttributes(attributes); - } - } - - async enqueue(taskType: string, payload: WorkerJobPayload, options?: EnqueueOptions): Promise { - return await this.withOptionalSpan( - { - name: 'queue.enqueue', - attributes: { - 'queue.task_type': taskType, - 'queue.priority': options?.priority ?? 0, - 'queue.max_attempts': options?.maxAttempts ?? 5, - 'queue.scheduled': options?.runAt !== undefined, - 'net.peer.name': new URL(this.baseUrl).hostname, - }, - }, - async () => { - const body = { - task_type: taskType, - payload, - priority: options?.priority ?? 0, - run_at: options?.runAt?.toISOString(), - max_attempts: options?.maxAttempts ?? 5, - }; - return this.withResponse( - `${this.baseUrl}/enqueue`, - { - method: 'POST', - headers: {'Content-Type': 'application/json'}, - body: JSON.stringify(body), - }, - async (response) => { - await this.requireSuccess(response, 'enqueue job'); - const jobIdResult = (await response.json()) as { - job_id: string; - }; - this.setSpanAttributes({'queue.job_id': jobIdResult.job_id}); - return jobIdResult.job_id; - }, - ); - }, - ); - } - - async dequeue(taskTypes: Array, limit = 1): Promise> { - return await this.withOptionalSpan( - { - name: 'queue.dequeue', - attributes: { - 'queue.task_types': taskTypes.join(','), - 'queue.limit': limit, - 'queue.service': 'fluxer-queue', - }, - }, - async () => { - this.addSpanEvent('dequeue.start'); - const url = new URL(`${this.baseUrl}/dequeue`); - url.searchParams.set('task_types', taskTypes.join(',')); - url.searchParams.set('limit', limit.toString()); - url.searchParams.set('wait_time_ms', '0'); - return this.withResponse(url, {method: 'GET'}, async (response) => { - await this.requireSuccess(response, 'dequeue job'); - this.addSpanEvent('dequeue.parse_response'); - const jobs = (await response.json()) as Array; - const jobCount = jobs?.length ?? 0; - this.setSpanAttributes({ - 'queue.jobs_returned': jobCount, - 'queue.empty': jobCount === 0, - }); - this.addSpanEvent('dequeue.complete'); - return jobs ?? []; - }); - }, - ); - } - - async upsertCron(id: string, taskType: string, payload: WorkerJobPayload, cronExpression: string): Promise { - await this.withResponse( - `${this.baseUrl}/cron`, - { - method: 'POST', - headers: {'Content-Type': 'application/json'}, - body: JSON.stringify({id, task_type: taskType, payload, cron_expression: cronExpression}), - }, - (response) => this.requireSuccess(response, 'upsert cron job'), - ); - } - - async complete(receipt: string): Promise { - return await this.withOptionalSpan( - { - name: 'queue.complete', - attributes: { - 'queue.receipt': receipt, - }, - }, - async () => - this.withResponse( - `${this.baseUrl}/ack`, - { - method: 'POST', - headers: {'Content-Type': 'application/json'}, - body: JSON.stringify({receipt}), - }, - (response) => this.requireSuccess(response, 'complete job'), - ), - ); - } - - async fail(receipt: string, error: string): Promise { - return await this.withOptionalSpan( - { - name: 'queue.fail', - attributes: { - 'queue.receipt': receipt, - 'queue.error_message': error, - }, - }, - async () => - this.withResponse( - `${this.baseUrl}/nack`, - { - method: 'POST', - headers: {'Content-Type': 'application/json'}, - body: JSON.stringify({receipt, error}), - }, - (response) => this.requireSuccess(response, 'fail job'), - ), - ); - } - - async cancelJob(jobId: string): Promise { - return this.withResponse(`${this.baseUrl}/job/${jobId}`, {method: 'DELETE'}, async (response) => { - if (!response.ok) { - const text = await response.text(); - if (response.status === 404) { - return false; - } - throw new Error(`Failed to cancel job: ${response.status} ${text}`); - } - const result = (await response.json()) as { - success: boolean; - }; - return result.success ?? true; - }); - } - - async retryDeadLetterJob(jobId: string): Promise { - return this.withResponse(`${this.baseUrl}/retry/${jobId}`, {method: 'POST'}, async (response) => { - if (!response.ok) { - const text = await response.text(); - if (response.status === 404) { - return false; - } - throw new Error(`Failed to retry job: ${response.status} ${text}`); - } - return true; - }); - } -} diff --git a/fluxer_api/pkgs/worker/src/providers/IQueueProvider.ts b/fluxer_api/pkgs/worker/src/providers/IQueueProvider.ts deleted file mode 100644 index 41f00f711..000000000 --- a/fluxer_api/pkgs/worker/src/providers/IQueueProvider.ts +++ /dev/null @@ -1,13 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import type {EnqueueOptions, LeasedQueueJob, WorkerJobPayload} from '@pkgs/worker/src/contracts/WorkerTypes'; - -export interface IQueueProvider { - enqueue(taskType: string, payload: WorkerJobPayload, options?: EnqueueOptions): Promise; - dequeue(taskTypes: Array, limit?: number): Promise>; - upsertCron(id: string, taskType: string, payload: WorkerJobPayload, cronExpression: string): Promise; - complete(receipt: string): Promise; - fail(receipt: string, error: string): Promise; - cancelJob(jobId: string): Promise; - retryDeadLetterJob(jobId: string): Promise; -} diff --git a/fluxer_api/pkgs/worker/src/providers/QueueProviderFactory.ts b/fluxer_api/pkgs/worker/src/providers/QueueProviderFactory.ts deleted file mode 100644 index 4c9b8ed6e..000000000 --- a/fluxer_api/pkgs/worker/src/providers/QueueProviderFactory.ts +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import type {TracingInterface} from '@pkgs/worker/src/contracts/WorkerTypes'; -import {HttpWorkerQueue} from '@pkgs/worker/src/providers/HttpWorkerQueue'; -import type {IQueueProvider} from '@pkgs/worker/src/providers/IQueueProvider'; - -export interface QueueProviderFactoryOptions { - queueProvider?: IQueueProvider | undefined; - queueBaseUrl?: string | undefined; - timeoutMs?: number | undefined; - tracing?: TracingInterface | undefined; -} - -export function createQueueProvider(options: QueueProviderFactoryOptions): IQueueProvider { - if (options.queueProvider) { - return options.queueProvider; - } - if (!options.queueBaseUrl) { - throw new Error('Queue provider requires either queueProvider or queueBaseUrl'); - } - return new HttpWorkerQueue({ - baseUrl: options.queueBaseUrl, - timeoutMs: options.timeoutMs, - tracing: options.tracing, - }); -} diff --git a/fluxer_api/pkgs/worker/src/runtime/WorkerFactory.ts b/fluxer_api/pkgs/worker/src/runtime/WorkerFactory.ts deleted file mode 100644 index 0970249cf..000000000 --- a/fluxer_api/pkgs/worker/src/runtime/WorkerFactory.ts +++ /dev/null @@ -1,175 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import type {LoggerInterface} from '@fluxer/logger/src/LoggerInterface'; -import {setWorkerDependencies} from '@pkgs/worker/src/context/WorkerContext'; -import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService'; -import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask'; -import type { - LeasedQueueJob, - TracingInterface, - WorkerConfig, - WorkerQueueConfig, - WorkerRuntimeConfig, -} from '@pkgs/worker/src/contracts/WorkerTypes'; -import type {IQueueProvider} from '@pkgs/worker/src/providers/IQueueProvider'; -import {WorkerRunner} from '@pkgs/worker/src/runtime/WorkerRunner'; -import {WorkerTaskRegistry} from '@pkgs/worker/src/runtime/WorkerTaskRegistry'; -import {WorkerService} from '@pkgs/worker/src/services/WorkerService'; - -export interface CreateWorkerOptions { - queue: WorkerQueueOptions; - runtime?: WorkerRuntimeConfig | undefined; - logger: LoggerInterface; - dependencies?: unknown; - taskRegistry?: WorkerTaskRegistry | undefined; - tracing?: TracingInterface | undefined; -} - -export interface CreateWorkerLegacyOptions { - config: WorkerConfig; - queueProvider?: IQueueProvider | undefined; - logger: LoggerInterface; - dependencies?: unknown; - taskRegistry?: WorkerTaskRegistry | undefined; - tracing?: TracingInterface | undefined; -} - -export interface WorkerQueueOptions { - queueProvider?: IQueueProvider | undefined; - queueBaseUrl?: string | undefined; - requestTimeoutMs?: number | undefined; -} - -interface ResolvedWorkerFactoryOptions { - queue: WorkerQueueOptions; - runtime: WorkerRuntimeConfig; - logger: LoggerInterface; - dependencies?: unknown; - taskRegistry?: WorkerTaskRegistry | undefined; - tracing?: TracingInterface | undefined; -} - -export interface WorkerResult { - start: () => Promise; - shutdown: () => Promise; - processTask: (job: LeasedQueueJob) => Promise; - getRunner: () => WorkerRunner; - getWorkerService: () => IWorkerService; - registerTask: >(name: string, handler: WorkerTaskHandler) => void; - registerTasks: (tasks: Record) => void; -} - -type WorkerFactoryOptions = CreateWorkerOptions | CreateWorkerLegacyOptions; - -function isLegacyCreateWorkerOptions(options: WorkerFactoryOptions): options is CreateWorkerLegacyOptions { - return 'config' in options; -} - -function resolveLegacyQueueOptions(config: WorkerQueueConfig, queueProvider?: IQueueProvider): WorkerQueueOptions { - return { - queueProvider, - queueBaseUrl: config.queueBaseUrl, - requestTimeoutMs: config.requestTimeoutMs, - }; -} - -function resolveWorkerFactoryOptions(options: WorkerFactoryOptions): ResolvedWorkerFactoryOptions { - if (isLegacyCreateWorkerOptions(options)) { - return { - queue: resolveLegacyQueueOptions(options.config, options.queueProvider), - runtime: { - workerId: options.config.workerId, - taskTypes: options.config.taskTypes, - concurrency: options.config.concurrency, - }, - logger: options.logger, - dependencies: options.dependencies, - taskRegistry: options.taskRegistry, - tracing: options.tracing, - }; - } - return { - queue: options.queue, - runtime: options.runtime ?? {}, - logger: options.logger, - dependencies: options.dependencies, - taskRegistry: options.taskRegistry, - tracing: options.tracing, - }; -} - -function assertTaskRegistryMutable(runner: WorkerRunner | null): void { - if (runner?.isRunning()) { - throw new Error('Cannot register tasks after worker start. Register tasks before starting the worker.'); - } -} - -export function createWorker(options: WorkerFactoryOptions): WorkerResult { - const resolvedOptions = resolveWorkerFactoryOptions(options); - const {queue, runtime, logger, dependencies, taskRegistry: providedRegistry, tracing} = resolvedOptions; - if (dependencies !== undefined) { - setWorkerDependencies(dependencies); - } - const taskRegistry = providedRegistry ?? new WorkerTaskRegistry(); - let runner: WorkerRunner | null = null; - let workerService: WorkerService | null = null; - function ensureRunner(): WorkerRunner { - if (!runner) { - runner = new WorkerRunner({ - tasks: taskRegistry.getTasks(), - queueBaseUrl: queue.queueBaseUrl, - queueProvider: queue.queueProvider, - logger, - workerId: runtime.workerId, - taskTypes: runtime.taskTypes, - concurrency: runtime.concurrency, - tracing, - requestTimeoutMs: queue.requestTimeoutMs, - }); - } - return runner; - } - function ensureWorkerService(): WorkerService { - if (!workerService) { - workerService = new WorkerService({ - queueBaseUrl: queue.queueBaseUrl, - queueProvider: queue.queueProvider, - logger, - tracing, - timeoutMs: queue.requestTimeoutMs, - }); - } - return workerService; - } - return { - async start() { - const r = ensureRunner(); - await r.start(); - }, - async shutdown() { - if (runner) { - await runner.stop(); - } - }, - async processTask(job: LeasedQueueJob) { - const r = ensureRunner(); - await r.processJob(job); - }, - getRunner() { - return ensureRunner(); - }, - getWorkerService() { - return ensureWorkerService(); - }, - registerTask>(name: string, handler: WorkerTaskHandler) { - assertTaskRegistryMutable(runner); - taskRegistry.register(name, handler); - runner = null; - }, - registerTasks(tasks: Record) { - assertTaskRegistryMutable(runner); - taskRegistry.registerAll(tasks); - runner = null; - }, - }; -} diff --git a/fluxer_api/pkgs/worker/src/runtime/WorkerRunner.ts b/fluxer_api/pkgs/worker/src/runtime/WorkerRunner.ts deleted file mode 100644 index 88dd10e9d..000000000 --- a/fluxer_api/pkgs/worker/src/runtime/WorkerRunner.ts +++ /dev/null @@ -1,187 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {randomUUID} from 'node:crypto'; -import type {LoggerInterface} from '@fluxer/logger/src/LoggerInterface'; -import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask'; -import type {LeasedQueueJob, TracingInterface} from '@pkgs/worker/src/contracts/WorkerTypes'; -import type {IQueueProvider} from '@pkgs/worker/src/providers/IQueueProvider'; -import {createQueueProvider} from '@pkgs/worker/src/providers/QueueProviderFactory'; -import {WorkerService} from '@pkgs/worker/src/services/WorkerService'; -import {ms} from 'itty-time'; - -export interface WorkerRunnerOptions { - tasks: Record; - queueBaseUrl?: string | undefined; - queueProvider?: IQueueProvider | undefined; - logger: LoggerInterface; - workerId?: string | undefined; - taskTypes?: Array | undefined; - concurrency?: number | undefined; - tracing?: TracingInterface | undefined; - requestTimeoutMs?: number | undefined; -} - -export class WorkerRunner { - private readonly tasks: Record; - private readonly workerId: string; - private readonly taskTypes: Array; - private readonly concurrency: number; - private readonly queue: IQueueProvider; - private readonly workerService: WorkerService; - private readonly logger: LoggerInterface; - private readonly tracing: TracingInterface | undefined; - private running = false; - private abortController: AbortController | null = null; - private workerLoopPromises: Array> = []; - - constructor(options: WorkerRunnerOptions) { - this.tasks = options.tasks; - this.workerId = options.workerId ?? `worker-${randomUUID()}`; - this.taskTypes = options.taskTypes ?? Object.keys(options.tasks); - this.concurrency = options.concurrency ?? 1; - this.logger = options.logger; - this.tracing = options.tracing; - this.queue = createQueueProvider({ - queueProvider: options.queueProvider, - queueBaseUrl: options.queueBaseUrl, - timeoutMs: options.requestTimeoutMs, - tracing: options.tracing, - }); - this.workerService = new WorkerService({ - queueProvider: this.queue, - logger: options.logger, - }); - } - - async start(): Promise { - if (this.running) { - this.logger.warn({workerId: this.workerId}, 'Worker already running'); - return; - } - this.running = true; - this.abortController = new AbortController(); - this.logger.info( - {workerId: this.workerId, taskTypes: this.taskTypes, concurrency: this.concurrency}, - 'Worker starting', - ); - this.workerLoopPromises = Array.from({length: this.concurrency}, (_, i) => - this.workerLoop(i, this.abortController!.signal), - ); - Promise.all(this.workerLoopPromises).catch((error) => { - this.logger.error({workerId: this.workerId, error}, 'Worker loop failed unexpectedly'); - }); - } - - async stop(): Promise { - if (!this.running) { - return; - } - this.running = false; - this.abortController?.abort(); - const stopTimeout = new Promise((resolve) => setTimeout(resolve, ms('2 seconds'))); - await Promise.race([Promise.all(this.workerLoopPromises), stopTimeout]); - this.workerLoopPromises = []; - this.logger.info({workerId: this.workerId}, 'Worker stopped'); - } - - async processJob(leasedJob: LeasedQueueJob): Promise { - await this.executeJob(leasedJob); - } - - getWorkerService(): WorkerService { - return this.workerService; - } - - getQueue(): IQueueProvider { - return this.queue; - } - - isRunning(): boolean { - return this.running; - } - - private async workerLoop(workerIndex: number, signal: AbortSignal): Promise { - this.logger.info({workerId: this.workerId, workerIndex}, 'Worker loop started'); - while (!signal.aborted) { - try { - const leasedJobs = await this.queue.dequeue(this.taskTypes, 1); - if (!leasedJobs || leasedJobs.length === 0) { - await this.sleep(100); - continue; - } - const leasedJob = leasedJobs[0]!; - const job = leasedJob.job; - this.logger.info( - { - workerId: this.workerId, - workerIndex, - jobId: job.id, - taskType: job.task_type, - attempts: job.attempts, - receipt: leasedJob.receipt, - }, - 'Processing job', - ); - await this.executeJob(leasedJob); - this.logger.info({workerId: this.workerId, workerIndex, jobId: job.id}, 'Job completed successfully'); - } catch (error) { - this.logger.error({workerId: this.workerId, workerIndex, error}, 'Worker loop error'); - await this.sleep(ms('1 second')); - } - } - this.logger.info({workerId: this.workerId, workerIndex}, 'Worker loop stopped'); - } - - private async executeJob(leasedJob: LeasedQueueJob): Promise { - const execute = async () => { - const task = this.tasks[leasedJob.job.task_type]; - if (!task) { - throw new Error(`Unknown task: ${leasedJob.job.task_type}`); - } - this.tracing?.addSpanEvent('job.execution.start'); - try { - await task(leasedJob.job.payload, { - logger: this.logger.child({jobId: leasedJob.job.id}), - jobId: 0n, - addJob: this.workerService.addJob.bind(this.workerService), - reportProgress: async () => {}, - shouldCancel: async () => false, - setContextLink: async () => {}, - }); - this.tracing?.addSpanEvent('job.execution.success'); - this.tracing?.setSpanAttributes({'job.status': 'success'}); - await this.queue.complete(leasedJob.receipt); - } catch (error) { - this.logger.error({jobId: leasedJob.job.id, error}, 'Job failed'); - this.tracing?.setSpanAttributes({ - 'job.status': 'failed', - 'job.error': error instanceof Error ? error.message : String(error), - }); - this.tracing?.addSpanEvent('job.execution.failed', { - error: error instanceof Error ? error.message : String(error), - }); - await this.queue.fail(leasedJob.receipt, String(error)); - } - }; - if (this.tracing) { - await this.tracing.withSpan( - { - name: 'worker.process_job', - attributes: { - 'worker.id': this.workerId, - 'job.id': leasedJob.job.id, - 'job.task_type': leasedJob.job.task_type, - 'job.attempts': leasedJob.job.attempts, - }, - }, - execute, - ); - } else { - await execute(); - } - } - - private async sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); - } -} diff --git a/fluxer_api/pkgs/worker/src/runtime/WorkerTaskRegistry.ts b/fluxer_api/pkgs/worker/src/runtime/WorkerTaskRegistry.ts deleted file mode 100644 index 55366e669..000000000 --- a/fluxer_api/pkgs/worker/src/runtime/WorkerTaskRegistry.ts +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask'; - -export class WorkerTaskRegistry { - private readonly tasks: Map = new Map(); - - register>(name: string, handler: WorkerTaskHandler): this { - this.tasks.set(name, handler as WorkerTaskHandler); - return this; - } - - registerAll(tasks: Record): this { - for (const [name, handler] of Object.entries(tasks)) { - this.tasks.set(name, handler); - } - return this; - } - - get(name: string): WorkerTaskHandler | undefined { - return this.tasks.get(name); - } - - has(name: string): boolean { - return this.tasks.has(name); - } - - getTaskNames(): Array { - return Array.from(this.tasks.keys()); - } - - getTasks(): Record { - return Object.fromEntries(this.tasks); - } - - get size(): number { - return this.tasks.size; - } -} - -export function createTaskRegistry(): WorkerTaskRegistry { - return new WorkerTaskRegistry(); -} diff --git a/fluxer_api/pkgs/worker/src/services/WorkerService.ts b/fluxer_api/pkgs/worker/src/services/WorkerService.ts deleted file mode 100644 index fd61c0446..000000000 --- a/fluxer_api/pkgs/worker/src/services/WorkerService.ts +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import type {LoggerInterface} from '@fluxer/logger/src/LoggerInterface'; -import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService'; -import type {TracingInterface, WorkerJobOptions, WorkerJobPayload} from '@pkgs/worker/src/contracts/WorkerTypes'; -import type {IQueueProvider} from '@pkgs/worker/src/providers/IQueueProvider'; -import {createQueueProvider} from '@pkgs/worker/src/providers/QueueProviderFactory'; - -export interface WorkerServiceOptions { - queueBaseUrl?: string | undefined; - queueProvider?: IQueueProvider | undefined; - logger: LoggerInterface; - tracing?: TracingInterface | undefined; - timeoutMs?: number | undefined; -} - -export class WorkerService implements IWorkerService { - private readonly queue: IQueueProvider; - private readonly logger: LoggerInterface; - - constructor(options: WorkerServiceOptions) { - this.queue = createQueueProvider({ - queueProvider: options.queueProvider, - queueBaseUrl: options.queueBaseUrl, - timeoutMs: options.timeoutMs, - tracing: options.tracing, - }); - this.logger = options.logger; - } - - async addJob( - taskType: string, - payload: TPayload, - options?: WorkerJobOptions, - ): Promise { - try { - await this.queue.enqueue(taskType, payload, { - runAt: options?.runAt, - maxAttempts: options?.maxAttempts, - priority: options?.priority, - }); - this.logger.debug({taskType, payload}, 'Job queued successfully'); - } catch (error) { - this.logger.error({error, taskType, payload}, 'Failed to queue job'); - throw error; - } - return 0n; - } - - async cancelJob(jobId: bigint): Promise { - try { - const cancelled = await this.queue.cancelJob(jobId.toString()); - if (cancelled) { - this.logger.info({jobId: jobId.toString()}, 'Job cancelled successfully'); - } else { - this.logger.debug({jobId: jobId.toString()}, 'Job not found (may have already been processed)'); - } - return cancelled; - } catch (error) { - this.logger.error({error, jobId: jobId.toString()}, 'Failed to cancel job'); - throw error; - } - } - - async retryDeadLetterJob(jobId: bigint): Promise { - try { - const retried = await this.queue.retryDeadLetterJob(jobId.toString()); - if (retried) { - this.logger.info({jobId: jobId.toString()}, 'Dead letter job retried successfully'); - } else { - this.logger.debug({jobId: jobId.toString()}, 'Job not found in dead letter queue'); - } - return retried; - } catch (error) { - this.logger.error({error, jobId: jobId.toString()}, 'Failed to retry dead letter job'); - throw error; - } - } - - getQueue(): IQueueProvider { - return this.queue; - } -} diff --git a/fluxer_api/src/api/Config.ts b/fluxer_api/src/api/Config.ts index 2d2af07aa..f9cf88602 100644 --- a/fluxer_api/src/api/Config.ts +++ b/fluxer_api/src/api/Config.ts @@ -39,28 +39,6 @@ function resolveEmailAppBaseUrl(master: MasterConfig): string { } } -function resolveGatewayInternalUrl(master: MasterConfig): string { - const configuredInternalGateway = ( - master.internal as { - gateway?: string; - } - ).gateway; - if (typeof configuredInternalGateway === 'string' && configuredInternalGateway.length > 0) { - return trimTrailingSlash(configuredInternalGateway); - } - try { - const gatewayUrl = new URL(master.endpoints.gateway); - if (gatewayUrl.protocol === 'ws:') { - gatewayUrl.protocol = 'http:'; - } else if (gatewayUrl.protocol === 'wss:') { - gatewayUrl.protocol = 'https:'; - } - return trimTrailingSlash(gatewayUrl.toString()); - } catch { - throw new Error(`Invalid gateway endpoint URL: ${master.endpoints.gateway}`); - } -} - function isBoolean(value: unknown): value is boolean { return typeof value === 'boolean'; } @@ -100,26 +78,23 @@ function normalizeIpBanExemptIps(values: Array): Array { return Array.from(normalized); } -function mapPushProviderApps( +function mapApnsApps( apps: | Array<{ app_id?: string; topic?: string; environment?: 'production' | 'development'; - project_id?: string; }> | undefined, - configName: string, ): APIConfig['push']['apns']['apps'] { return (apps ?? []).map((app) => { if (!app.app_id) { - throw new Error(`${configName} contains an entry with no app_id`); + throw new Error('FLUXER_PUSH_APNS_APPS contains an entry with no app_id'); } return { appId: app.app_id, topic: app.topic, environment: app.environment, - projectId: app.project_id, }; }); } @@ -194,34 +169,8 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig { backend: master.database.backend, }, kv: { - provider: 'redis' as const, url: master.internal.kv, - mode: (( - master.internal as { - kv_mode?: string; - } - ).kv_mode ?? 'standalone') as 'standalone' | 'cluster', - clusterNodes: - ( - master.internal as { - kv_cluster_nodes?: Array<{ - host: string; - port: number; - }>; - } - ).kv_cluster_nodes ?? [], - clusterNatMap: - ( - master.internal as { - kv_cluster_nat_map?: Record< - string, - { - host: string; - port: number; - } - >; - } - ).kv_cluster_nat_map ?? {}, + mode: master.internal.kv_mode, }, nats: { coreUrl: master.services.nats?.core_url ?? 'nats://127.0.0.1:4222', @@ -277,7 +226,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig { staticCdn: master.endpoints.static_cdn, }, internal: { - gateway: resolveGatewayInternalUrl(master), gatewayRpcAuthToken: master.services.gateway.rpc_auth_token ?? '', donationProxyKey, }, @@ -285,15 +233,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig { marketing: extractHostname(master.endpoints.marketing), unfurlIgnored: master.services.api.unfurl_ignored_hosts, }, - embeds: { - oEmbedHtmlEnabled: master.services.api.embeds.oembed_html_enabled, - oEmbedHtmlAllowUntrustedOnSelfHosted: master.services.api.embeds.oembed_html_allow_untrusted_on_self_hosted, - oEmbedHtmlAllowedHosts: master.services.api.embeds.oembed_html_allowed_hosts, - cacheDefaultTtlSeconds: master.services.api.embeds.cache_default_ttl_seconds, - cacheMaxTtlSeconds: master.services.api.embeds.cache_max_ttl_seconds, - cacheMinTtlSeconds: master.services.api.embeds.cache_min_ttl_seconds, - cacheRespectRemoteTtl: master.services.api.embeds.cache_respect_remote_ttl, - }, s3: { endpoint: s3Config.endpoint, presignedUrlBase: s3Config.presigned_url_base, @@ -329,14 +268,10 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig { breachedPasswordCheck: { enabled: master.integrations.breached_password_check.enabled ?? !master.instance.self_hosted, }, - contentModeration: { - nsfwThreshold: master.services.api.content_moderation?.nsfw_threshold ?? 0.7, - }, voice: { enabled: master.integrations.voice.enabled, apiKey: master.integrations.voice.api_key, apiSecret: master.integrations.voice.api_secret, - webhookUrl: master.integrations.voice.webhook_url, url: master.integrations.voice.url, internalUrl: master.integrations.voice.internal_url, defaultRegion: master.integrations.voice.default_region, @@ -409,7 +344,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig { reporterEmail: master.integrations.ncmec.reporter_email ?? '', }, admin: { - basePath: master.services.admin.base_path, oauthClientSecret: master.services.admin.oauth_client_secret, }, auth: { @@ -454,9 +388,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig { configured: master.instance.setup.configured, }, }, - domain: { - baseDomain: master.domain.base_domain, - }, discovery: { enabled: master.discovery.enabled, minMemberCount: master.discovery.min_member_count, @@ -481,18 +412,7 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig { keyId: master.integrations.push.apns.key_id, privateKey: master.integrations.push.apns.private_key, privateKeyPath: master.integrations.push.apns.private_key_path, - defaultEnvironment: master.integrations.push.apns.default_environment ?? 'production', - apps: mapPushProviderApps(master.integrations.push.apns.apps, 'FLUXER_PUSH_APNS_APPS'), - }, - fcm: { - enabled: master.integrations.push.fcm.enabled, - projectId: master.integrations.push.fcm.project_id, - clientEmail: master.integrations.push.fcm.client_email, - privateKey: master.integrations.push.fcm.private_key, - privateKeyPath: master.integrations.push.fcm.private_key_path, - serviceAccountJsonPath: master.integrations.push.fcm.service_account_json_path, - tokenUri: master.integrations.push.fcm.token_uri ?? 'https://oauth2.googleapis.com/token', - apps: mapPushProviderApps(master.integrations.push.fcm.apps, 'FLUXER_PUSH_FCM_APPS'), + apps: mapApnsApps(master.integrations.push.apns.apps), }, }, worker: { diff --git a/fluxer_api/src/api/config/APIConfig.ts b/fluxer_api/src/api/config/APIConfig.ts index 53fd1642b..c23b8688e 100644 --- a/fluxer_api/src/api/config/APIConfig.ts +++ b/fluxer_api/src/api/config/APIConfig.ts @@ -11,7 +11,6 @@ export interface PushProviderAppConfig { appId: string; topic?: string; environment?: PushProviderEnvironment; - projectId?: string; } export interface APICachePurgeConfig { @@ -69,20 +68,8 @@ export interface APIConfig { backend: 'cassandra' | 'postgres'; }; kv: { - provider: 'redis'; url: string; mode: 'standalone' | 'cluster'; - clusterNodes: Array<{ - host: string; - port: number; - }>; - clusterNatMap: Record< - string, - { - host: string; - port: number; - } - >; }; nats: { coreUrl: string; @@ -136,7 +123,6 @@ export interface APIConfig { gift: string; }; internal: { - gateway: string; gatewayRpcAuthToken: string; donationProxyKey: string; }; @@ -144,15 +130,6 @@ export interface APIConfig { marketing: string; unfurlIgnored: Array; }; - embeds: { - oEmbedHtmlEnabled: boolean; - oEmbedHtmlAllowUntrustedOnSelfHosted: boolean; - oEmbedHtmlAllowedHosts: Array; - cacheDefaultTtlSeconds: number; - cacheMaxTtlSeconds: number; - cacheMinTtlSeconds: number; - cacheRespectRemoteTtl: boolean; - }; s3: { endpoint: string; presignedUrlBase: string | undefined; @@ -191,14 +168,10 @@ export interface APIConfig { breachedPasswordCheck: { enabled: boolean; }; - contentModeration: { - nsfwThreshold: number; - }; voice: { enabled: boolean; apiKey?: string; apiSecret?: string; - webhookUrl?: string; url?: string; internalUrl?: string; defaultRegion?: { @@ -261,7 +234,6 @@ export interface APIConfig { failOpen: boolean; }; admin: { - basePath: string; oauthClientSecret?: string; }; auth: { @@ -306,9 +278,6 @@ export interface APIConfig { configured: boolean; }; }; - domain: { - baseDomain: string; - }; discovery: { enabled: boolean; minMemberCount: number; @@ -333,17 +302,6 @@ export interface APIConfig { keyId?: string; privateKey?: string; privateKeyPath?: string; - defaultEnvironment: PushProviderEnvironment; - apps: Array; - }; - fcm: { - enabled: boolean; - projectId?: string; - clientEmail?: string; - privateKey?: string; - privateKeyPath?: string; - serviceAccountJsonPath?: string; - tokenUri: string; apps: Array; }; }; diff --git a/fluxer_api/src/api/gif/NatsGifProvider.ts b/fluxer_api/src/api/gif/NatsGifProvider.ts index fabf89676..380fec41f 100644 --- a/fluxer_api/src/api/gif/NatsGifProvider.ts +++ b/fluxer_api/src/api/gif/NatsGifProvider.ts @@ -3,7 +3,7 @@ import {Config} from '@app/api/Config'; import type {GifProviderMeta, IGifProvider} from '@app/api/gif/IGifProvider'; import {Logger} from '@app/api/Logger'; -import {readOptionalIntegerEnv, requireIntegerInRange} from '@app/api/utils/IntegerOptions'; +import {readOptionalEnv, readOptionalIntegerEnv, requireIntegerInRange} from '@app/api/utils/IntegerOptions'; import {isJsonRecord, parseJsonUnknown} from '@app/api/utils/JsonBoundaryUtils'; import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError'; import {ServiceUnavailableError} from '@fluxer/errors/src/domains/core/ServiceUnavailableError'; @@ -19,7 +19,7 @@ import {NatsConnectionManager} from '@pkgs/nats/src/NatsConnectionManager'; const textEncoder = new TextEncoder(); const textDecoder = new TextDecoder(); -const GIF_SERVICE_SUBJECT = process.env.FLUXER_GIF_SERVICE_SUBJECT || 'svc.gifs'; +const GIF_SERVICE_SUBJECT = readOptionalEnv('FLUXER_GIF_SERVICE_SUBJECT') ?? 'svc.gifs'; const DEFAULT_GIF_SERVICE_TIMEOUT_MS = 12_000; const DEFAULT_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS = 3_000; const MAX_REQUEST_TIMEOUT_MS = 2_147_483_647; @@ -279,7 +279,7 @@ export function createNatsGifProvider(apiKeyResolver: GifApiKeyResolver): NatsGi const manager = new NatsConnectionManager({ url: Config.nats.coreUrl, token: Config.nats.authToken || undefined, - name: process.env.FLUXER_GIF_SERVICE_NATS_CLIENT_NAME || 'fluxer-api-gifs', + name: readOptionalEnv('FLUXER_GIF_SERVICE_NATS_CLIENT_NAME') ?? 'fluxer-api-gifs', }); const provider = new NatsGifProvider( manager, diff --git a/fluxer_api/src/api/infrastructure/UsersServiceClient.ts b/fluxer_api/src/api/infrastructure/UsersServiceClient.ts index 58729cc40..4f56126a5 100644 --- a/fluxer_api/src/api/infrastructure/UsersServiceClient.ts +++ b/fluxer_api/src/api/infrastructure/UsersServiceClient.ts @@ -5,7 +5,7 @@ import {Config} from '@app/api/Config'; import {throwForSvcErrorReply} from '@app/api/infrastructure/SvcErrorReply'; import {Logger} from '@app/api/Logger'; import {awaitAll} from '@app/api/utils/ConcurrencyUtils'; -import {readOptionalIntegerEnv, requireIntegerInRange} from '@app/api/utils/IntegerOptions'; +import {readOptionalEnv, readOptionalIntegerEnv, requireIntegerInRange} from '@app/api/utils/IntegerOptions'; import {isJsonRecord, parseJsonRecord, parseJsonWithGuard} from '@app/api/utils/JsonBoundaryUtils'; import type {UserPartialResponse} from '@fluxer/schema/src/domains/user/UserResponseSchemas'; import type {INatsConnectionManager} from '@pkgs/nats/src/INatsConnectionManager'; @@ -14,7 +14,7 @@ import {NatsConnectionManager} from '@pkgs/nats/src/NatsConnectionManager'; const textEncoder = new TextEncoder(); const textDecoder = new TextDecoder(); -const USERS_SERVICE_SUBJECT = process.env.FLUXER_USERS_SERVICE_SUBJECT || 'svc.users'; +const USERS_SERVICE_SUBJECT = readOptionalEnv('FLUXER_USERS_SERVICE_SUBJECT') ?? 'svc.users'; const DEFAULT_USERS_SERVICE_TIMEOUT_MS = 6000; const DEFAULT_USERS_SERVICE_INFLIGHT_MAX_ENTRIES = 10000; const MAX_REQUEST_TIMEOUT_MS = 2_147_483_647; @@ -196,7 +196,7 @@ export function createUsersServiceClient(): IUsersServiceClient { const manager = new NatsConnectionManager({ url: Config.nats.coreUrl, token: Config.nats.authToken || undefined, - name: process.env.FLUXER_USERS_SERVICE_NATS_CLIENT_NAME || 'fluxer-api-users', + name: readOptionalEnv('FLUXER_USERS_SERVICE_NATS_CLIENT_NAME') ?? 'fluxer-api-users', }); usersServiceClient = new NatsUsersServiceClient( manager, diff --git a/fluxer_api/src/api/middleware/IpBanMiddleware.ts b/fluxer_api/src/api/middleware/IpBanMiddleware.ts index d662d8db0..9804d0437 100644 --- a/fluxer_api/src/api/middleware/IpBanMiddleware.ts +++ b/fluxer_api/src/api/middleware/IpBanMiddleware.ts @@ -7,6 +7,7 @@ import {IP_BAN_REFRESH_CHANNEL} from '@app/api/constants/IpBan'; import {sharedListHas} from '@app/api/infrastructure/activity/SharedLists'; import {Logger} from '@app/api/Logger'; import type {HonoEnv} from '@app/api/types/HonoEnv'; +import {readOptionalEnv} from '@app/api/utils/IntegerOptions'; import {parseIpBanEntry, tryParseSingleIp} from '@app/api/utils/IpRangeUtils'; import {RefreshSubscription} from '@app/api/utils/RefreshSubscription'; import {getRequestClientIp} from '@app/api/utils/RequestClientIp'; @@ -70,7 +71,7 @@ class IpBanCache { channels: [IP_BAN_REFRESH_CHANNEL], refresh: () => this.refresh(), periodicIntervalMs: () => { - const intervalMs = Number(process.env.FLUXER_IP_BAN_REFRESH_INTERVAL_MS ?? '300000'); + const intervalMs = Number(readOptionalEnv('FLUXER_IP_BAN_REFRESH_INTERVAL_MS') ?? '300000'); return Number.isFinite(intervalMs) && intervalMs > 0 ? intervalMs : null; }, onRefreshError: (err, trigger) => { diff --git a/fluxer_api/src/api/middleware/ServiceRegistry.ts b/fluxer_api/src/api/middleware/ServiceRegistry.ts index 03fd1087c..3bf78aab5 100644 --- a/fluxer_api/src/api/middleware/ServiceRegistry.ts +++ b/fluxer_api/src/api/middleware/ServiceRegistry.ts @@ -22,7 +22,7 @@ import type {InstanceConfigRepository} from '@app/api/instance/InstanceConfigRep import {Logger} from '@app/api/Logger'; import {setInjectedSearchProvider} from '@app/api/SearchFactory'; import type {ISearchProvider} from '@app/api/search/ISearchProvider'; -import {readOptionalIntegerEnv} from '@app/api/utils/IntegerOptions'; +import {readOptionalEnv, readOptionalIntegerEnv} from '@app/api/utils/IntegerOptions'; import {VoiceAvailabilityService} from '@app/api/voice/VoiceAvailabilityService'; import {VoiceRepository} from '@app/api/voice/VoiceRepository'; import {VoiceServerLoadTracker} from '@app/api/voice/VoiceServerLoad'; @@ -37,11 +37,11 @@ export function createSnowflakeService(): SnowflakeService { const connectionManager = new NatsConnectionManager({ url: Config.nats.coreUrl, token: Config.nats.authToken || undefined, - name: process.env.FLUXER_SNOWFLAKE_SERVICE_NATS_CLIENT_NAME || 'fluxer-api-snowflakes', + name: readOptionalEnv('FLUXER_SNOWFLAKE_SERVICE_NATS_CLIENT_NAME') ?? 'fluxer-api-snowflakes', }); return new SnowflakeService({ connectionManager, - subject: process.env.FLUXER_SNOWFLAKE_SERVICE_SUBJECT || undefined, + subject: readOptionalEnv('FLUXER_SNOWFLAKE_SERVICE_SUBJECT'), batchSize: readOptionalIntegerEnv('FLUXER_SNOWFLAKE_SERVICE_BATCH_SIZE'), lowWatermark: readOptionalIntegerEnv('FLUXER_SNOWFLAKE_SERVICE_LOW_WATERMARK'), maxBufferAgeMs: readOptionalIntegerEnv('FLUXER_SNOWFLAKE_SERVICE_MAX_BUFFER_AGE_MS'), @@ -64,8 +64,6 @@ export function getKVClient(): IKVProvider { _kvClient = new KVClient({ url: Config.kv.url, mode: Config.kv.mode, - clusterNodes: Config.kv.clusterNodes, - clusterNatMap: Config.kv.clusterNatMap, }); } return _kvClient; diff --git a/fluxer_api/src/api/rpc/RpcTimings.ts b/fluxer_api/src/api/rpc/RpcTimings.ts index b364ad7e2..58c2b6015 100644 --- a/fluxer_api/src/api/rpc/RpcTimings.ts +++ b/fluxer_api/src/api/rpc/RpcTimings.ts @@ -1,6 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import {hostname} from 'node:os'; +import {readOptionalEnv} from '@app/api/utils/IntegerOptions'; import type {RpcSessionTimings, RpcTimingNode} from '@fluxer/schema/src/domains/rpc/RpcSchemas'; export type RpcTimingSteps = Record; @@ -20,18 +21,8 @@ export function createRpcTimingNode(startedAtNs: bigint, steps?: RpcTimingSteps) }; } -function firstRuntimeName(names: Array, fallback: string): string { - for (const name of names) { - const value = process.env[name]; - if (value && value.length > 0) { - return value; - } - } - return fallback; -} - function apiPodName(): string { - return firstRuntimeName(['POD_NAME', 'HOSTNAME'], hostname()); + return readOptionalEnv('POD_NAME') ?? readOptionalEnv('HOSTNAME') ?? hostname(); } export async function timeRpcStep(steps: RpcTimingSteps, name: string, operation: () => Promise): Promise { diff --git a/fluxer_api/src/api/test/Setup.ts b/fluxer_api/src/api/test/Setup.ts index 2dc8e5091..b2ed4d28d 100644 --- a/fluxer_api/src/api/test/Setup.ts +++ b/fluxer_api/src/api/test/Setup.ts @@ -53,8 +53,6 @@ function setDefaultTestEnv(): void { FLUXER_NATS_URL: natsUrl, FLUXER_NATS_CORE_URL: natsUrl, FLUXER_NATS_JETSTREAM_URL: natsUrl, - FLUXER_INTERNAL_API_ENDPOINT: 'http://127.0.0.1:8088/api', - FLUXER_INTERNAL_GATEWAY_ENDPOINT: 'http://127.0.0.1:8088/gateway', FLUXER_INTERNAL_MEDIA_PROXY_ENDPOINT: 'http://127.0.0.1:8088/media', FLUXER_S3_ENDPOINT: 'http://127.0.0.1:3900', FLUXER_S3_REGION: 'local', @@ -65,8 +63,6 @@ function setDefaultTestEnv(): void { FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64: 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=', FLUXER_ADMIN_SECRET_KEY_BASE: 'test-admin-secret', FLUXER_ADMIN_OAUTH_CLIENT_SECRET: 'test-admin-oauth-secret', - FLUXER_APP_PROXY_PORT: '8773', - FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: 'http://127.0.0.1:8088/media', FLUXER_GATEWAY_RPC_AUTH_TOKEN: 'test-gateway-rpc-token', FLUXER_SUDO_MODE_SECRET: 'test-sudo-secret', FLUXER_CONNECTION_INITIATION_SECRET: 'test-connection-secret', diff --git a/fluxer_api/src/api/utils/IntegerOptions.ts b/fluxer_api/src/api/utils/IntegerOptions.ts index 1f6b8a92c..589b744eb 100644 --- a/fluxer_api/src/api/utils/IntegerOptions.ts +++ b/fluxer_api/src/api/utils/IntegerOptions.ts @@ -1,6 +1,12 @@ +import {readEnvValue} from '@fluxer/config/src/config_loader/EnvironmentOverrides'; + +export function readOptionalEnv(name: string): string | undefined { + return readEnvValue(process.env, name); +} + export function readOptionalIntegerEnv(name: string): number | undefined { - const raw = process.env[name]?.trim(); - if (raw === undefined || raw === '') return undefined; + const raw = readOptionalEnv(name)?.trim(); + if (raw === undefined) return undefined; const value = Number(raw); if (!/^-?\d+$/.test(raw) || !Number.isSafeInteger(value)) { throw new Error(`${name} must be a safe integer`); diff --git a/fluxer_app_proxy/Cargo.toml b/fluxer_app_proxy/Cargo.toml index 9c995e906..fff339ddf 100644 --- a/fluxer_app_proxy/Cargo.toml +++ b/fluxer_app_proxy/Cargo.toml @@ -21,5 +21,5 @@ tokio-util = { version = "0.7.19", features = ["io"] } tower = { version = "0.5.3", features = ["util"] } tower-http = { version = "0.7.1", features = ["compression-gzip", "trace"] } tracing = "0.1.44" -tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } +tracing-subscriber = "0.3.23" diff --git a/fluxer_app_proxy/src/config.rs b/fluxer_app_proxy/src/config.rs index 44e796a04..ba33d45a6 100644 --- a/fluxer_app_proxy/src/config.rs +++ b/fluxer_app_proxy/src/config.rs @@ -434,27 +434,23 @@ fn read_csp_sources(name: &'static str) -> Vec { } fn read_csp_report_uri(name: &'static str) -> Option { - let value = cfg::non_empty_env(name)?; - CspReportUri::parse(name, &value) + let value = cfg::env_value(name)?; + CspReportUri::parse(name, value.trim()) .inspect_err(warn_invalid) .ok() } impl AppProxyConfig { pub fn from_env() -> Self { - let release_channel = ReleaseChannel::from_env_value(&cfg::read_env_preferred( - &["RELEASE_CHANNEL"], - "stable", - )); - let geoip_source = cfg::parse_geoip_source_config( - &cfg::read_first_env(&["FLUXER_GEOIP_DB_PATH", "MAXMIND_DB_PATH"], ""), - "app_proxy", - ); + let release_channel = + ReleaseChannel::from_env_value(&cfg::read_env("RELEASE_CHANNEL", "stable")); + let geoip_source = + cfg::parse_geoip_source_config(&cfg::read_env("FLUXER_GEOIP_DB_PATH", ""), "app_proxy"); let geoip_s3_config = cfg::read_geoip_s3_config_from_env(&geoip_source); let s3_public_endpoint = parse_optional_http_endpoint( "FLUXER_S3_PUBLIC_ENDPOINT", - cfg::non_empty_env("FLUXER_S3_PUBLIC_ENDPOINT"), + cfg::env_value("FLUXER_S3_PUBLIC_ENDPOINT"), ); let s3_uploads_bucket = cfg::read_env("FLUXER_S3_BUCKET_UPLOADS", "fluxer-uploads"); let s3_uploads_endpoint = s3_public_endpoint.as_ref().and_then(|endpoint| { @@ -474,11 +470,11 @@ impl AppProxyConfig { static_dir: cfg::read_env("FLUXER_STATIC_DIR", "./static"), index_upstream_url: parse_optional_http_url( "FLUXER_APP_PROXY_INDEX_UPSTREAM_URL", - cfg::non_empty_env("FLUXER_APP_PROXY_INDEX_UPSTREAM_URL"), + cfg::env_value("FLUXER_APP_PROXY_INDEX_UPSTREAM_URL"), ), static_cdn_endpoint: parse_optional_http_endpoint( "FLUXER_STATIC_CDN_ENDPOINT", - cfg::non_empty_env("FLUXER_STATIC_CDN_ENDPOINT"), + cfg::env_value("FLUXER_STATIC_CDN_ENDPOINT"), ), s3_public_endpoint, s3_uploads_endpoint, @@ -489,7 +485,7 @@ impl AppProxyConfig { 60_000u64, ), release_channel, - build_version: cfg::read_env_preferred( + build_version: cfg::read_first_env( &["BUILD_VERSION", "FLUXER_BUILD_VERSION"], env!("CARGO_PKG_VERSION"), ), @@ -498,21 +494,10 @@ impl AppProxyConfig { csp: CspConfig::from_env(), geoip_source, geoip_s3_config, - trust_client_ip_header: cfg::read_bool_env( - &["FLUXER_TRUST_CLIENT_IP_HEADER", "TRUST_CLIENT_IP_HEADER"], - false, - ), - client_ip_header_name: cfg::read_first_env( - &[ - "FLUXER_CLIENT_IP_HEADER_NAME", - "FLUXER_CLIENT_IP_HEADER", - "CLIENT_IP_HEADER_NAME", - "CLIENT_IP_HEADER", - ], - "x-forwarded-for", - ) - .trim() - .to_ascii_lowercase(), + trust_client_ip_header: cfg::read_bool_env("FLUXER_TRUST_CLIENT_IP_HEADER", false), + client_ip_header_name: cfg::read_env("FLUXER_CLIENT_IP_HEADER_NAME", "x-forwarded-for") + .trim() + .to_ascii_lowercase(), same_origin_hosts: parse_same_origin_hosts( "FLUXER_APP_PROXY_SAME_ORIGIN_HOSTS", &cfg::read_env("FLUXER_APP_PROXY_SAME_ORIGIN_HOSTS", ""), @@ -609,12 +594,11 @@ fn parse_same_origin_host( } fn resolve_discovery_upstream_url_from_env() -> String { - resolve_discovery_upstream_url(|name| env::var(name).ok()) + resolve_discovery_upstream_url(cfg::env_value) } fn resolve_bootstrap_api_public_endpoint_from_env() -> Option { - resolve_bootstrap_api_public_endpoint(|name| env::var(name).ok()) - .unwrap_or_else(|error| panic!("{error}")) + resolve_bootstrap_api_public_endpoint(cfg::env_value).unwrap_or_else(|error| panic!("{error}")) } fn resolve_bootstrap_api_public_endpoint(mut read_var: F) -> anyhow::Result> diff --git a/fluxer_app_proxy/src/main.rs b/fluxer_app_proxy/src/main.rs index adac7c7db..d66131df3 100644 --- a/fluxer_app_proxy/src/main.rs +++ b/fluxer_app_proxy/src/main.rs @@ -17,9 +17,7 @@ use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; fn main() -> anyhow::Result<()> { tracing_subscriber::registry() - .with( - tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()), - ) + .with(fluxer_common::config::env_filter("info")) .with(tracing_subscriber::fmt::layer()) .init(); diff --git a/fluxer_common/Cargo.toml b/fluxer_common/Cargo.toml index eacbcd6ae..62f34dc51 100644 --- a/fluxer_common/Cargo.toml +++ b/fluxer_common/Cargo.toml @@ -16,6 +16,7 @@ serde_json = "1.0.151" tempfile = "3.27.0" time = { version = "0.3.55", features = ["formatting", "macros", "parsing"] } tracing = "0.1.44" +tracing-subscriber = { version = "0.3.23", default-features = false, features = ["env-filter"] } base64 = "0.23" hex = "0.4.3" hmac = "0.13.0" diff --git a/fluxer_common/src/config.rs b/fluxer_common/src/config.rs index 441115f87..5c847d386 100644 --- a/fluxer_common/src/config.rs +++ b/fluxer_common/src/config.rs @@ -34,7 +34,7 @@ pub struct GeoipS3Config { } pub fn read_geoip_s3_config_from_env(source: &GeoipSourceConfig) -> Option { - read_geoip_s3_config(source, |name| env::var(name).ok()) + read_geoip_s3_config(source, env_value) } fn read_geoip_s3_config(source: &GeoipSourceConfig, mut read_var: F) -> Option @@ -123,26 +123,23 @@ fn percent_decode(value: &str) -> String { .unwrap_or_else(|_| value.to_owned()) } -pub fn read_env(name: &str, fallback: &str) -> String { - env::var(name).unwrap_or_else(|_| fallback.to_owned()) +pub fn env_value(name: &str) -> Option { + env::var(name).ok().filter(|value| !value.trim().is_empty()) } -pub fn read_env_preferred(names: &[&str], fallback: &str) -> String { - names - .iter() - .find_map(|name| env::var(name).ok().filter(|value| !value.trim().is_empty())) - .unwrap_or_else(|| fallback.to_owned()) +pub fn read_env(name: &str, fallback: &str) -> String { + env_value(name).unwrap_or_else(|| fallback.to_owned()) } pub fn read_first_env(names: &[&str], fallback: &str) -> String { names .iter() - .find_map(|name| env::var(name).ok()) + .find_map(|name| env_value(name)) .unwrap_or_else(|| fallback.to_owned()) } -pub fn read_bool_env(names: &[&str], fallback: bool) -> bool { - let Some(value) = names.iter().find_map(|name| env::var(name).ok()) else { +pub fn read_bool_env(name: &str, fallback: bool) -> bool { + let Some(value) = env_value(name) else { return fallback; }; matches!( @@ -151,11 +148,10 @@ pub fn read_bool_env(names: &[&str], fallback: bool) -> bool { ) } -pub fn non_empty_env(name: &str) -> Option { - env::var(name) - .ok() - .map(|v| v.trim().to_owned()) - .filter(|v| !v.is_empty()) +pub fn env_filter(default: &str) -> tracing_subscriber::EnvFilter { + env_value("RUST_LOG") + .and_then(|filter| tracing_subscriber::EnvFilter::try_new(filter).ok()) + .unwrap_or_else(|| tracing_subscriber::EnvFilter::new(default)) } pub fn normalize_base_path(value: &str) -> String { @@ -295,7 +291,7 @@ pub fn normalize_public_endpoint(url: &str, base_domain: &str, public_port: Opti } pub fn try_normalize_public_endpoint_from_env(url: &str) -> anyhow::Result { - let (base_domain, public_port) = resolve_public_domain_and_port(|name| env::var(name).ok())?; + let (base_domain, public_port) = resolve_public_domain_and_port(env_value)?; Ok(normalize_public_endpoint(url, &base_domain, public_port)) } @@ -767,6 +763,45 @@ mod tests { assert_eq!(None, port); } + #[test] + fn a_blank_value_reads_as_unset() { + unsafe { + env::set_var("FLUXER_COMMON_TEST_BLANK_EMPTY", ""); + env::set_var("FLUXER_COMMON_TEST_BLANK_SPACES", " \t"); + env::set_var("FLUXER_COMMON_TEST_BLANK_SET", "value"); + env::set_var("FLUXER_COMMON_TEST_BLANK_TRUE", "true"); + } + assert_eq!(None, env_value("FLUXER_COMMON_TEST_BLANK_EMPTY")); + assert_eq!(None, env_value("FLUXER_COMMON_TEST_BLANK_SPACES")); + assert_eq!(None, env_value("FLUXER_COMMON_TEST_BLANK_MISSING")); + assert_eq!( + "fallback", + read_env("FLUXER_COMMON_TEST_BLANK_EMPTY", "fallback") + ); + assert_eq!( + "fallback", + read_env("FLUXER_COMMON_TEST_BLANK_SPACES", "fallback") + ); + assert_eq!( + "value", + read_env("FLUXER_COMMON_TEST_BLANK_SET", "fallback") + ); + assert_eq!( + "value", + read_first_env( + &[ + "FLUXER_COMMON_TEST_BLANK_EMPTY", + "FLUXER_COMMON_TEST_BLANK_SPACES", + "FLUXER_COMMON_TEST_BLANK_SET", + ], + "fallback" + ) + ); + assert!(read_bool_env("FLUXER_COMMON_TEST_BLANK_EMPTY", true)); + assert!(read_bool_env("FLUXER_COMMON_TEST_BLANK_SPACES", true)); + assert!(read_bool_env("FLUXER_COMMON_TEST_BLANK_TRUE", false)); + } + #[test] fn trim_trailing_slash_works() { assert_eq!( diff --git a/fluxer_docs/src/content/docs/operator/configuration.mdx b/fluxer_docs/src/content/docs/operator/configuration.mdx index 5fd0c8c57..269f5e74a 100644 --- a/fluxer_docs/src/content/docs/operator/configuration.mdx +++ b/fluxer_docs/src/content/docs/operator/configuration.mdx @@ -16,7 +16,9 @@ The installer in [Get started](/operator/get-started/) writes `.env` for you and ## How configuration is loaded -Compose passes only variables listed in `docker-compose.yml`. For an unlisted setting, add it to the service's environment through a local Compose override. Adding it to `.env` alone has no effect. +Compose forwards every setting a self-hosted instance can change from `.env` to each service that reads it. A few names keep a fixed value or are not passed at all, and [Keys Compose does not forward](#keys-compose-does-not-forward) lists them with the reason. Setting one of those in `.env` has no effect, so add it through a local Compose override. + +Compose adds a default of its own only where the stack has to differ from the built-in default, and the entry below names it. Every other forwarded name reaches its service empty when `.env` leaves it out, and the service applies its built-in default. Single-quote values containing a literal `$` so Compose does not expand them as variable references: @@ -32,7 +34,7 @@ Precedence, highest first: 2. The environment variable. 3. The built-in default. -Use `true` or `false` for booleans, decimal integers for integer settings, and the specified object or array for JSON settings. Defaults and accepted values are listed below. Leave an unwanted override unset, since an empty value does not always restore the default. +Use `true` or `false` for booleans, decimal integers for integer settings, and the specified object or array for JSON settings. Defaults and accepted values are listed below. Every service built from this version of the stack files or later reads an empty or blank value the same as an unset one, so an empty value restores the default. Older images do not, which [Match the images to the stack files](/operator/upgrading/#match-the-images-to-the-stack-files) covers. ## Core identity and public address @@ -52,13 +54,9 @@ The API and worker require hostname-only domain settings: no scheme, port, crede `FLUXER_PUBLIC_ORIGIN` states the same public address as one string, and [The public origin](#the-public-origin) has it. -#### `FLUXER_INTERNAL_SCHEME` - -Default `http`. The scheme for internal service URLs. Must be `http` or `https`, and anything else fails startup. - ## Secrets you must generate -Every value below ships as `CHANGE_ME`. Replace them all before the first start. Compose refuses to start when one is unset or empty, and names the variable. Generate each with `openssl rand -hex 32`, except the relay secret, the attachment URL secrets, and the VAPID pair, whose own entries name the command. The installer in [Get started](/operator/get-started/) generates them all, so come here to rotate one value later or to set them up by hand. +Every value below ships as `CHANGE_ME`. Replace them all before the first start. Compose forwards each one from `.env`, and refuses to start when one is unset or empty, and names the variable. Generate each with `openssl rand -hex 32`, except the relay secret, the attachment URL secrets, and the VAPID pair, whose own entries name the command. The installer in [Get started](/operator/get-started/) generates them all, so come here to rotate one value later or to set them up by hand. #### `POSTGRES_PASSWORD` @@ -96,7 +94,7 @@ Generate with `openssl rand -base64 32`. [Upload relay](/media-proxy/upload-rela Generate each entry with `openssl rand -base64 32`. Signs the [attachment URLs](/media-proxy/overview/#signed-attachment-urls) Fluxer returns. A comma-separated list, where each entry must be standard base64 decoding to at least 32 bytes. Spaces around an entry and empty entries are ignored. The first entry signs and every entry verifies. Must match on `api`, `worker`, and `media-proxy`, and must differ from `FLUXER_MEDIA_PROXY_SECRET_KEY`. `api` and `worker` refuse to start when an entry is invalid, and with no entry they return unsigned URLs. `media-proxy` reads it only when `FLUXER_MEDIA_PROXY_MODE` is `mp` or `upload`, and then refuses to start with an invalid entry, or with no entry when `FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE` is `report` or `enforce`. -The Compose stack leaves it empty, so a self-hosted instance returns unsigned URLs until an operator sets one. Setting it alone changes nothing on the read side, because `FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE` stays `off` until it is set too. +Compose forwards it from `.env` and passes it empty when unset, so a self-hosted instance returns unsigned URLs until an operator sets one. Setting it alone changes nothing on the read side, because `FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE` stays `off` until it is set too. Rotate it in three steps, and apply each one with `docker compose up -d` before starting the next: @@ -142,7 +140,7 @@ The values below ship with a usable value. Both are required. ## Endpoint derivation from FLUXER_BASE_DOMAIN -Fluxer builds its public endpoints from the scheme, the base domain, and the port. The docs endpoint is a fixed value. A port of `443` under `https` or `wss`, or `80` under `http` or `ws`, is omitted. +Fluxer builds its public endpoints from the scheme, the base domain, and the port. A port of `443` under `https` or `wss`, or `80` under `http` or `ws`, is omitted. | Endpoint | Derived value | | --- | --- | @@ -152,7 +150,6 @@ Fluxer builds its public endpoints from the scheme, the base domain, and the por | media | scheme, base domain, optional port, `/media` | | static_cdn | scheme, base domain, optional port. With `FLUXER_STATIC_CDN_DOMAIN` set it becomes `https://` and that domain, with no port | | admin | scheme, base domain, optional port, `/admin` | -| docs | Always `https://fluxer.dev`. Not built from the domain settings | | marketing | scheme, base domain, optional port, `/marketing` | | invite | scheme, `FLUXER_INVITE_DOMAIN` or the base domain, optional port, `/invite` | | gift | scheme, `FLUXER_GIFT_DOMAIN` or the base domain, optional port, `/gift` | @@ -177,97 +174,93 @@ The API and worker validate endpoint URLs at startup. The Gateway endpoint requi #### `FLUXER_STATIC_CDN_DOMAIN` -Default empty. A separate host for static assets. When set, the static endpoint is forced to `https` with no port. +Default empty. A separate host for static assets. When set, the static endpoint of `api` and `worker` is forced to `https` with no port. Only `api` and `worker` read it, so set `FLUXER_STATIC_CDN_ENDPOINT` to the same origin as well for `admin`, `app-proxy`, `gateway` and `unfurl`. Compose forwards it from `.env`. #### `FLUXER_INVITE_DOMAIN` -Default empty. The host used in invite links. A hostname with no scheme and no path. +Default empty. The host used in invite links. A hostname with no scheme and no path. Compose forwards it from `.env`. #### `FLUXER_GIFT_DOMAIN` -Default empty. The host used in gift links. A hostname with no scheme and no path. +Default empty. The host used in gift links. A hostname with no scheme and no path. Compose forwards it from `.env`. #### `FLUXER_API_ENDPOINT` -Defaults to the derived endpoint. The public API base. The `admin` service reads the same name as its internal API target, which is why Compose sets it to `http://api:8080` for that container only. +Defaults to the derived endpoint. The public API base. Compose forwards it from `.env` to the services on the shared block, which leaves out `app-proxy`. The `admin` service reads the same name as its internal API target, so Compose sets it to `http://api:8080` for that container. #### `FLUXER_API_CLIENT_ENDPOINT` -Defaults to the derived endpoint. The API base handed to clients. Setting only one of the API endpoints splits what clients see from what the instance advertises. +Defaults to the derived endpoint. The API base handed to clients. Setting only one of the API endpoints splits what clients see from what the instance advertises. Compose forwards it from `.env`. #### `FLUXER_APP_ENDPOINT` -Defaults to the derived endpoint. The web app origin. Also one of the allowed CORS origins. Serving the client from another hostname requires setting this. +Defaults to the derived endpoint. The web app origin. Also one of the allowed CORS origins. Serving the client from another hostname requires setting this. Compose forwards it from `.env`, and builds it from the public origin when it is unset. #### `FLUXER_APP_ORIGIN_ALIASES` -Default empty. Further web app origins, comma separated. Each entry must be an HTTP or HTTPS origin, or the API refuses to start. The API treats each one like the `FLUXER_APP_ENDPOINT` origin. It allows it through CORS, reads invite links on its host, skips unfurling client routes on its host, and refuses webhook token calls from it. Set it when one instance serves the web app on more than one hostname. +Default empty. Further web app origins, comma separated. Each entry must be an HTTP or HTTPS origin, or the API refuses to start. The API treats each one like the `FLUXER_APP_ENDPOINT` origin. It allows it through CORS, reads invite links on its host, skips unfurling client routes on its host, and refuses webhook token calls from it. Set it when one instance serves the web app on more than one hostname. Compose forwards it from `.env`. #### `FLUXER_GATEWAY_ENDPOINT` -Defaults to the derived endpoint. The public Gateway WebSocket URL. Also the source of the internal Gateway URL when `FLUXER_INTERNAL_GATEWAY_ENDPOINT` is unset, with `ws` rewritten to `http`. +Defaults to the derived endpoint. The public Gateway WebSocket URL. Compose forwards it from `.env`. #### `FLUXER_MEDIA_ENDPOINT` -Defaults to the derived endpoint. The public Media Proxy URL. The `gifs` service accepts it as a fallback for `FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT`. +Defaults to the derived endpoint. The public Media Proxy URL. The `gifs` service accepts it as a fallback for `FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT`. Compose forwards it from `.env`, and builds it from the public origin and `/media` when it is unset. #### `FLUXER_STATIC_CDN_ENDPOINT` -Defaults to the derived endpoint. The static asset origin. Read by `api`, `worker`, `admin`, `app-proxy`, and `unfurl`. +Defaults to the derived endpoint. The static asset origin. Read by `api`, `worker`, `admin`, `app-proxy`, and `unfurl`. Compose forwards it from `.env`. When it is unset, `admin` and `unfurl-shard` get the public origin, and every other service derives it itself. #### `FLUXER_ADMIN_ENDPOINT` -Defaults to the derived endpoint. The admin origin. The API accepts exactly this value plus `/oauth2_callback` as the admin OAuth redirect URI. +Defaults to the derived endpoint. The admin origin. The API accepts exactly this value plus `/oauth2_callback` as the admin OAuth redirect URI. Compose forwards it from `.env`, and builds it from the public origin and `FLUXER_ADMIN_BASE_PATH` when it is unset. -The admin dashboard sets its cookie flags from the scheme of this value, which Compose builds from `FLUXER_PUBLIC_SCHEME`. Over HTTPS its CSRF cookie is named `__Host-csrf_token` and has `Secure`. Over HTTP it is named `csrf_token` without `Secure`. +The admin dashboard sets its cookie flags from the scheme of this value. Over HTTPS its CSRF cookie is named `__Host-csrf_token` and has `Secure`. Over HTTP it is named `csrf_token` without `Secure`. #### `FLUXER_DOCS_ENDPOINT` -Defaults to `https://fluxer.dev`. The docs origin. Never handed to a client. +Defaults to an origin built from `FLUXER_PUBLIC_SCHEME`, `FLUXER_BASE_DOMAIN` and `FLUXER_PUBLIC_PORT`. The docs origin. Read by the docs server only, which the stack does not run, so Compose does not forward it. #### `FLUXER_MARKETING_ENDPOINT` -Defaults to the derived endpoint. The marketing origin. The second allowed CORS origin. Its hostname is extracted and matched. +Defaults to the derived endpoint. The marketing origin. The second allowed CORS origin. Its hostname is extracted and matched. Compose forwards it from `.env`, and uses the public origin with no path when it is unset. #### `FLUXER_INVITE_ENDPOINT` -Defaults to the derived endpoint. The invite base. The API reads links on its hostname as invite links and does not unfurl them. +Defaults to the derived endpoint. The invite base. The API reads links on its hostname as invite links and does not unfurl them. Compose forwards it from `.env`. #### `FLUXER_GIFT_ENDPOINT` -Defaults to the derived endpoint. The gift base. The API does not unfurl links on its hostname. +Defaults to the derived endpoint. The gift base. The API does not unfurl links on its hostname. Compose forwards it from `.env`. Internal endpoints address one container from another and never appear in a browser. `FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT` is required by `gifs`. The rest are optional. #### `FLUXER_INTERNAL_API_ENDPOINT` -Default `http://127.0.0.1:8080`. The API address used by `worker` and `gateway`. Compose sets `http://api:8080`. - -#### `FLUXER_INTERNAL_GATEWAY_ENDPOINT` - -No default. The Gateway address the API calls. Falls back to the public Gateway URL with the scheme rewritten. Compose sets `http://gateway:8080`. +Default `http://127.0.0.1:8080` in the Gateway. The API address used by `gateway` and `push`, and the discovery fallback of `app-proxy`. `push` refuses to start without it. Compose sets `http://api:8080`. #### `FLUXER_INTERNAL_MEDIA_PROXY_ENDPOINT` -Default `http://127.0.0.1:8082`. The internal Media Proxy address for `api` and `worker`. `FLUXER_MEDIA_PROXY_ENDPOINT` is read as an alias when this name is unset. +Default `http://127.0.0.1:8082`. The internal Media Proxy address for `api` and `worker`. `FLUXER_MEDIA_PROXY_ENDPOINT` is read as an alias when this name is unset or empty. Compose sets `http://media-proxy:8080`. #### `FLUXER_MEDIA_PROXY_ENDPOINT` -No default. The internal Media Proxy address. `unfurl-shard` reads this name alone and exits without it. On `api` and `worker` it is an alias of `FLUXER_INTERNAL_MEDIA_PROXY_ENDPOINT`, read only when that name is unset. +No default. The internal Media Proxy address. `unfurl-shard` reads this name alone and exits without it. On `api` and `worker` it is an alias of `FLUXER_INTERNAL_MEDIA_PROXY_ENDPOINT`, read only when that name is unset or empty. Compose sets `http://media-proxy:8080` on `unfurl-shard` only. #### `FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT` -No default. The public Media Proxy URL used by `gifs`, `unfurl`, and `media-proxy`. `gifs` requires this or `FLUXER_MEDIA_ENDPOINT` to start. Set it on `media-proxy` so requests for its own assets use local storage instead of an HTTP round trip. For `api` and `worker`, use `FLUXER_MEDIA_ENDPOINT` instead. +No default. The public Media Proxy URL used by `gifs`, `unfurl`, and `media-proxy`. `gifs` requires this or `FLUXER_MEDIA_ENDPOINT` to start. Set it on `media-proxy` so requests for its own assets use local storage instead of an HTTP round trip. For `api` and `worker`, use `FLUXER_MEDIA_ENDPOINT` instead. Compose forwards it from `.env` to every app service. When it is unset, they get `FLUXER_MEDIA_ENDPOINT`, or the public media URL when that is unset too. #### `FLUXER_UNFURL_STATIC_CDN_ENDPOINT` -Falls back to `FLUXER_STATIC_CDN_ENDPOINT`. The static origin used by `unfurl`. Read only by `unfurl`. +Falls back to `FLUXER_STATIC_CDN_ENDPOINT`. The static origin used by `unfurl`. Read only by `unfurl`. Compose forwards it from `.env` to `unfurl-shard`. ## The edge The `edge` container is the only HTTP entry point. Neither layout below needs a change to the edge settings. -Bundled TLS is the default. `docker compose up -d` binds `80/tcp`, `443/tcp`, and `443/udp` and obtains its own certificate for `FLUXER_DOMAIN`. Point DNS at the host and set nothing else. The `443` publishes follow `FLUXER_PUBLIC_PORT`, so a non-default public port moves them with it. +Bundled TLS is the default. `docker compose up -d` binds `80/tcp`, `443/tcp`, and `443/udp` and obtains its own certificate for `FLUXER_DOMAIN`. Point DNS at the host and set nothing else. The `443` publishes stay on `443` whatever `FLUXER_PUBLIC_PORT` says, and `FLUXER_HTTPS_PORT` moves them. Put your own reverse proxy in front by adding `docker-compose.proxy.yml`, a second Compose file that overrides parts of the first. Load it with `-f` twice, or set `COMPOSE_FILE=docker-compose.yml:docker-compose.proxy.yml` in `.env` once. The edge then serves plain HTTP on one port, and the proxy in front terminates TLS. [Behind your own reverse proxy](/operator/reverse-proxy/) has the steps to enable the overlay, the requirements, and the per-proxy configuration. @@ -275,11 +268,15 @@ All are optional. #### `FLUXER_EDGE_SITE_ADDRESS` -Defaults to the address Compose builds from `FLUXER_PUBLIC_SCHEME`, `FLUXER_DOMAIN` and `FLUXER_PUBLIC_PORT`. What the edge listens on, and the default keeps the listener on the address the instance advertises. Honoured in the bundled layout only, because `docker-compose.proxy.yml` sets the literal `:8080` and `tunnel.compose.yml` the literal `:80`, and either discards any `.env` value. A site address that lists more than one hostname therefore needs the bundled layout. Write the scheme into any value you set here, because a bare hostname means automatic HTTPS on `443` whatever `FLUXER_PUBLIC_SCHEME` says. +Defaults to the address Compose builds from `FLUXER_PUBLIC_SCHEME` and `FLUXER_DOMAIN`, with no port, so the edge listens on `443` or `80` in the container and `FLUXER_HTTPS_PORT` or `FLUXER_HTTP_PORT` maps it on the host. What the edge listens on, and the default keeps the listener on the address the instance advertises. Honoured in the bundled layout only, because `docker-compose.proxy.yml` sets the literal `:8080` and `tunnel.compose.yml` the literal `:80`, and either discards any `.env` value. A site address that lists more than one hostname therefore needs the bundled layout. Write the scheme into any value you set here, because a bare hostname means automatic HTTPS on `443` whatever `FLUXER_PUBLIC_SCHEME` says. Compose forwards it from `.env` to `edge`. #### `FLUXER_EDGE_TRUSTED_PROXIES` -Default `private_ranges`. The peer addresses whose `X-Forwarded-For` header the edge trusts. The default is `192.168.0.0/16`, `172.16.0.0/12`, `10.0.0.0/8`, `127.0.0.1/8`, `fd00::/8` and `::1`, which covers every same-host proxy. Narrow it to the proxy's own address when the proxy reaches the instance from a public address, or when clients reach the proxy from a range the default already covers. +Default `private_ranges`. The peer addresses whose `X-Forwarded-For` header the edge trusts. The default is `192.168.0.0/16`, `172.16.0.0/12`, `10.0.0.0/8`, `127.0.0.1/8`, `fd00::/8` and `::1`, which covers every same-host proxy. Narrow it to the proxy's own address when the proxy reaches the instance from a public address, or when clients reach the proxy from a range the default already covers. Compose forwards it from `.env` to `edge`. + +#### `FLUXER_EDGE_ENCODE` + +Default `zstd gzip`. The response compression the edge offers, written as the arguments of the Caddy `encode` directive. `gzip` alone stops the edge offering zstd. Read by the edge container only, and it takes effect on `docker compose up -d edge`. Compose forwards it from `.env` to `edge`. #### `FLUXER_EDGE_BIND` @@ -287,11 +284,11 @@ Default `127.0.0.1:8080`. Where the plain-HTTP port binds. Read only under the o #### `FLUXER_HTTP_PORT` -Default `80`. The host side of the edge's HTTP publish, which serves the redirect to HTTPS and the ACME HTTP challenge under an `https` scheme and nothing at all under an `http` one. The container still listens on `80` inside. It takes an optional bind address in front of the port, so `127.0.0.1:80` keeps the publish off every public interface. Do not set it to the port `FLUXER_PUBLIC_PORT` already names, because that publishes one host port twice and the edge refuses to start. Not read under the overlay, which publishes `FLUXER_EDGE_BIND` instead. +Default `80`. The host side of the edge's HTTP publish, which serves the redirect to HTTPS and the ACME HTTP challenge under an `https` scheme and nothing at all under an `http` one. The container still listens on `80` inside. It takes an optional bind address in front of the port, so `127.0.0.1:80` keeps the publish off every public interface. Do not set it to the same host port as `FLUXER_HTTPS_PORT`, because that publishes one host port twice and the edge refuses to start. `docker-compose.proxy.yml` ignores it and publishes `FLUXER_EDGE_BIND` instead. `tunnel.compose.yml` reads it with a default of `127.0.0.1:80`. #### `FLUXER_HTTPS_PORT` -Defaults to `FLUXER_PUBLIC_PORT`. The host side of the edge's HTTPS publish, and the only thing it moves is that host side. Set it when the host already has something on the port the instance advertises. It moves the TCP and the UDP publish together, because HTTP/3 needs both on the same port. It takes the same optional bind address in front of the port. The overlay does not read it. +Default `443`. The host side of the edge's HTTPS publish, and the only thing it moves is that host side. Set it when the host already has something on the port the instance advertises. It moves the TCP and the UDP publish together, because HTTP/3 needs both on the same port. It takes the same optional bind address in front of the port. The overlay does not read it. #### `COMPOSE_FILE` @@ -307,29 +304,21 @@ All are optional. #### `FLUXER_TRUST_CLIENT_IP_HEADER` -Default `false`. Whether the client-IP header is trusted. Compose sets `true` for every service that merges the shared environment block, which `app-proxy` does not. The API has no peer-address fallback, so `false` on `api` makes every non-exempt request a 403. +Default `false`. Whether the client-IP header is trusted. Compose forwards it from `.env` with a default of `true`, because the edge sets the header on every upstream hop. It reaches every service that reads it, `app-proxy` included. The API has no peer-address fallback, so `false` on `api` makes every non-exempt request a 403. #### `FLUXER_CLIENT_IP_HEADER_NAME` -Default `x-forwarded-for`. Which header has the client IP. Read by `api`, `worker`, `admin`, `app-proxy`, and `gateway`. It must match what the edge sets. - -#### `FLUXER_CLIENT_IP_HEADER` - -No default. Alias for `FLUXER_CLIENT_IP_HEADER_NAME`. Accepted only by `admin` and `app-proxy`. - -#### `CLIENT_IP_HEADER_NAME` and `CLIENT_IP_HEADER` - -No default. Legacy unprefixed aliases. Accepted only by `admin` and `app-proxy`, last in the preference chain. +Default `x-forwarded-for`. Which header has the client IP. Read by `api`, `worker`, `admin`, `app-proxy`, and `gateway`. It must match what the edge sets. Compose forwards it from `.env`. #### `FLUXER_API_IP_BAN_EXEMPT_IPS` -Default empty. Addresses and CIDR ranges exempt from IP bans. Comma separated. A bare IPv4 address exempts that address, a bare IPv6 address exempts its /64, and a CIDR range such as `2001:db8:1200::/56` exempts every address in it. Every entry must parse as an IP or a CIDR range or the API fails at boot. +Default empty. Addresses and CIDR ranges exempt from IP bans. Comma separated. A bare IPv4 address exempts that address, a bare IPv6 address exempts its /64, and a CIDR range such as `2001:db8:1200::/56` exempts every address in it. Every entry must parse as an IP or a CIDR range or the API fails at boot. Compose forwards it from `.env`. The API returns 403 for any request whose client-IP header is missing, empty, or not a parsable address, and for every request while `FLUXER_TRUST_CLIENT_IP_HEADER` is `false`. The exceptions are `/_health`, `/webhooks/livekit`, `/test`, and the Bluesky client metadata and JWKS routes. The edge sets the header on every upstream hop, so a missing or unparsable header happens only in a layout that puts something other than the edge directly in front of `api`. A proxy in front of the edge that never sets the header passes the check, and every request then looks as though it came from the proxy. #### `FLUXER_API_DONATION_PROXY_KEY` -Default empty, which keys donation rate limits on the caller. Set it to let a trusted front end, such as the marketing site, send the donor address the limits apply to. The key must be at least 32 characters or the API fails at boot. The front end sends the same value in `X-Fluxer-Internal-Key` and the donor address in `X-Fluxer-Donor-Ip`. The API ignores the address header unless the key matches, so a browser cannot set it. Self-hosters leave this empty. +Default empty, which keys donation rate limits on the caller. Set it to let a trusted front end, such as the marketing site, send the donor address the limits apply to. The key must be at least 32 characters or the API fails at boot. The front end sends the same value in `X-Fluxer-Internal-Key` and the donor address in `X-Fluxer-Donor-Ip`. The API ignores the address header unless the key matches, so a browser cannot set it. Self-hosters leave this empty. Compose forwards it from `.env`. ## Images @@ -343,7 +332,17 @@ These pick which container images Compose pulls. All are optional. `FLUXER_REGISTRY_OWNER` is the owner segment of the image names and falls back to `fluxerapp`. `FLUXER_REGISTRY` is the registry images are pulled from and falls back to `ghcr.io/` followed by the owner. `FLUXER_IMAGE_TAG` is the tag on every Fluxer image and falls back to `v1`. -These affect only the Fluxer images. `docker-compose.yml` pins the `caddy`, `postgres`, `valkey`, `nats`, `meilisearch`, `seaweedfs`, and `livekit` images, and `.env` cannot change them. +These affect only the Fluxer images. Each bundled third-party service has an image name of its own, which replaces the whole reference, registry and tag included. Set one to pull from a mirror or to move a service to another tag. All are optional. + +- `FLUXER_CADDY_IMAGE`: default `caddy:2.11-alpine`, for `edge`. +- `FLUXER_POSTGRES_IMAGE`: default `postgres:16-alpine`, for `postgres`. +- `FLUXER_VALKEY_IMAGE`: default `valkey/valkey:9.1-alpine`, for `valkey`. +- `FLUXER_NATS_IMAGE`: default `nats:2.14-alpine`, for `nats`. +- `FLUXER_MEILISEARCH_IMAGE`: default `getmeili/meilisearch:v1.53`, for `meilisearch`. +- `FLUXER_SEAWEEDFS_IMAGE`: default `chrislusf/seaweedfs:4.47`, for `seaweedfs` and `seaweedfs-init`. +- `FLUXER_LIVEKIT_IMAGE`: default `livekit/livekit-server:v1.12.0`, for `livekit`. + +A new Postgres major does not read the data directory an older one wrote. Move `FLUXER_POSTGRES_IMAGE` to another major only through [Move to a new Postgres major version](/operator/upgrading/#move-to-a-new-postgres-major-version). An upgrade refuses a refresh that would change the major the stack runs. ## Database @@ -351,53 +350,69 @@ These affect only the Fluxer images. `docker-compose.yml` pins the `caddy`, `pos #### `FLUXER_DATABASE_BACKEND` -Default `postgres`. Which backend is used. Node accepts only `postgres` or `cassandra`. The internal services also accept `postgresql`, `pg`, `scylla`, and `scylladb`. Every service rejects any other value at startup. +Default `postgres`. Which backend is used. Node accepts only `postgres` or `cassandra`. The internal services also accept `postgresql`, `pg`, `scylla`, and `scylladb`. Every service rejects any other value at startup. Compose sets `postgres`. #### `FLUXER_POSTGRES_URL` -Default empty. A full connection URL. When set, the discrete host, database, user, and password production checks are skipped. +Default empty. A full connection URL. When set, the discrete host, database, user, and password production checks are skipped. Compose forwards it from `.env`. #### `FLUXER_POSTGRES_HOST` -Default `127.0.0.1`. The database host. In production with `postgres` and no URL it must not be `127.0.0.1` or `localhost`. +Default `127.0.0.1`. The database host. In production with `postgres` and no URL it must not be `127.0.0.1` or `localhost`. Compose forwards it from `.env` with a default of `postgres`, the bundled service. #### `FLUXER_POSTGRES_PORT` -Default `5432`. The database port. Integer 1 to 65535. +Default `5432`. The database port. Integer 1 to 65535. Compose forwards it from `.env`. #### `FLUXER_POSTGRES_DATABASE` -Default `fluxer`. The database name. Must be non-empty. +Default `fluxer`. The database name. Must be non-empty. Compose forwards it from `.env`, and passes the same value to the bundled `postgres` as `POSTGRES_DB`. The Postgres image creates that database only on its first start with an empty data directory, so set it before the first start. The installer's backup takes it from the `postgres` container's environment, as `POSTGRES_DB`, and skips the dump when `FLUXER_POSTGRES_HOST` or `FLUXER_POSTGRES_URL` points outside the stack. #### `FLUXER_POSTGRES_USERNAME` -Default `fluxer`. The role. Must be non-empty. +Default `fluxer`. The role. Must be non-empty. Compose forwards it from `.env`, and passes the same value to the bundled `postgres` as `POSTGRES_USER`, under the same first-start rule. The installer's backup takes it from the `postgres` container's environment, as `POSTGRES_USER`, and skips the dump when `FLUXER_POSTGRES_HOST` or `FLUXER_POSTGRES_URL` points outside the stack. #### `FLUXER_POSTGRES_PASSWORD` -Default `fluxer`. The password. The literal `fluxer` is rejected in production. `CHANGE_ME` is not. +Default `fluxer`. The password. The literal `fluxer` is rejected in production. `CHANGE_ME` is not. Compose fills it from `POSTGRES_PASSWORD`. #### `FLUXER_POSTGRES_SSL` -Default `false`. TLS to the database. Must be `true` or `false`. In production it must be `true` unless `FLUXER_SELF_HOSTED=true`. +Default `false`. TLS to the database. Must be `true` or `false`. In production it must be `true` unless `FLUXER_SELF_HOSTED=true`. Compose forwards it from `.env`. #### `FLUXER_POSTGRES_SSL_CA` -Default empty. A CA certificate in PEM form. Used only when SSL is on. +Default empty. A CA certificate in PEM form. Used only when SSL is on. Compose forwards it from `.env`. #### `FLUXER_POSTGRES_MAX_CONNECTIONS` -Default `20`. Pool size. Integer 1 to 1000, per process. The total is the sum across every container. Compose sets 25 for `api` and `worker` and 20 for the database-backed shards. +Default `20`. Pool size. Integer 1 to 1000, per process. The total is the sum across every container. Only `api`, `worker`, `users-shard` and `messages-shard` open a pool, and Compose sets it in each from the per-service name below. + +#### `FLUXER_API_POSTGRES_MAX_CONNECTIONS` + +Default `25`. The pool size of `api`. Read by Compose, which passes it to `api` as `FLUXER_POSTGRES_MAX_CONNECTIONS`. + +#### `FLUXER_WORKER_POSTGRES_MAX_CONNECTIONS` + +Default `25`. The pool size of `worker`, passed the same way. + +#### `FLUXER_USERS_SHARD_POSTGRES_MAX_CONNECTIONS` + +Default `20`. The pool size of `users-shard`, passed the same way. + +#### `FLUXER_MESSAGES_SHARD_POSTGRES_MAX_CONNECTIONS` + +Default `20`. The pool size of `messages-shard`, passed the same way. #### `FLUXER_POSTGRES_KV_TABLE` -Default `fluxer_kv`. The key-value table name. Must match a safe Postgres identifier or startup fails. +Default `fluxer_kv`. The key-value table name. Must match a safe Postgres identifier or startup fails. Compose forwards it from `.env`. #### `FLUXER_POSTGRES_PREPARED_STATEMENTS` -Default `true`. Named prepared statements. Must be `true` or `false`. Compose passes it in the shared block, so one value governs `api`, `worker` and the Rust services at once. Set it to `false` behind a transaction-pooling pooler such as PgBouncer, where a named statement outlives the session that declared it. +Default `true`. Named prepared statements. Must be `true` or `false`. Compose forwards it in the shared block, so one value governs `api`, `worker` and the Rust services at once. Set it to `false` behind a transaction-pooling pooler such as PgBouncer, where a named statement outlives the session that declared it. -Cassandra or Scylla is an alternative backend, selected with `FLUXER_DATABASE_BACKEND=cassandra`. Supply a reachable database and configure the settings below when selecting it. The bundled stack uses Postgres and does not include a Cassandra or Scylla service. +Cassandra or Scylla is an alternative backend, selected with `FLUXER_DATABASE_BACKEND=cassandra`. Supply a reachable database and configure the settings below when selecting it. The bundled stack uses Postgres, does not include a Cassandra or Scylla service, and forwards none of the names below. #### `FLUXER_CASSANDRA_HOSTS` @@ -429,35 +444,27 @@ The bundled stack requires Valkey for shared cache, live updates and deletion qu #### `FLUXER_KV_URL` -Default `redis://localhost:6379/0`. The Redis-compatible service address. The `admin` service defaults to empty instead. Compose sets `redis://valkey:6379/0`. +Default `redis://localhost:6379/0`. The Redis-compatible service address. Read by `api` and `worker`. Compose forwards it from `.env` with a default of `redis://valkey:6379/0`, the bundled service. #### `FLUXER_KV_MODE` -Default `standalone`. Use `standalone` for a single Valkey server or `cluster` for a Redis-compatible cluster. - -#### `FLUXER_KV_PROVIDER` - -Default `redis`, which is the only accepted value. +Default `standalone`. Use `standalone` for a single Valkey server or `cluster` for a Redis-compatible cluster. Compose forwards it from `.env`. #### `FLUXER_SVC_CACHE_TTL_MS` -Default `30000`. Soft cache lifetime in the internal services. Milliseconds. The `users` and `messages` routers ignore this setting because they keep no configurable response cache. Their shard caches still honour it. - -#### `FLUXER_SVC_CACHE_HARD_TTL_MS` - -Default `600000`. Hard cache lifetime. Clamped to at least the soft TTL. +Default `30000`. Cache lifetime in the internal services, in milliseconds. The `users` and `messages` routers ignore this setting because they keep no configurable response cache. Their shard caches still honour it. Compose forwards it from `.env`. #### `FLUXER_SVC_CACHE_MAX_ENTRIES` -Default `100000`. Cache entry ceiling. Per process. The `users` and `messages` routers ignore this setting because they keep no configurable response cache. Their shard caches still honour it. +Default `100000`. Cache entry ceiling. Per process. The `users` and `messages` routers ignore this setting because they keep no configurable response cache. Their shard caches still honour it. Compose forwards it from `.env`. #### `FLUXER_GIFS_SHARD_CACHE_MAX_BYTES` -Default `536870912`. GIF cache size. Must be at least 16777216. Set it to 134217728 to keep the cache inside the `256mb` ceiling `FLUXER_GIFS_SHARD_MEMORY_LIMIT` gives `gifs-shard`. +Default `536870912`. GIF cache size. Must be at least 16777216. Set it to 134217728 to keep the cache inside the `256mb` ceiling `FLUXER_GIFS_SHARD_MEMORY_LIMIT` gives `gifs-shard`. Compose forwards it from `.env` to `gifs-shard`. #### `FLUXER_IP_BAN_REFRESH_INTERVAL_MS` -Default `300000`. How often the API refreshes its IP-ban cache. A non-finite value or one at or below zero disables the timer. +Default `300000`. How often the API refreshes its IP-ban cache. A non-finite value or one at or below zero, such as `0`, disables the timer. Compose forwards it from `.env`. Keep persistent storage and the bundled `noeviction` policy to protect deletion queues and other shared state. Back up `valkey-data`. See [Volumes and buckets](#volumes-and-buckets) for recovery implications. @@ -467,71 +474,69 @@ Every uploaded file lands in an S3-compatible object store, which the stack prov #### `FLUXER_S3_ENDPOINT` -Default `http://localhost:3900`. The S3 API address. `media-proxy` defaults to empty instead. +Default `http://localhost:3900`. The S3 API address. `media-proxy` defaults to empty instead. Compose forwards it from `.env` with a default of `http://seaweedfs:8333`, the bundled store. #### `FLUXER_S3_PUBLIC_ENDPOINT` -No default. The host substituted into presigned URLs, which are the only place it appears. +No default. The host substituted into presigned URLs, which are the only place it appears. Compose forwards it from `.env`, and uses `FLUXER_S3_ENDPOINT` when it is unset, except on `app-proxy`, which gets only a value `.env` sets and adds it to its Content Security Policy. #### `FLUXER_S3_FORCE_PATH_STYLE` -Default `false`. Path-style addressing. `media-proxy` defaults to `true` instead. Compose sets `true`. +Default `false`. Path-style addressing. `media-proxy` defaults to `true` instead. Compose forwards it from `.env` with a default of `true`, which the bundled store needs. #### `FLUXER_S3_REGION` -Default `local`. The region string. `media-proxy` defaults to `us-east-1`. Compose sets `us-east-1`. +Default `local`. The region string. `media-proxy` defaults to `us-east-1`. Compose forwards it from `.env` with a default of `us-east-1`, so every service signs for the same region. #### `FLUXER_S3_ACCESS_KEY_ID` -Default empty. The access key. Config validation requires it non-empty on `api` and `worker`. Optional for `media-proxy`. +Default empty. The access key. Config validation requires it non-empty on `api` and `worker`. Optional for `media-proxy`. `app-proxy` reads it only to fetch an `s3://` GeoIP database. Compose fills it from `FLUXER_S3_ACCESS_KEY`. #### `FLUXER_S3_SECRET_ACCESS_KEY` -Default empty. The secret key. Config validation requires it non-empty on `api` and `worker`. Optional for `media-proxy`. +Default empty. The secret key. Config validation requires it non-empty on `api` and `worker`. Optional for `media-proxy`. Compose fills it from `FLUXER_S3_SECRET_KEY`. #### `FLUXER_S3_SESSION_TOKEN` -Default empty. A temporary session token. Read by `media-proxy` only. +Default empty. A temporary session token. Read by `media-proxy` only. Compose forwards it from `.env`. #### `FLUXER_S3_BUCKET_CDN` -Default `fluxer`. Processed assets. `media-proxy` defaults to `cdn`. +Default `fluxer`. Processed assets. `media-proxy` defaults to `cdn`. Compose forwards it from `.env` with a default of `fluxer`, so every service and `seaweedfs-init` name the same bucket. #### `FLUXER_S3_BUCKET_UPLOADS` -Default `fluxer-uploads`. Raw uploads. `media-proxy` defaults to `uploads`, `app-proxy` to `fluxer-uploads`. +Default `fluxer-uploads`. Raw uploads. `media-proxy` defaults to `uploads`, `app-proxy` to `fluxer-uploads`. Compose forwards it from `.env` with a default of `fluxer-uploads` for the same reason. #### `FLUXER_S3_BUCKET_REPORTS` -Default `fluxer-reports`. Abuse report evidence. Read by `api` and `worker`. +Default `fluxer-reports`. Abuse report evidence. Read by `api` and `worker`. Compose forwards it from `.env` with a default of `fluxer-reports`, so every service and `seaweedfs-init` name the same bucket. #### `FLUXER_S3_BUCKET_HARVESTS` -Default `fluxer-harvests`. Data archives. Read by `api` and `worker`. +Default `fluxer-harvests`. Data archives. Read by `api` and `worker`. Compose forwards it from `.env` with a default of `fluxer-harvests`, so every service and `seaweedfs-init` name the same bucket. #### `FLUXER_S3_BUCKET_STATIC` -Default `static`. Static assets. Read by `media-proxy` only in `static` mode, which the stack does not use. `api` and `worker` never read it, and `seaweedfs-init` does not create this bucket. +Default `static`. Static assets. Read by `media-proxy` only in `static` mode, which the stack does not use. `api` and `worker` never read it, and `seaweedfs-init` does not create this bucket. Compose does not forward it. The Media Proxy read path has overrides of its own. All are optional. #### `FLUXER_S3_READ_ENDPOINT` -Falls back to `FLUXER_S3_ENDPOINT`. The read-side S3 address. Must be http or https with a host, and have no credentials, query string, or fragment. +Falls back to `FLUXER_S3_ENDPOINT`. The read-side S3 address. Must be http or https with a host, and have no credentials, query string, or fragment. Compose forwards it from `.env` to `media-proxy`. #### `FLUXER_S3_READ_BUCKET` -Falls back to `FLUXER_S3_BUCKET_CDN`. The read-side bucket. Read by `media-proxy` only. +Falls back to `FLUXER_S3_BUCKET_CDN`. The read-side bucket. Read by `media-proxy` only. Compose forwards it from `.env`. #### `FLUXER_S3_READ_BUCKET_STYLE` -Defaults to `path` when path style is on, else `virtual`. How the bucket appears in the URL. `path`, `virtual`, or `root`. Anything else is a startup error. +Defaults to `path` when path style is on, else `virtual`. How the bucket appears in the URL. `path`, `virtual`, or `root`. Anything else is a startup error. Compose forwards it from `.env` to `media-proxy`. #### `FLUXER_S3_READ_SIGNED` -Default `false`. Whether read requests are signed. Read by `media-proxy` only. Compose sets `true`, because `seaweedfs-init` installs an S3 identity and the store then refuses anonymous reads. - -Compose sets `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`, and `AWS_EC2_METADATA_DISABLED` for the AWS SDKs. No Fluxer code reads them directly. +Default `false`. Whether read requests are signed. Read by `media-proxy` only. Compose forwards it from `.env` with a default of `true`, because `seaweedfs-init` installs an S3 identity and the store then refuses anonymous reads. Set `false` only for a public-read bucket. ## Search @@ -539,43 +544,43 @@ The bundled stack provides Meilisearch for message search. These settings select #### `FLUXER_SEARCH_ENGINE` -Default `elasticsearch`. Which engine is used. `elasticsearch` or `meilisearch`. Compose sets `meilisearch`. +Default `elasticsearch`. Which engine is used. `elasticsearch` or `meilisearch`. Compose forwards it from `.env` with a default of `meilisearch`, the bundled engine. #### `FLUXER_SEARCH_URL` -Default `http://127.0.0.1:9200`. The engine address. Compose sets `http://meilisearch:7700`. +Default `http://127.0.0.1:9200`. The engine address. Compose forwards it from `.env` with a default of `http://meilisearch:7700`. #### `FLUXER_SEARCH_API_KEY` -Default empty. The API key. Meilisearch uses it as the key. Elasticsearch uses it as an API key that takes precedence over the username and password. +Default empty. The API key. Meilisearch uses it as the key. Elasticsearch uses it as an API key that takes precedence over the username and password. Compose fills it from `MEILI_MASTER_KEY`. #### `FLUXER_SEARCH_USERNAME` -Default empty. Basic auth user. Elasticsearch only. Ignored under Meilisearch. +Default empty. Basic auth user. Elasticsearch only. Ignored under Meilisearch. Compose forwards it from `.env`. #### `FLUXER_SEARCH_PASSWORD` -Default empty. Basic auth password. Elasticsearch only. +Default empty. Basic auth password. Elasticsearch only. Compose forwards it from `.env`. #### `FLUXER_SEARCH_TLS_REJECT_UNAUTHORIZED` -Default `true`. Certificate verification. Elasticsearch only. Never passed to the Meilisearch client. +Default `true`. Certificate verification. Elasticsearch only. Never passed to the Meilisearch client. Compose forwards it from `.env`. ## Message bus and internal services -The bundled stack requires NATS for communication between services and JetStream for background jobs. Compose supplies the connection settings. Change them when using an external NATS service, which must have JetStream enabled for the worker connection. +The bundled stack requires NATS for communication between services and JetStream for background jobs. Compose forwards the connection settings from `.env`, with the bundled service as the default. Change them when using an external NATS service, which must have JetStream enabled for the worker connection. #### `FLUXER_NATS_URL` -Default `nats://127.0.0.1:4222`. The core NATS address for `api`, `worker`, and `gateway`. The Gateway defaults to `nats://nats:4222` instead. +Default `nats://127.0.0.1:4222`. The core NATS address for `api`, `worker`, and `gateway`. The Gateway defaults to `nats://nats:4222` instead. Compose forwards it from `.env` with a default of `nats://nats:4222`. #### `FLUXER_NATS_CORE_URL` -Alias of `FLUXER_NATS_URL` on `api` and `worker`, read only when that name is unset. The core NATS address. Not read by the Gateway. +Alias of `FLUXER_NATS_URL` on `api` and `worker`, read only when that name is unset or empty. The core NATS address. Not read by the Gateway. Compose does not forward it, because it forwards `FLUXER_NATS_URL`. #### `FLUXER_NATS_JETSTREAM_URL` -Default `nats://127.0.0.1:4222`. The JetStream address for `api` and `worker`. Compose uses `FLUXER_NATS_URL` unless this setting is supplied separately. +Default `nats://127.0.0.1:4222`. The JetStream address for `api` and `worker`. Compose forwards it from `.env`, and uses `FLUXER_NATS_URL` when it is unset. #### `FLUXER_NATS_AUTH_TOKEN` @@ -583,54 +588,51 @@ Default empty. NATS authentication. Read by `api`, `worker`, `gateway`, and the #### `FLUXER_SVC_NATS_URL` -Default `nats://127.0.0.1:4222`. The NATS address for the internal services. A separate variable from `FLUXER_NATS_URL`. +Default `nats://127.0.0.1:4222`. The NATS address for the internal services and `push`. A separate variable from `FLUXER_NATS_URL`. Compose forwards it from `.env`, and uses `FLUXER_NATS_URL` when it is unset. #### `FLUXER_GATEWAY_API_RPC_ENDPOINT` -No default. Where the Gateway calls the API. Read by the Gateway. +No default. Where the Gateway calls the API. Read by the Gateway, which derives it from `FLUXER_INTERNAL_API_ENDPOINT` when it is unset. Compose does not forward it. Compose configures the internal services. Change the following settings only when customising their deployment. #### `FLUXER_SVC_NAME` -Default `default`. The metrics prefix. It also selects the built-in default of `FLUXER_SVC_MAX_CONCURRENT_REQUESTS`. Changing it does not rename NATS subjects. +Default `default`. The metrics prefix. It also selects the built-in default of `FLUXER_SVC_MAX_CONCURRENT_REQUESTS`. Changing it does not rename NATS subjects. Compose sets it per service. #### `FLUXER_SVC_MODE` -Default `router`. Must be `router` or `shard`. Any other value prevents startup. +Default `router`. Must be `router` or `shard`. Any other value prevents startup. Compose sets it per service. #### `FLUXER_SVC_SHARD_COUNT` -Default `1`. How many shards exist. Fixed at 1 in the shipped stack. +Default `1`. How many shards exist. Compose fixes it at `1`. #### `FLUXER_SVC_SHARD_ID` -Derived from the numeric suffix of `POD_NAME`, else `0`. Which shard this process is. A shard ID at or above the shard count is a startup error. +Derived from the numeric suffix of `POD_NAME`, else `0`. Which shard this process is. A shard ID at or above the shard count is a startup error. Compose sets `0` on every shard. #### `FLUXER_SVC_LISTEN_HOST` -Default `0.0.0.0`. The bind address. Serves health and metrics only. +Default `0.0.0.0`. The bind address. Serves health and metrics only. Compose does not forward it. #### `FLUXER_SVC_PORT` -Default `8090`. The health and metrics port. Not published. +Default `8090`. The health and metrics port. Not published. Compose does not forward it, because every internal service health check probes `8090`. #### `FLUXER_SVC_MAX_CONCURRENT_REQUESTS` -Defaults to 192 for messages, 320 for snowflakes, 64 otherwise. In-flight request ceiling. The built-in defaults key off `FLUXER_SVC_NAME`. Compose forwards `FLUXER_SVC_MAX_CONCURRENT_REQUESTS` to `users`, `users-shard`, `messages`, and `messages-shard`, and leaves the other containers on the built-in defaults. Left unset, those four keep their built-in defaults too. Once set, the one value replaces the default on all four, so a value below 192 also lowers the messages ceiling. A router holds a slot for the whole round trip to its shard. A request over the ceiling is rejected at once. The API request behind it fails with a 503, and the API logs `shard rejected the request because it is at its concurrency limit`. +Defaults to 192 for messages, 320 for snowflakes, 64 otherwise. In-flight request ceiling. The built-in defaults key off `FLUXER_SVC_NAME`. Compose forwards it from `.env` to every internal service router and shard. Left unset, each keeps its built-in default. Once set, the one value replaces the default on all of them, so a value below 320 also lowers the snowflakes ceiling. A router holds a slot for the whole round trip to its shard. A request over the ceiling is rejected at once. The API request behind it fails with a 503, and the API logs `shard rejected the request because it is at its concurrency limit`. #### `POD_NAME` -No default. The shard ordinal source and node identity. Also read by the Gateway, and by the API as the `pod_name` in its RPC timing records. +No default. The shard ordinal source and node identity. Also read by the Gateway, and by the API as the `pod_name` in its RPC timing records. Compose does not set it. -For custom service routing, add these settings to the API container environment. The bundled Compose file does not forward them: - -- Snowflake service: `FLUXER_SNOWFLAKE_SERVICE_SUBJECT` and `FLUXER_SNOWFLAKE_SERVICE_NATS_CLIENT_NAME`. -- Users service: `FLUXER_USERS_SERVICE_SUBJECT` and `FLUXER_USERS_SERVICE_NATS_CLIENT_NAME`. -- GIF service: `FLUXER_GIF_SERVICE_SUBJECT` and `FLUXER_GIF_SERVICE_NATS_CLIENT_NAME`. +The API reaches the internal services over NATS. Compose forwards the client names and the tuning values below from `.env`. It does not forward `FLUXER_SNOWFLAKE_SERVICE_SUBJECT`, `FLUXER_USERS_SERVICE_SUBJECT` or `FLUXER_GIF_SERVICE_SUBJECT`, because each has to match the subject its internal service listens on. Numeric settings require decimal safe integers. Surrounding whitespace is trimmed, and omitted or blank settings use the defaults. Explicit malformed or out-of-range values are rejected. +- `FLUXER_SNOWFLAKE_SERVICE_NATS_CLIENT_NAME`, `FLUXER_USERS_SERVICE_NATS_CLIENT_NAME` and `FLUXER_GIF_SERVICE_NATS_CLIENT_NAME`: the name the API gives each NATS connection, as NATS monitoring shows it. Each defaults to a name for its service. - `FLUXER_SNOWFLAKE_SERVICE_BATCH_SIZE`: defaults to `128`. Accepts 1 to 512. - `FLUXER_SNOWFLAKE_SERVICE_LOW_WATERMARK`: defaults to `32`, capped below the batch size. Accepts 0 through batch size minus one. - `FLUXER_SNOWFLAKE_SERVICE_MAX_BUFFER_AGE_MS`: defaults to `5000`. Accepts 1 to 60000 milliseconds. @@ -648,7 +650,7 @@ LiveKit is the media server for voice and video. `FLUXER_LIVEKIT_API_KEY` and `F #### `FLUXER_LIVEKIT_ENABLED` -Default `false`. The master switch for all voice and video. Compose sets `true`. +Default `false`. The master switch for all voice and video. Compose forwards it from `.env` with a default of `true`, because the stack ships LiveKit. #### `FLUXER_LIVEKIT_API_KEY` @@ -660,7 +662,7 @@ Default empty. The LiveKit secret. Compose fills it from `LIVEKIT_API_SECRET`. #### `FLUXER_LIVEKIT_URL` -Default empty. The client-facing LiveKit URL. Set it only when LiveKit is served from another host. `docker-compose.yml` never passes it empty: it builds the value from `FLUXER_PUBLIC_ORIGIN`, or from `FLUXER_PUBLIC_SCHEME`, `FLUXER_DOMAIN` and `FLUXER_PUBLIC_PORT` when the origin is unset, followed by `/livekit`. An empty value outside Compose has the API derive `wss://host/livekit` from the public API origin, or `ws://` under `http`, with a non-default port kept. +Default empty. The client-facing LiveKit URL. Set it only when LiveKit is served from another host. `docker-compose.yml` never passes it empty: it builds the value from `FLUXER_PUBLIC_ORIGIN`, or from `FLUXER_PUBLIC_SCHEME`, `FLUXER_DOMAIN` and `FLUXER_PUBLIC_PORT` when the origin is unset, followed by `/livekit`, and forwards a value `.env` sets. An empty value outside Compose has the API derive `wss://host/livekit` from the public API origin, or `ws://` under `http`, with a non-default port kept. #### `FLUXER_LIVEKIT_USE_EXTERNAL_IP` @@ -680,15 +682,15 @@ Default `stun1.l.google.com:19302`. The second STUN server, used the same way. #### `FLUXER_LIVEKIT_INTERNAL_URL` -Default empty. The server-side LiveKit control API. Compose sets `http://livekit:7880`. - -#### `FLUXER_LIVEKIT_WEBHOOK_URL` - -Default empty. Where LiveKit posts webhooks. The API accepts requests on this route without user authentication and without a client-IP header. +Default empty. The server-side LiveKit control API. Compose forwards it from `.env` with a default of `http://livekit:7880`, the bundled service. #### `FLUXER_LIVEKIT_DEFAULT_REGION` -No default. The default voice region. JSON with `id`, `name`, `emoji`, `latitude`, and `longitude`. +No default. The default voice region. JSON with `id`, `name`, `emoji`, `latitude`, and `longitude`. Compose forwards it from `.env`, with a default region named `Default` at latitude and longitude `0`. + +#### `FLUXER_LIVEKIT_LOG_LEVEL` + +Default `info`. How much LiveKit logs. Read by Compose only, which writes it into the LiveKit configuration as `log_level`. #### `FLUXER_LIVEKIT_TCP_PORT` @@ -698,7 +700,7 @@ Default `7881`. The TCP media port. Read by Compose only. Compose publishes it o Default `7882`. The UDP media port, published and passed to LiveKit as `rtc.udp_port` the same way. -LiveKit media does not go through the edge. Compose publishes both media ports directly, so both must stay open in the host firewall and be forwarded to the host when it sits behind NAT. Compose points LiveKit at the webhook target `http://api:8080/webhooks/livekit` and configures no TURN server. A client that cannot use UDP falls back to ICE-TCP on the TCP port. +LiveKit media does not go through the edge. Compose publishes both media ports directly, so both must stay open in the host firewall and be forwarded to the host when it sits behind NAT. Compose points LiveKit at the webhook target `http://api:8080/webhooks/livekit` and configures no TURN server. The API accepts requests on that route without user authentication and without a client-IP header. A client that cannot use UDP falls back to ICE-TCP on the TCP port. Moving a media port is one line in `.env` followed by `docker compose up -d livekit`. Compose puts the same value on the host side of the mapping, on the container side, and on the `rtc` port LiveKit advertises in the ICE candidates it hands to clients. @@ -714,45 +716,45 @@ The same conditions turn on a DNS check at registration. The check runs when ema #### `FLUXER_EMAIL_ENABLED` -`.env.example` `false`. The delivery switch. The admin dashboard value wins over this. +`.env.example` `false`. The delivery switch. The admin dashboard value wins over this. Compose forwards it from `.env`. #### `FLUXER_EMAIL_PROVIDER` -`.env.example` `none`. The transport. `smtp` or `none`. Anything else fails startup. +`.env.example` `none`. The transport. `smtp` or `none`. Anything else fails startup. Compose forwards it from `.env`. #### `FLUXER_EMAIL_FROM_EMAIL` -`.env.example` `noreply@example.com`. The sender address. Compose falls back to `noreply@localhost`. +`.env.example` `noreply@example.com`. The sender address. Default empty. Compose forwards it from `.env`, and uses `noreply@localhost` when it is unset. #### `FLUXER_EMAIL_FROM_NAME` -`.env.example` `Fluxer`. The sender name. An empty value sets an empty sender name. +`.env.example` `Fluxer`. The sender name. Default `Fluxer`, which an empty value also gives. Compose forwards it from `.env`. #### `FLUXER_EMAIL_APP_BASE_URL` -`.env.example` empty. The base URL used in links. Must be http or https with no username, password, query, or fragment, or the API fails at boot. Falls back to the app endpoint. +`.env.example` empty. The base URL used in links. Must be http or https with no username, password, query, or fragment, or the API fails at boot. Falls back to the app endpoint. Compose forwards it from `.env`. #### `FLUXER_EMAIL_SMTP_HOST` -`.env.example` empty. The SMTP host. Setting any SMTP variable creates the whole SMTP block, which is absent by default. +`.env.example` empty. The SMTP host. Setting any SMTP variable creates the whole SMTP block, which is absent by default. Compose forwards it from `.env`. #### `FLUXER_EMAIL_SMTP_PORT` -`.env.example` `587`. The SMTP port. Falls back to 587. +`.env.example` `587`. The SMTP port. Falls back to 587. Compose forwards it from `.env`. #### `FLUXER_EMAIL_SMTP_USERNAME` -`.env.example` empty. The SMTP login. Part of the completeness check. +`.env.example` empty. The SMTP login. Part of the completeness check. Compose forwards it from `.env`. #### `FLUXER_EMAIL_SMTP_PASSWORD` -`.env.example` empty. The SMTP password. Part of the completeness check. +`.env.example` empty. The SMTP password. Part of the completeness check. Compose forwards it from `.env`. #### `FLUXER_EMAIL_SMTP_SECURE` -`.env.example` `true`. Implicit TLS. Defaults to `true` whenever the provider is `smtp`. +`.env.example` `true`. Implicit TLS. Defaults to `true` whenever the provider is `smtp`. Compose forwards it from `.env`. -The API reads `FLUXER_EMAIL_WEBHOOK_SECRET` for inbound delivery webhooks. Neither `.env.example` nor `docker-compose.yml` has it. +The API reads `FLUXER_EMAIL_WEBHOOK_SECRET` for inbound delivery webhooks. Compose forwards it from `.env`, and `.env.example` lists it commented out. The API registers `POST /webhooks/sweego` on every deployment. The route authenticates the Standard Webhooks header triple `webhook-id`, `webhook-timestamp`, and `webhook-signature` against that secret, and answers 404 `Email not enabled` while email is off. The route sits outside the client-IP exempt list that covers `/webhooks/livekit`, so give the provider a delivery URL that goes through the edge, `https://chat.example.com/api/webhooks/sweego`. @@ -768,19 +770,19 @@ All are optional. #### `FLUXER_SSO_ALLOW_PRIVATE_ADDRESSES` -Default `false`. Whether the SSO provider URL may resolve to a private address. Off by default as SSRF protection. Turn it on only for split-horizon DNS or a LAN identity provider. +Default `false`. Whether the SSO provider URL may resolve to a private address. Off by default as SSRF protection. Turn it on only for split-horizon DNS or a LAN identity provider. Compose forwards it from `.env`. #### `FLUXER_PASSKEY_RP_ID` -Defaults to `FLUXER_BASE_DOMAIN`. The WebAuthn relying party identifier. Changing it invalidates every passkey already registered. +Defaults to the host of `FLUXER_PUBLIC_ORIGIN` when that is set, and to `FLUXER_BASE_DOMAIN` otherwise. The WebAuthn relying party identifier. Changing it invalidates every passkey already registered. Compose forwards it from `.env` with a default of `FLUXER_DOMAIN`, so a public origin on another host leaves existing passkeys valid. #### `FLUXER_PASSKEY_RP_NAME` -Default `Fluxer`. The relying party name browsers display. Does not follow `FLUXER_DOMAIN`. +Default `Fluxer`. The relying party name browsers display. Does not follow `FLUXER_DOMAIN`. Compose forwards it from `.env`. #### `FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS` -The complete accepted origin set, comma separated. Despite the name, an explicit value replaces the defaults and must list every origin clients use. Compose defaults it to the public web app origin. When the variable is omitted outside Compose, the API accepts the configured app origin alongside the built-in Fluxer web and Android origins. An empty list selects only the configured app origin. +The complete accepted origin set, comma separated. Despite the name, an explicit value replaces the defaults and must list every origin clients use. Compose forwards it from `.env` with a default of the public web app origin, so a self-hosted instance accepts its own origin alone. When the variable is omitted or empty outside Compose, the API accepts the configured app origin alongside the built-in Fluxer web and Android origins. To accept only the app origin, list it alone. Each web entry must be an HTTP(S) origin with no credentials, path beyond an optional final slash, query, or fragment. Web origins are normalised to browser form using the configured public port unless the entry supplies one. Android entries must use `android:apk-key-hash:` followed by the signing certificate's SHA-256 fingerprint in canonical, unpadded base64url (43 characters). Invalid entries and control characters fail startup. @@ -792,7 +794,7 @@ A passkey is bound to the `FLUXER_PASSKEY_RP_ID` it was registered under. Member Bluesky connections are off by default. Enable them in the admin dashboard's Runtime Integrations panel and supply an ES256 private signing key with a unique key identifier. The public API must serve the [client metadata and signing keys](/http-api/connections/#get-bluesky-client-metadata). -For environment-based configuration, use `FLUXER_AUTH_BLUESKY_ENABLED`, `FLUXER_AUTH_BLUESKY_KEYS`, and the optional `FLUXER_AUTH_BLUESKY_CLIENT_NAME`, `FLUXER_AUTH_BLUESKY_CLIENT_URI`, `FLUXER_AUTH_BLUESKY_LOGO_URI`, `FLUXER_AUTH_BLUESKY_TOS_URI`, and `FLUXER_AUTH_BLUESKY_POLICY_URI` settings. Add them to the API container environment. The shipped Compose file does not forward them. +For environment-based configuration, use `FLUXER_AUTH_BLUESKY_ENABLED`, `FLUXER_AUTH_BLUESKY_KEYS`, and the optional `FLUXER_AUTH_BLUESKY_CLIENT_NAME`, `FLUXER_AUTH_BLUESKY_CLIENT_URI`, `FLUXER_AUTH_BLUESKY_LOGO_URI`, `FLUXER_AUTH_BLUESKY_TOS_URI`, and `FLUXER_AUTH_BLUESKY_POLICY_URI` settings. Compose forwards every one from `.env`. ## Web push @@ -802,105 +804,115 @@ For environment-based configuration, use `FLUXER_AUTH_BLUESKY_ENABLED`, `FLUXER_ | --- | --- | --- | | FLUXER_VAPID_PUBLIC_KEY | `CHANGE_ME` | The VAPID public key. Base64url of the 65-byte uncompressed P-256 point | | FLUXER_VAPID_PRIVATE_KEY | `CHANGE_ME` | The VAPID private key. Base64url of the 32-byte scalar, and the matching half of the pair | -| FLUXER_VAPID_EMAIL | unset | The VAPID contact address. Compose derives `admin@` followed by `FLUXER_DOMAIN` when it is unset | +| FLUXER_VAPID_EMAIL | unset | The VAPID contact address. Compose forwards it from `.env`, with a default of `admin@` followed by `FLUXER_DOMAIN` | -`FLUXER_GATEWAY_PUSH_ENABLED` is read by the Gateway alone and defaults to `true`. When it is `false`, the Gateway hands no message or clear notification to `push`. +`FLUXER_GATEWAY_PUSH_ENABLED` is read by the Gateway alone and defaults to `true`. When it is `false`, the Gateway hands no message or clear notification to `push`. Compose forwards it from `.env`. ## Mobile push -`api` and `push` read the APNs and FCM names. None appear in `.env.example` or in `docker-compose.yml`, so configuring mobile push means editing the Compose file. All are optional. +`push` reads every name below. `api` also reads the APNs names, apart from `FLUXER_PUSH_APNS_DEFAULT_ENVIRONMENT`. Compose forwards the APNs names from `.env` to both, and `FLUXER_PUSH_APNS_DEFAULT_ENVIRONMENT` and the FCM names to `push` alone. All are optional. #### `FLUXER_PUSH_APNS_ENABLED` -Default `false`. The APNs switch. Must be set on `api` and `push`. +Default `false`. The APNs switch. Read by `api` and `push`. Compose forwards it from `.env`. #### `FLUXER_PUSH_APNS_TEAM_ID` -No default. The Apple team. Paired with the key ID. +No default. The Apple team. Paired with the key ID. Compose forwards it from `.env`. #### `FLUXER_PUSH_APNS_KEY_ID` -No default. The signing key ID. Paired with the team ID. +No default. The signing key ID. Paired with the team ID. Compose forwards it from `.env`. #### `FLUXER_PUSH_APNS_PRIVATE_KEY` -No default. The signing key in PEM form. Set exactly one of this and the path. +No default. The signing key in PEM form. Set exactly one of this and the path. Compose forwards it from `.env`. #### `FLUXER_PUSH_APNS_PRIVATE_KEY_PATH` -No default. A path to the signing key. Must be readable inside the container. +No default. A path to the signing key. Must be readable inside the container. Compose forwards it from `.env`. #### `FLUXER_PUSH_APNS_DEFAULT_ENVIRONMENT` -Default `production`. Which APNs environment is used. `production` or `development`. +Default `production`. Which APNs environment is used. `production` or `development`. Read by `push` only. Compose forwards it from `.env`. #### `FLUXER_PUSH_APNS_APPS` -Default `[]`. Per-app APNs configuration. JSON array. An entry with no `app_id` fails startup. +Default `[]`. Per-app APNs configuration. JSON array. An entry with no `app_id` fails startup. Compose forwards it from `.env`. #### `FLUXER_PUSH_FCM_ENABLED` -Default `false`. The FCM switch. Must be set on `api` and `push`. +Default `false`. The FCM switch. Read by `push` only, like every FCM name. Compose forwards it from `.env`. #### `FLUXER_PUSH_FCM_PROJECT_ID` -No default. The Firebase project. Paired with the client email. +No default. The Firebase project. Paired with the client email. Compose forwards it from `.env`. #### `FLUXER_PUSH_FCM_CLIENT_EMAIL` -No default. The service account address. Paired with the project. +No default. The service account address. Paired with the project. Compose forwards it from `.env`. #### `FLUXER_PUSH_FCM_PRIVATE_KEY` -No default. The service account key. Set one of this, `FLUXER_PUSH_FCM_PRIVATE_KEY_PATH`, or `FLUXER_PUSH_FCM_SERVICE_ACCOUNT_JSON_PATH`. +No default. The service account key. Set one of this, `FLUXER_PUSH_FCM_PRIVATE_KEY_PATH`, or `FLUXER_PUSH_FCM_SERVICE_ACCOUNT_JSON_PATH`. Compose forwards it from `.env`. #### `FLUXER_PUSH_FCM_PRIVATE_KEY_PATH` -No default. A path to the key. Must be readable inside the container. +No default. A path to the key. Must be readable inside the container. Compose forwards it from `.env`. #### `FLUXER_PUSH_FCM_SERVICE_ACCOUNT_JSON_PATH` -No default. A path to the whole service account JSON. Must be readable inside the container. +No default. A path to the whole service account JSON. Must be readable inside the container. Compose forwards it from `.env`. #### `FLUXER_PUSH_FCM_TOKEN_URI` -Default `https://oauth2.googleapis.com/token`. The OAuth token endpoint. Change only for a proxy or a test double. +Default `https://oauth2.googleapis.com/token`. The OAuth token endpoint. Change only for a proxy or a test double. Compose forwards it from `.env`. #### `FLUXER_PUSH_FCM_APPS` -Default `[]`. Per-app FCM configuration. JSON array, under the same `app_id` rule as APNs. +Default `[]`. Per-app FCM configuration. JSON array, under the same `app_id` rule as APNs. Compose forwards it from `.env`. ## Push service settings -`push` is the push notification service in the bundled stack. It reads the VAPID pair from [Web push](#web-push), the APNs and FCM names from [Mobile push](#mobile-push), and `FLUXER_SVC_NATS_URL` with `FLUXER_NATS_AUTH_TOKEN` from [Message bus and internal services](#message-bus-and-internal-services). It refuses to start without `FLUXER_INTERNAL_API_ENDPOINT` and `FLUXER_GATEWAY_RPC_AUTH_TOKEN`. Compose supplies both. The names below belong to `push`. All are optional. +`push` is the push notification service in the bundled stack. It reads the VAPID pair from [Web push](#web-push), the APNs and FCM names from [Mobile push](#mobile-push), and `FLUXER_SVC_NATS_URL` with `FLUXER_NATS_AUTH_TOKEN` from [Message bus and internal services](#message-bus-and-internal-services). It refuses to start without `FLUXER_INTERNAL_API_ENDPOINT` and `FLUXER_GATEWAY_RPC_AUTH_TOKEN`. Compose supplies both. The names below belong to `push`, and Compose forwards every one from `.env` except the bind address and port. All are optional. #### `FLUXER_PUSH_SERVICE_HOST` -Default `0.0.0.0`. The bind address. It must parse as an IP address. Also settable with `--bind-host`. Compose sets `0.0.0.0`. +Default `0.0.0.0`. The bind address. It must parse as an IP address. Also settable with `--bind-host`. Compose does not set it, so the default applies. #### `FLUXER_PUSH_SERVICE_PORT` -Default `8126`. The listen port for the health and metrics endpoints. Also settable with `--port`. Compose sets `8126`. +Default `8126`. The listen port for the health and metrics endpoints. Also settable with `--port`. Compose does not set it, so the default applies. #### `FLUXER_PUSH_SERVICE_QUEUE_CAPACITY` -Default `10000`. Notification jobs `push` holds at once. Accepts 1 to 1000000. Compose passes it through from `.env`. An empty value keeps the default. +Default `10000`. Notification jobs `push` holds at once. Accepts 1 to 1000000. Compose forwards it from `.env`. #### `FLUXER_PUSH_SERVICE_SEND_CONCURRENCY` -Default `256`. Provider requests in flight at once, across every job. Accepts 1 to 65536. Compose passes it through from `.env`. An empty value keeps the default. +Default `256`. Provider requests in flight at once, across every job. Accepts 1 to 65536. Compose forwards it from `.env`. #### `FLUXER_PUSH_SERVICE_APNS_BASE_URL` -No default. Replaces the APNs host in both environments. Set it only for a test double. +No default. Replaces the APNs host in both environments. Set it only for a test double. Compose forwards it from `.env`. #### `FLUXER_PUSH_SERVICE_FCM_BASE_URL` -Default `https://fcm.googleapis.com`. The FCM host. Set it only for a proxy or a test double. +Default `https://fcm.googleapis.com`. The FCM host. Set it only for a proxy or a test double. Compose forwards it from `.env`. #### `FLUXER_PUSH_SERVICE_RELAY_CONSENT_ACCEPTED` -Default `false`. Accepts the push relay supplemental privacy notice for this `push` process. `true`, `1`, or `yes` accepts it, and `false`, `0`, or `no` leaves the decision to the [push relay configuration](/admin-api/instance/#push-relay-configuration-object). Any other value fails startup. When it is `true`, `push` sends through the Fluxer-run relay even while the stored consent is off. Compose does not forward it. +Default `false`. Accepts the push relay supplemental privacy notice for this `push` process. `true`, `1`, or `yes` accepts it, and `false`, `0`, or `no` leaves the decision to the [push relay configuration](/admin-api/instance/#push-relay-configuration-object). Any other value fails startup. When it is `true`, `push` sends through the Fluxer-run relay even while the stored consent is off. Compose forwards it from `.env`. + +#### `FLUXER_PUSH_SERVICE_MANAGED_RELAY_HOSTS` + +Default `push.fluxer.com`. Hostnames, comma separated, of the Fluxer-run push relay. A browser push endpoint on one of them needs the relay consent above, and a refusal from it counts against the relay quota. Compose forwards it from `.env`. + +#### `FLUXER_PUSH_SERVICE_OWN_RELAY_HOSTS` + +Default empty. Hostnames, comma separated, of a push relay this instance runs itself. `push` delivers a browser push endpoint on one of them straight through its own APNs or FCM credentials, with no HTTP hop. Compose forwards it from `.env`. + +`push` also runs as a relay under `--mode relay`, which the stack does not use. Only that mode reads `FLUXER_PUSH_RELAY_MAX_CONCURRENT`, default `1024`, `FLUXER_PUSH_RELAY_MAX_BODY_BYTES`, default `2816`, `FLUXER_PUSH_RELAY_TRUSTED_PROXY_HOPS`, default `1`, and `FLUXER_PUSH_RELAY_SOURCE_BUCKET_ENABLED`, default `false`. Compose does not forward them, and [Names the stack has no use for](#names-the-stack-has-no-use-for) lists them. ## Payments @@ -914,23 +926,23 @@ Default `false`. The Stripe switch. Compose forwards it from `.env`. #### `FLUXER_STRIPE_SECRET_KEY` -Default empty. The Stripe secret key. Not forwarded by the shipped Compose file. +Default empty. The Stripe secret key. Compose forwards it from `.env`. #### `FLUXER_STRIPE_WEBHOOK_SECRET` -Default empty. The webhook signing secret. Not forwarded by the shipped Compose file. +Default empty. The webhook signing secret. Compose forwards it from `.env`. #### `FLUXER_STRIPE_PRICES` -Default `{}`. Every price ID at once. JSON object. An individual price variable overrides the matching price in this object. Not forwarded by the shipped Compose file. +Default `{}`. Every price ID at once. JSON object. Compose forwards it from `.env`. -Individual price variables also exist, one per product and currency: `FLUXER_STRIPE_PRICE_MONTHLY_`, `FLUXER_STRIPE_PRICE_YEARLY_`, `FLUXER_STRIPE_PRICE_GIFT_1_MONTH_` and `FLUXER_STRIPE_PRICE_GIFT_1_YEAR_` in USD, EUR, BRL, DKK, INR, NOK, PLN, SEK, and TRY. +Individual price variables also exist, one per product and currency: `FLUXER_STRIPE_PRICE_MONTHLY_`, `FLUXER_STRIPE_PRICE_YEARLY_`, `FLUXER_STRIPE_PRICE_GIFT_1_MONTH_` and `FLUXER_STRIPE_PRICE_GIFT_1_YEAR_` in USD, EUR, BRL, DKK, INR, NOK, PLN, SEK, and TRY. Each overrides the matching price in this object. Compose does not forward them, so a self-hosted instance sets prices with this variable or in the admin dashboard. Prices saved in the dashboard replace this catalogue entirely. They take any three-letter currency code. A buyer gets the currency mapped to their country, then the default currency, then the first configured one. Fluxer's country checks for localised currencies apply only to prices set by these variables. #### `FLUXER_STRIPE_LEGACY_PRICES` -Default `{}`. Retired price IDs, keyed by the same slot names `FLUXER_STRIPE_PRICES` uses, each mapped to a list: `{"monthly_brl": ["price_..."]}`. JSON object. Existing subscriptions on these prices keep renewing. The localised checkout catalogue uses `FLUXER_STRIPE_PRICES` alone. Legacy prices saved in the dashboard replace this variable, and there the currency in a slot name is upper case, such as `monthly_GBP`. +Default `{}`. Retired price IDs, keyed by the same slot names `FLUXER_STRIPE_PRICES` uses, each mapped to a list: `{"monthly_brl": ["price_..."]}`. JSON object. Compose forwards it from `.env`. Existing subscriptions on these prices keep renewing. The localised checkout catalogue uses `FLUXER_STRIPE_PRICES` alone. Legacy prices saved in the dashboard replace this variable, and there the currency in a slot name is upper case, such as `monthly_GBP`. Repricing a slot has a fixed order. Doing the steps in another order leaves a price ID unknown to the API, and invoices on that price fail. Create the new price in Stripe, move the ID it replaces into this variable, roll the API, and only then point `FLUXER_STRIPE_PRICES` at the new price. A price ID that neither variable names is unknown to the API: a renewal invoice on it fails the webhook with `Unknown product for invoice renewal`, a checkout completing on it fails with `Unknown price ID for checkout session`, and both keep failing until the ID is registered. The API answers Stripe as soon as the signature verifies and hands the event to `worker`, so any retry comes from `worker` rerunning the `processStripeWebhook` job: the Stripe dashboard shows the delivery as succeeded and the error is in the `worker` logs. Keep a retired ID listed for as long as any subscription still bills on it, which for a yearly price is at least a year after the switch. @@ -942,57 +954,53 @@ All are optional. #### `FLUXER_NCMEC_ENABLED` -Default `false`. NCMEC reporting. Compose hardcodes `false`. +Default `false`. NCMEC reporting. Compose forwards it from `.env`. #### `FLUXER_NCMEC_BASE_URL` -Default empty. The reporting endpoint. Required when reporting is on. +Default empty. The reporting endpoint, `https://report.cybertip.org/ispws` in production. Required when reporting is on. Compose forwards it from `.env`. #### `FLUXER_NCMEC_USERNAME` -Default empty. The reporting login. Required when reporting is on. +Default empty. The reporting login. Required when reporting is on. Compose forwards it from `.env`. #### `FLUXER_NCMEC_PASSWORD` -Default empty. The reporting password. Required when reporting is on. +Default empty. The reporting password. Required when reporting is on. Compose forwards it from `.env`. #### `FLUXER_NCMEC_REPORTER_EMAIL` -Default empty. The contact address on reports. Required when reporting is on. +Default empty. The contact address on reports. Required when reporting is on. Compose forwards it from `.env`. #### `FLUXER_CLAMAV_ENABLED` -Default `false`. Upload virus scanning. Compose hardcodes `false`, and no ClamAV container ships. +Default `false`. Upload virus scanning. Compose forwards it from `.env`, and no ClamAV container ships. #### `FLUXER_CLAMAV_HOST` -Default `127.0.0.1`. The scanner host. Needs a reachable scanner. +Default `127.0.0.1`. The scanner host. Needs a reachable scanner. Compose forwards it from `.env`. #### `FLUXER_CLAMAV_PORT` -Default `3310`. The scanner port. Integer. +Default `3310`. The scanner port. Integer. Compose forwards it from `.env`. #### `FLUXER_CLAMAV_FAIL_OPEN` -Default `false`. Behaviour when the scanner is unreachable. With scanning on and this off, an unreachable scanner rejects every upload. - -#### `FLUXER_API_CONTENT_MODERATION_NSFW_THRESHOLD` - -Default `0.7`. The API-side NSFW score cutoff. No range check. Distinct from the Media Proxy threshold, which defaults to `0.85`. +Default `false`. Behaviour when the scanner is unreachable. With scanning on and this off, an unreachable scanner rejects every upload. Compose forwards it from `.env`. #### `FLUXER_IPINFO_API_KEY` -Default empty. The ipinfo key. Admin and guild IP bans use it to skip carrier-grade NAT and other shared addresses. Without it those checks treat every address as unknown. The previous name `FLUXER_RISK_IPINFO_API_KEY` is still read when this one is unset. +Default empty. The ipinfo key. Admin and guild IP bans use it to skip carrier-grade NAT and other shared addresses. Without it those checks treat every address as unknown. The previous name `FLUXER_RISK_IPINFO_API_KEY` is still read when this one is unset or empty. Compose forwards this name from `.env`, and not the previous one. #### `FLUXER_BLOCKLIST_FEEDS_ENABLED` Defaults to the inverse of `FLUXER_SELF_HOSTED`. Off by default on a self-hosted instance. With feeds on, the worker downloads the URLhaus and PhishTank URL lists every six hours, and MalwareBazaar file hashes every twelve hours. Fluxer checks posted links against the URLs and uploads against the hashes. -With feeds off, Fluxer checks none of this data. The first worker start with feeds off removes the URL feed file and every `malware_bazaar` file-SHA ban that no Admin added. File-SHA bans added through the Admin API stay. Turning feeds back on downloads the URLs within six hours and the hashes within twelve. +With feeds off, Fluxer checks none of this data. The first worker start with feeds off removes the URL feed file and every `malware_bazaar` file-SHA ban that no Admin added. File-SHA bans added through the Admin API stay. Turning feeds back on downloads the URLs within six hours and the hashes within twelve. Compose forwards it from `.env`. #### `FLUXER_BREACHED_PASSWORD_CHECK_ENABLED` -Defaults to the inverse of `FLUXER_SELF_HOSTED`. Breached password rejection. Sends the first five characters of the password's SHA-1 hash to `api.pwnedpasswords.com`. Off by default on a self-hosted instance. +Defaults to the inverse of `FLUXER_SELF_HOSTED`. Breached password rejection. Sends the first five characters of the password's SHA-1 hash to `api.pwnedpasswords.com`. Off by default on a self-hosted instance. Compose forwards it from `.env`. ## Stored instance policy @@ -1018,55 +1026,51 @@ Request concurrency is separate from instance limits, and each process sets its #### `FLUXER_API_MAX_INFLIGHT_REQUESTS` -Default `512`. The API in-flight ceiling. Integer 1 to 100000, checked at startup. +Default `512`. The API in-flight ceiling. Integer 1 to 100000, checked at startup. Compose forwards it from `.env`. #### `FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY` -Default `512`. Gateway HTTP RPC concurrency. +Default `512`. Gateway HTTP RPC concurrency. Compose forwards it from `.env` to `gateway`. #### `FLUXER_GATEWAY_NATS_RPC_MAX_HANDLERS` -Default `512`. Gateway NATS handler count. +Default `512`. Gateway NATS handler count. Compose forwards it from `.env` to `gateway`. #### `FLUXER_DISABLE_RATE_LIMITS` -Default `false`. Turns rate limits off. Read by the API and by the Gateway. The Gateway turns rate limits off only for the values `1`, `true` and `TRUE`. +Default `false`. Turns rate limits off. Read by the API and by the Gateway. The Gateway turns rate limits off only for the values `1`, `true` and `TRUE`. A test setting, which Compose does not forward. #### `FLUXER_RELAX_REGISTRATION_RATE_LIMITS` -Default `false`. Loosens registration rate limits. Reaches production containers if set. +Default `false`. Loosens registration rate limits. A test setting, which Compose does not forward. -## Telemetry, logging and build metadata +## Logging and build metadata All are optional. #### `LOG_LEVEL` -Defaults to `debug` in development, `info` otherwise. The Node log level. Read by `api` and `worker`. +Defaults to `debug` in development, `info` otherwise. The Node log level. Read by `api` and `worker`. Compose forwards it from `.env`. #### `RUST_LOG` -Default `info`. The Rust log filter. Read by `media-proxy`, `app-proxy`, `admin`, `push`, and the internal services. Not in `.env.example` or the Compose file. +Default `info`. The Rust log filter. Read by `media-proxy`, `app-proxy`, `admin`, `push`, and the internal services. An empty or invalid filter also gives `info`, and `off` silences them. Compose forwards it from `.env`. #### `FLUXER_GATEWAY_LOGGER_LEVEL` -Default `info`. The Gateway log level. Compose sets `info`. +Default `info`. The Gateway log level. An unrecognised level also gives `info`. Compose forwards it from `.env` to `gateway`. #### `LOGGER_LEVEL` -Falls back to `FLUXER_GATEWAY_LOGGER_LEVEL`. The Gateway log level at runtime. Overrides the prefixed name. +Falls back to `FLUXER_GATEWAY_LOGGER_LEVEL`. The Gateway log level at runtime. Overrides the prefixed name, and an unrecognised level falls back to it. Compose forwards it from `.env` to `gateway`. #### `BUILD_VERSION` -Default `dev`. The reported build. Baked into the images. When `BUILD_VERSION` is unset or empty outside development, the process prints a warning. +Default `dev`. The reported build. Baked into the images, so Compose does not set it. When `BUILD_VERSION` is unset or empty outside development, the process prints a warning. #### `RELEASE_CHANNEL` -Default `stable`. The reported channel. Only `canary` is recognised as non-stable. - -#### `FLUXER_TELEMETRY_ENABLED` - -Defaults to `false` at the Gateway's environment layer. Gateway telemetry. Read only by the Gateway, and in neither `.env.example` nor the Compose file. +Default `stable`. The reported channel. Only `canary` is recognised as non-stable. Read by `app-proxy` only. It describes the shipped build, so Compose does not set it. #### `HOSTNAME` @@ -1088,151 +1092,135 @@ All are optional. #### `FLUXER_DISCOVERY_ENABLED` -Default `true`. The public guild [discovery](/http-api/discovery/) surface. With it off, discovery search, discovery join, and the guild discovery application routes return 400 `DISCOVERY_DISABLED`. +Default `true`. The public guild [discovery](/http-api/discovery/) surface. With it off, discovery search, discovery join, and the guild discovery application routes return 400 `DISCOVERY_DISABLED`. Compose forwards it from `.env`. #### `FLUXER_DISCOVERY_MIN_MEMBER_COUNT` -Default `1`. Minimum members for discovery eligibility. Integer. +Default `1`. Minimum members for discovery eligibility. Integer. Compose forwards it from `.env`. #### `FLUXER_DELETION_GRACE_PERIOD_HOURS` -Default `336`. How long a deleted account is recoverable. Forced to 0.01 hours when `FLUXER_TEST_MODE_ENABLED` is on. +Default `336`. How long a deleted account is recoverable. Forced to 0.01 hours when `FLUXER_TEST_MODE_ENABLED` is on. Compose forwards it from `.env`. #### `FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED` -Default `false`. Presigned attachment uploads. Compose sets `true`. +Default `false`. Presigned attachment uploads. Only `api` reads it. Compose forwards it from `.env` to `api` with a default of `true`. #### `FLUXER_API_PRESIGNED_HARVEST_DOWNLOADS_ENABLED` -Default `true`. Presigned harvest downloads. Applies to the harvests bucket. +Default `true`. Presigned harvest downloads. Applies to the harvests bucket. A presigned link points at `FLUXER_S3_PUBLIC_ENDPOINT`, which in the bundled stack is the internal `http://seaweedfs:8333` that no browser reaches. Compose therefore forwards it from `.env` with a default of `false`. Turn it on once `FLUXER_S3_PUBLIC_ENDPOINT` names an address browsers can reach. -#### `FLUXER_API_EMBEDS_OEMBED_HTML_ENABLED` +#### `FLUXER_API_STORAGE_CHANGE_FEED_ENABLED` -Default `false`. oEmbed HTML in embeds. +Default `false`. Whether `api` and `worker` publish a record of every object-store write and delete to a JetStream stream. Compose forwards it from `.env`. -#### `FLUXER_API_EMBEDS_OEMBED_HTML_ALLOW_UNTRUSTED_ON_SELF_HOSTED` +#### `FLUXER_API_STORAGE_CHANGE_FEED_STREAM` -Default `false`. Untrusted oEmbed HTML on a self-hosted instance. Security relevant. Leave it off. +Default `STORAGE_CHANGES`. The JetStream stream the feed publishes to. Letters, digits, underscores or hyphens, checked at startup only while the feed is on. Compose forwards it from `.env`. -#### `FLUXER_API_EMBEDS_OEMBED_HTML_ALLOWED_HOSTS` +#### `FLUXER_API_STORAGE_CHANGE_FEED_SKIP_BUCKETS` -Default empty. Hosts allowed to supply oEmbed HTML. Comma separated. - -#### `FLUXER_API_EMBEDS_CACHE_DEFAULT_TTL_SECONDS` - -Default `86400`. Default embed cache lifetime. Seconds. - -#### `FLUXER_API_EMBEDS_CACHE_MAX_TTL_SECONDS` - -Default `604800`. Maximum embed cache lifetime. Seconds. - -#### `FLUXER_API_EMBEDS_CACHE_MIN_TTL_SECONDS` - -Default `300`. Minimum embed cache lifetime. Seconds. - -#### `FLUXER_API_EMBEDS_CACHE_RESPECT_REMOTE_TTL` - -Default `true`. Whether a remote cache header is honoured. Clamped by `FLUXER_API_EMBEDS_CACHE_MIN_TTL_SECONDS` and `FLUXER_API_EMBEDS_CACHE_MAX_TTL_SECONDS`. +Defaults to the uploads bucket. Buckets the feed leaves out, comma separated. `none` leaves out no bucket. Compose forwards it from `.env`. #### `FLUXER_API_UNFURL_IGNORED_HOSTS` -Default empty. Hosts never unfurled. Comma separated, unvalidated. +Default empty. Hosts never unfurled. Comma separated, unvalidated. Compose forwards it from `.env`. #### `FLUXER_TEST_MODE_ENABLED` -Default `false`. Test mode. Collapses the deletion grace period. Reaches production containers if set. +Default `false`. Test mode. Collapses the deletion grace period. A test setting, which Compose does not forward. #### `FLUXER_TEST_HARNESS_TOKEN` -No default. The test harness credential. Reaches production containers if set. +No default. The test harness credential. A test setting, which Compose does not forward. #### `FLUXER_VALIDATE_RESPONSES` -Defaults to on outside production. Response schema validation. Costs latency when forced on. +Defaults to on outside production. Response schema validation. Costs latency when forced on. A development setting, which Compose does not forward. #### `FLUXER_KLIPY_API_KEY` -Default empty. The Klipy GIF provider key. GIF search reports itself unavailable while this and the admin dashboard key are both empty. Also read by `unfurl`, which accepts `KLIPY_API_KEY` as a fallback. +Default empty. The Klipy GIF provider key. GIF search reports itself unavailable while this and the admin dashboard key are both empty. Also read by `unfurl`, which accepts `KLIPY_API_KEY` as a fallback. Compose forwards this name from `.env`, and not the fallback. #### `FLUXER_YOUTUBE_API_KEY` -Default empty. The YouTube Data API key. Also read by `unfurl`, which accepts `YOUTUBE_API_KEY` as a fallback. +Default empty. The YouTube Data API key. Also read by `unfurl`, which accepts `YOUTUBE_API_KEY` as a fallback. Compose forwards this name from `.env`, and not the fallback. #### `FLUXER_CACHE_PURGE_ADAPTER` -Default `none`. How cached copies of deleted or replaced media are purged. `none` or `http`. Anything else fails startup. With `none`, nothing is queued or sent. +Default `none`. How cached copies of deleted or replaced media are purged. `none` or `http`. Anything else fails startup. With `none`, nothing is queued or sent. Compose forwards it from `.env`. #### `FLUXER_CACHE_PURGE_HTTP_ENDPOINT` -Default empty. Required when the adapter is `http`. Must be an absolute `http` or `https` URL without credentials, or startup fails. While purges are queued, the worker posts JSON `{"exact": [...], "prefix": [...]}` here every 10 seconds. Redirects are not followed. Any 2xx marks the batch done. Your endpoint then purges those URLs from every proxy that caches media. A `400` or `422` makes the worker resend each URL alone, and a URL refused alone moves to the `cache_purge:rejected` key. Any other failure is retried on a later run. +Default empty. Required when the adapter is `http`. Must be an absolute `http` or `https` URL without credentials, or startup fails. While purges are queued, the worker posts JSON `{"exact": [...], "prefix": [...]}` here every 10 seconds. Redirects are not followed. Any 2xx marks the batch done. Your endpoint then purges those URLs from every proxy that caches media. A `400` or `422` makes the worker resend each URL alone, and a URL refused alone moves to the `cache_purge:rejected` key. Any other failure is retried on a later run. Compose forwards it from `.env`. #### `FLUXER_CACHE_PURGE_HTTP_TOKEN` -Default empty. The bearer token sent in the `Authorization` header of each purge request. Empty sends no header. When the adapter is `http`, a token with spaces, line breaks or non-ASCII characters fails startup. +Default empty. The bearer token sent in the `Authorization` header of each purge request. Empty sends no header. When the adapter is `http`, a token with spaces, line breaks or non-ASCII characters fails startup. Compose forwards it from `.env`. #### `FLUXER_CACHE_PURGE_HTTP_TIMEOUT_MS` -Default `10000`. Purge request timeout. Accepts 1000 to 10000. The range is checked only when the adapter is `http`. +Default `10000`. Purge request timeout. Accepts 1000 to 10000. The range is checked only when the adapter is `http`. Compose forwards it from `.env`. #### `FLUXER_GEOIP_DB_PATH` -Default empty. The GeoIP database. A filesystem path, or an `s3://bucket/key` URL whose `download_path` query parameter is mandatory and must be absolute. `app-proxy` also accepts `MAXMIND_DB_PATH`. +Default empty. The GeoIP database. A filesystem path, or an `s3://bucket/key` URL whose `download_path` query parameter is mandatory and must be absolute. Read by `api`, `worker` and `app-proxy`. Compose forwards it from `.env`. ## Instance identity and branding -None of these are in `.env.example` or in `docker-compose.yml`. Set branding from the admin dashboard instead. All are optional. +Compose forwards every name below from `.env`. The admin dashboard also sets the product name and the brand assets, and a value saved there wins. All are optional. #### `FLUXER_APP_PRODUCT_NAME` -Default `Fluxer`. The product name clients display. Also settable in the admin dashboard, which wins. +Default `Fluxer`. The product name clients display. Also settable in the admin dashboard, which wins. Compose forwards it from `.env`. #### `FLUXER_APP_ICON_URL` -Default empty. The client icon. Its origin is added to the CSP by `app-proxy`. +Default empty. The client icon. Its origin is added to the CSP by `app-proxy`. Compose forwards it from `.env`. #### `FLUXER_APP_SYMBOL_URL` -Default empty. The symbol mark. Its origin is added to the CSP by `app-proxy`. +Default empty. The symbol mark. Its origin is added to the CSP by `app-proxy`. Compose forwards it from `.env`. #### `FLUXER_APP_LOGO_URL` -Default empty. The logo. Its origin is added to the CSP by `app-proxy`. +Default empty. The logo. Its origin is added to the CSP by `app-proxy`. Compose forwards it from `.env`. #### `FLUXER_APP_WORDMARK_URL` -Default empty. The wordmark. Its origin is added to the CSP by `app-proxy`. +Default empty. The wordmark. Its origin is added to the CSP by `app-proxy`. Compose forwards it from `.env`. #### `FLUXER_APP_FAVICON_URL` -Default empty. The favicon. Its origin is added to the CSP by `app-proxy`. +Default empty. The favicon. Its origin is added to the CSP by `app-proxy`. Compose forwards it from `.env`. #### `FLUXER_APP_THEME_COLOR` -Default empty. The theme colour. +Default empty. The theme colour. Compose forwards it from `.env`. #### `FLUXER_APP_STATUS_PAGE_URL` -Default empty. The public status page. The web app links to it when loading stalls, and polls its Instatus `summary.json` and `components.json` for incidents and maintenance. Empty removes the link and stops the polling. `app-proxy` does not add this origin to the CSP, so a status page off `fluxerstatus.com` also needs it in `FLUXER_CSP_EXTRA_CONNECT_SRC`. +Default empty. The public status page. The web app links to it when loading stalls, and polls its Instatus `summary.json` and `components.json` for incidents and maintenance. Empty removes the link and stops the polling. `app-proxy` does not add this origin to the CSP, so a status page off `fluxerstatus.com` also needs it in `FLUXER_CSP_EXTRA_CONNECT_SRC`. Compose forwards it from `.env`. #### `FLUXER_APP_STATUS_PAGE_INCIDENT_HISTORY_URL` -Default empty. The incident history page of the status page. +Default empty. The incident history page of the status page. Compose forwards it from `.env`. #### `FLUXER_INSTANCE_SETUP_CONFIGURED` -Default `false`. Whether setup is marked complete. The stored `app_public_config` row wins whenever its `setup.configured` is a boolean. This variable supplies the default only while no such row exists, and only on a self-hosted instance. Off a self-hosted instance the state is always true whatever this says. +Default `false`. Whether setup is marked complete. The stored `app_public_config` row wins whenever its `setup.configured` is a boolean. This variable supplies the default only while no such row exists, and only on a self-hosted instance. Off a self-hosted instance the state is always true whatever this says. Compose forwards it from `.env`. #### `FLUXER_AUTO_JOIN_INVITE_CODE` -Default empty. An invite every new account joins. Must be a live invite code. +Default empty. An invite every new account joins. Must be a live invite code. Compose forwards it from `.env`. #### `FLUXER_VISIONARIES_GUILD_ID` -Default empty. The guild that grants the visionary role. Snowflake. +Default empty. The guild that grants the visionary role. Snowflake. Compose forwards it from `.env`. #### `FLUXER_VISIONARIES_GUILD_VISIONARY_ROLE_ID` -Default empty. The role granted there. Snowflake. +Default empty. The role granted there. Snowflake. Compose forwards it from `.env`. Completing setup saves that state independently of the environment default. Later branding or legal updates preserve it. Setting `FLUXER_INSTANCE_SETUP_CONFIGURED` back to `false` does not reopen the wizard or restore the unauthenticated setup access and first-registration admin grant described in [Get started](/operator/get-started/). @@ -1242,27 +1230,27 @@ Completing setup saves that state independently of the environment default. Late #### `FLUXER_API_PORT` -Default `8080`. The API listen port. Compose sets `8080` and the edge proxies to it. +Default `8080`. The API listen port. Compose sets `8080`, which the edge proxies to. #### `FLUXER_API_HEADERS_TIMEOUT_MS` -Default `30000`. How long a client may take to send the request line and the headers. Integer 1000 to 3600000, checked at startup. It is clamped down to `FLUXER_API_REQUEST_TIMEOUT_MS`, so raising it alone does nothing. +Default `30000`. How long a client may take to send the request line and the headers. Integer 1000 to 3600000, checked at startup. It is clamped down to `FLUXER_API_REQUEST_TIMEOUT_MS`, so raising it alone does nothing. Compose forwards it from `.env`. #### `FLUXER_API_REQUEST_TIMEOUT_MS` -Default `120000`. How long a client may take over the whole request. Integer 1000 to 3600000, checked at startup. This is the one to raise for large uploads or high latency links. +Default `120000`. How long a client may take over the whole request. Integer 1000 to 3600000, checked at startup. This is the one to raise for large uploads or high latency links. Compose forwards it from `.env`. #### `FLUXER_API_WORKER_MODE` -Default `all_lanes`. Which lanes the worker runs. `all_lanes`, `single_lane`, or `single_task`. Anything else fails startup. +Default `all_lanes`. Which lanes the worker runs. `all_lanes`, `single_lane`, or `single_task`. Anything else fails startup. Compose sets `all_lanes`, because the stack runs one `worker`. #### `FLUXER_API_WORKER_LANE` -No default. Which lane, under `single_lane`. `realtime`, `unfurl`, `lifecycle`, or `batch`. +No default. Which lane, under `single_lane`. `realtime`, `unfurl`, `lifecycle`, or `batch`. Compose does not forward it. #### `FLUXER_API_WORKER_TASK` -No default. Which task, under `single_task`. Must name a known task. +No default. Which task, under `single_task`. Must name a known task. Compose does not forward it. #### `FLUXER_API_WORKER_ENABLE_CRON_SCHEDULER` @@ -1270,7 +1258,11 @@ No default. Whether the cron scheduler runs. Compose sets `true`. Without it no #### `FLUXER_API_WORKER_LANE_CONCURRENCY_OVERRIDES` -No default. Per-lane concurrency. JSON object keyed by lane. Each value must be an integer of at least 1 or startup fails. Built-in concurrency is realtime 10, unfurl 20, lifecycle 8, batch 12. +No default. Per-lane concurrency. JSON object keyed by lane. Each value must be an integer of at least 1 or startup fails. Built-in concurrency is realtime 10, unfurl 20, lifecycle 8, batch 12. Compose forwards it from `.env`. + +#### `FLUXER_NODE_EXTRA_CA_CERTS` + +Default `/etc/ssl/certs/ca-certificates.crt`, the system bundle in the image. A PEM file of extra CA certificates that `api` and `worker` trust for every outbound TLS connection, such as SMTP, object storage, search and webhooks. Node trusts its own built-in roots plus this one file. A file you set replaces the system bundle, so append your CA to a copy of that bundle, mount it into both containers and point this at it. Compose passes it to `api` and `worker` as `NODE_EXTRA_CA_CERTS`, and keeps the image path when `.env` leaves it out. It reads the prefixed name so a `NODE_EXTRA_CA_CERTS` in the host shell does not reach the containers. ## Media Proxy settings @@ -1278,169 +1270,179 @@ The Media Proxy takes uploads, transforms images, and serves media back. All are #### `FLUXER_MEDIA_PROXY_MODE` -Default `mp`. Which routes are served. `mp`, `static`, `upload`, or `relay`. `relay` serves the upload relay and the operator and internal endpoints, and returns 404 for every read. Anything else is a startup error. Also settable with `--mode`. Compose sets `upload`. +Default `mp`. Which routes are served. `mp`, `static`, `upload`, or `relay`. `relay` serves the upload relay and the operator and internal endpoints, and returns 404 for every read. Anything else is a startup error. Also settable with `--mode`. Compose sets `upload`, which the upload relay needs. #### `FLUXER_MEDIA_PROXY_HOST` -Default `0.0.0.0`. The bind address. Also settable with `--bind-host`. +Default `0.0.0.0`. The bind address. Also settable with `--bind-host`. Compose does not set it, so the default applies. #### `FLUXER_MEDIA_PROXY_PORT` -Default `8080`. The listen port. Also settable with `--port`. +Default `8080`. The listen port. Also settable with `--port`. Compose sets `8080`, which the edge proxies to. #### `FLUXER_MEDIA_PROXY_READ_ONLY` -Default `false`. Refuses writes. `--read-only` can force it on. +Default `false`. Refuses writes. `--read-only` can force it on. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_STORAGE_BACKEND` -Default `local`. Where objects live. `local` or `s3`. Compose sets `s3`. +Default `local`. Where objects live. `local` or `s3`. Compose sets `s3`, the bundled object store. #### `FLUXER_MEDIA_PROXY_STORAGE_ROOT` -Default `./media_proxy_storage`. The local storage directory. Used only by the local backend. +Default `./media_proxy_storage`. The local storage directory. Used only by the local backend, so Compose does not forward it. #### `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_MAX_BODY_BYTES` -Default `524288000`. The upload size cap on both sides. Rust accepts 1 byte to 5 GiB and refuses to start when the value is above `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_MAX_TOTAL_BYTES`. The effective cap is the smaller of the token's value and this. +Default `524288000`. The upload size cap on both sides. Rust accepts 1 byte to 5 GiB and refuses to start when the value is above `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_MAX_TOTAL_BYTES`. The effective cap is the smaller of the token's value and this. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_TOKEN_TTL_SECS` -Default `900`. Relay token lifetime. Read by the API alone. +Default `900`. Relay token lifetime. Read by the API alone. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT` -Default `http://localhost:8088/media`. The relay URL handed to clients. Node side only. A trailing slash and a trailing `/v1/relay` are stripped. +Default `http://localhost:8088/media`. The relay URL handed to clients. Node side only. A trailing slash and a trailing `/v1/relay` are stripped. Compose forwards it from `.env`, and builds it from the public origin and `/media` when it is unset. #### `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_KEEP_DIRECT_COUNTRIES` -Default empty. Countries that upload directly to S3. Node side only. Empty means every client uses the relay, which is what keeps the internal S3 address out of browsers. +Default empty. Countries that upload directly to S3. Node side only. Empty means every client uses the relay, which is what keeps the internal S3 address out of browsers. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_S3_TIMEOUT_MS` -Default `900000`. Relay upload timeout. Accepts 1000 to 3600000. +Default `900000`. Relay upload timeout. Accepts 1000 to 3600000. Compose forwards it from `.env`. + +#### `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_BYTES` + +Default `33554432`, 32 MiB. The largest declared upload the relay holds in memory so it can retry the S3 write. Accepts 0 to 268435456. A larger upload is streamed to S3 and gets no retry. Compose forwards it from `.env`. + +#### `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_TOTAL_BYTES` + +Default `536870912`, 512 MiB. The memory all buffered relay uploads may hold at once. Accepts 0 to 8589934592. An upload that would pass it is streamed instead. The default equals the `512mb` that `FLUXER_MEDIA_PROXY_MEMORY_LIMIT` gives `media-proxy`, so lower this or raise that limit when many uploads arrive at once. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_DIR` -Defaults to the system temporary directory. Where relay bodies spool. Must be writable. +Defaults to the system temporary directory. Where relay bodies spool. Must be writable. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_CHUNK_BYTES` -Default `1048576`. Spool chunk size. Accepts 64 KiB to 64 MiB. +Default `1048576`. Spool chunk size. Accepts 64 KiB to 64 MiB. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_MAX_TOTAL_BYTES` -Default `8589934592`. Total spool ceiling. Accepts 0 to 256 GiB, and must be at or above `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_MAX_BODY_BYTES`, which puts the usable floor at 500 MiB by default. +Default `8589934592`. Total spool ceiling. Accepts 0 to 256 GiB, and must be at or above `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_MAX_BODY_BYTES`, which puts the usable floor at 500 MiB by default. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_MAX_NATIVE_TRANSFORMS` -Defaults to available parallelism, clamped to 2 to 8. Concurrent image transforms. Accepts 1 to 128. +Defaults to available parallelism, clamped to 2 to 8. Concurrent image transforms. Accepts 1 to 128. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_WORKER_QUEUE_CAPACITY` -Defaults to eight times the transform limit. Transform queue depth. Accepts 1 to 8192. +Defaults to eight times the transform limit. Transform queue depth. Accepts 1 to 8192. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_BYTES` -Default `268435456`. Transform cache size. Accepts 0 to 4 GiB. +Default `268435456`. Transform cache size. Accepts 0 to 4 GiB. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_MAX_ENTRY_BYTES` -Default `67108864`. Largest cacheable result. Accepts 0 to 512 MiB. +Default `67108864`. Largest cacheable result. Accepts 0 to 512 MiB. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_TTL_MS` -Default `120000`. Transform cache lifetime. Accepts 0 to 3600000. +Default `120000`. Transform cache lifetime. Accepts 0 to 3600000. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_SOCKET_IO_TIMEOUT_MS` -Default `30000`. Socket read and write timeout. Accepts 0 to 300000. +Default `30000`. Socket read and write timeout. Accepts 0 to 300000. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_SHUTDOWN_GRACE_MS` -Default `30000`. Milliseconds allowed to finish requests and media transforms after SIGTERM or Ctrl-C. Accepts 0 to 300000. Exceeding the deadline exits the process with an error. +Default `30000`. Milliseconds allowed to finish requests and media transforms after SIGTERM or Ctrl-C. Accepts 0 to 300000. Exceeding the deadline exits the process with an error. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_TRANSFORM_TIMEOUT_MS` -Default `15000`. Per-transform timeout. Accepts 1000 to 120000. +Default `15000`. Per-transform timeout. Accepts 1000 to 120000. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_MAX_ENCODE_FRAMES` -Default `20000`. Animation frame ceiling. Accepts 1 to 100000. +Default `20000`. Animation frame ceiling. Accepts 1 to 100000. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_MAX_ENCODE_DURATION_MS` -Default `30000`. Animation duration ceiling. Accepts 100 to 600000. +Default `30000`. Animation duration ceiling. Accepts 100 to 600000. Compose forwards it from `.env`. #### `FLUXER_NSFW_SERVICE_ENDPOINT` -Default empty. An external NSFW classifier. No such service ships with the stack. +Default empty. An external NSFW classifier. No such service ships with the stack. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_NSFW_THRESHOLD` -Default `0.85`. The media-side NSFW cutoff. Accepts 0.0 to 1.0 and finite. Distinct from the API threshold. +Default `0.85`. The media-side NSFW cutoff. Accepts 0.0 to 1.0 and finite. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_CORS_MODE` -Default `off`. The [cross-origin read policy](/media-proxy/overview/#cross-origin-reads). `off`, `report`, or `enforce`. Anything else is a startup error. `report` logs each read whose origin would be refused and changes no response. `enforce` refuses those reads with 403 and sets `Access-Control-Allow-Origin` to the request origin. When `FLUXER_MEDIA_PROXY_MODE` is `static` or `relay`, the Media Proxy ignores it, so a bad value is no startup error. Compose leaves it `off`. +Default `off`. The [cross-origin read policy](/media-proxy/overview/#cross-origin-reads). `off`, `report`, or `enforce`. Anything else is a startup error. `report` logs each read whose origin would be refused and changes no response. `enforce` refuses those reads with 403 and sets `Access-Control-Allow-Origin` to the request origin. When `FLUXER_MEDIA_PROXY_MODE` is `static` or `relay`, the Media Proxy ignores it, so a bad value is no startup error. Compose forwards it from `.env`. #### `FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS` Default empty. Comma-separated origins that may read media through CORS, such as `https://web.fluxer.app`. Spaces around an entry and empty entries are ignored. Each entry is parsed as a URL and must be `http` or `https` with no credentials, query, fragment, or path beyond `/`. Its host must be an IPv4 address, a bracketed IPv6 address, or a domain name whose labels are ASCII letters, digits, and hyphens after parsing. So a wildcard such as `https://*.fluxer.app`, a brace pattern, an underscore, a trailing dot on a domain name, or any non-ASCII character fails. Write an internationalised domain name in its punycode form. Any failing entry makes the process exit. An entry is compared in the form a browser sends. `https://Web.Fluxer.App:443/` matches `https://web.fluxer.app`. Required when `FLUXER_MEDIA_PROXY_CORS_MODE` is `report` or `enforce`. When the policy is `report` or `enforce`, the Media Proxy logs the parsed list once at startup. When `FLUXER_MEDIA_PROXY_MODE` is `static` or `relay`, the Media Proxy ignores it. -Compose sets it to the public origin, built from `FLUXER_PUBLIC_ORIGIN`, or from `FLUXER_PUBLIC_SCHEME`, `FLUXER_DOMAIN`, and `FLUXER_PUBLIC_PORT`. That origin is the only entry, so the web app the instance serves reads its own media. The hosted web client and the desktop app run on `https://web.fluxer.app`. To let them read this instance's media through CORS, set the list in `.env` to the public origin and `https://web.fluxer.app`, such as `https://chat.example.com,https://web.fluxer.app`. A value set in `.env` replaces the Compose value, so it must name the public origin too. +Compose forwards it from `.env`, and defaults it to the public origin, built from `FLUXER_PUBLIC_ORIGIN`, or from `FLUXER_PUBLIC_SCHEME`, `FLUXER_DOMAIN`, and `FLUXER_PUBLIC_PORT`. That origin is the only entry, so the web app the instance serves reads its own media. The hosted web client and the desktop app run on `https://web.fluxer.app`. To let them read this instance's media through CORS, set the list in `.env` to the public origin and `https://web.fluxer.app`, such as `https://chat.example.com,https://web.fluxer.app`. A value set in `.env` replaces the Compose value, so it must name the public origin too. #### `FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE` -Default `off`. The [signed attachment URL policy](/media-proxy/overview/#signed-attachment-urls). `off`, `report`, or `enforce`. Anything else is a startup error, an empty value included. `report` logs each attachment read `enforce` would refuse and serves it. `enforce` refuses those reads with 404. Under both, a read whose signature is valid gets a `Cache-Control` that ends with the signature instead of the usual year. `report` and `enforce` require `FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64`, and the Media Proxy logs the mode and the number of secrets once at startup. When `FLUXER_MEDIA_PROXY_MODE` is `static` or `relay`, the Media Proxy ignores it and the secrets, so a bad value is no startup error. Compose leaves it `off`, so a self-hosted instance signs nothing until an operator turns it on. +Default `off`. The [signed attachment URL policy](/media-proxy/overview/#signed-attachment-urls). `off`, `report`, or `enforce`. An empty value means `off`, and anything else is a startup error. `report` logs each attachment read `enforce` would refuse and serves it. `enforce` refuses those reads with 404. Under both, a read whose signature is valid gets a `Cache-Control` that ends with the signature instead of the usual year. `report` and `enforce` require `FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64`, and the Media Proxy logs the mode and the number of secrets once at startup. When `FLUXER_MEDIA_PROXY_MODE` is `static` or `relay`, the Media Proxy ignores it and the secrets, so a bad value is no startup error. A self-hosted instance leaves it `off`, and signs nothing, until an operator turns it on. Compose forwards it from `.env`. This variable is the whole switch, and moving between the three states needs nothing else changed. The Media Proxy reads it once at start, so apply a change with `docker compose up -d media-proxy`, which replaces the container. `docker compose restart media-proxy` reuses the old environment. A cache or proxy in front of the instance has a switch of its own, and [Turning the signature policy on](/operator/reverse-proxy/#turning-the-signature-policy-on) gives the order to move the two in and what each pairing serves. -`media-proxy` range-checks these values at startup and then reads them nowhere: `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_BYTES` and `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_TOTAL_BYTES`. A bad value still fails the boot. - ## Gateway settings The Gateway is the WebSocket service clients hold open for live events. All are optional. #### `FLUXER_GATEWAY_PORT` -Default `8771`. The listen port. Compose sets `8080`. +Default `8771`. The listen port. Compose sets `8080`, which the edge proxies to. #### `FLUXER_GATEWAY_ROLE` -Default `all`. Which subsystems this node runs. `websocket`, `sessions`, `presence`, `guilds`, `calls`, `push`, or `all`. An unrecognised value also becomes `all`. +Default `all`. Which subsystems this node runs. `websocket`, `sessions`, `presence`, `guilds`, `calls`, `push`, or `all`. An unrecognised value also becomes `all`. Compose does not forward it, because the stack runs one Gateway node. #### `FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT` -Default `http://localhost:8088/media`. The public media URL used in payloads. Compose builds it from the public origin. +Default `http://localhost:8088/media`. The public media URL used in payloads. Compose forwards it from `.env`, and uses `FLUXER_MEDIA_ENDPOINT` or the public media URL when it is unset. #### `FLUXER_GATEWAY_STATIC_CDN_ENDPOINT` -Default `http://localhost:8088`. The public static origin used in payloads. Compose builds it from the public origin. +Default `http://localhost:8088`. The public static origin used in payloads. Compose forwards it from `.env`, and uses `FLUXER_STATIC_CDN_ENDPOINT` or the public origin when it is unset. #### `FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_ENTRIES` -Default `128`. Presence push buffer depth. +Default `128`. Presence push buffer depth. Compose forwards it from `.env`. #### `FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_BYTES` -Default `1048576`. Presence push buffer size. +Default `1048576`. Presence push buffer size. Compose forwards it from `.env`. -#### `FLUXER_GATEWAY_SHUTDOWN_DRAIN_WAIT_MS` +#### `FLUXER_GATEWAY_PUSH_ENROLLED_CLEAR_NOTIFICATIONS_ENABLED` -Default `5000`. Milliseconds. The Gateway parses it into its configuration at startup, and nothing reads it after that, so it controls no drain. +Default `true`. Whether the Gateway asks `push` to clear a channel's notifications once the channel is read. Compose forwards it from `.env`. + +#### `FLUXER_GATEWAY_PUSH_OUTBOX_REQUEST_TIMEOUT_MS` + +Default `100000`. How long the Gateway waits on one push outbox request. Milliseconds. Compose forwards it from `.env`. #### `FLUXER_GATEWAY_HTTP_FAILURE_THRESHOLD` -Default `6`. Failures before the API circuit opens. Integer. +Default `6`. Failures before the API circuit opens. Integer. Compose forwards it from `.env`. #### `FLUXER_GATEWAY_HTTP_RECOVERY_TIMEOUT_MS` -Default `15000`. How long the circuit stays open. Milliseconds. +Default `15000`. How long the circuit stays open. Milliseconds. Compose forwards it from `.env`. #### `FLUXER_GATEWAY_CLUSTER_ENABLED` -Default `false`. BEAM clustering. Single-node by default. +Default `false`. BEAM clustering. Single-node by default. Compose forwards none of the clustering names, because the stack runs one Gateway node. #### `FLUXER_GATEWAY_CLUSTER_DISCOVERY_DNS_NAME` @@ -1460,7 +1462,7 @@ Default empty. A fixed peer list. Comma separated Erlang node names, capped at 2 #### `FLUXER_ERLANG_NODE_NAME` -Default `fluxer_gateway@127.0.0.1`. The BEAM node name. Must be resolvable by peers when clustering. +Default `fluxer_gateway@127.0.0.1`. The BEAM node name. Must be resolvable by peers when clustering. Compose does not forward it. #### `FLUXER_ERLANG_COOKIE` @@ -1468,31 +1470,31 @@ No default. The BEAM distribution secret. The Gateway refuses to start without i #### `FLUXER_ERLANG_DIST_PORT` -Default `8081`. The BEAM distribution port. Not published by Compose. Never expose it. +Default `8081`. The BEAM distribution port. Not published or forwarded by Compose. Never expose it. #### `FLUXER_ERLANG_SCHEDULERS` -Defaults to the container CPU count, clamped to 2 through 16. Normal scheduler count. Positive integer, passed to the BEAM as `+S N:N`. A value that is not a positive integer is ignored and the derivation runs instead. +Defaults to the container CPU count, clamped to 2 through 16. Normal scheduler count. Positive integer, passed to the BEAM as `+S N:N`. A value that is not a positive integer is ignored and the derivation runs instead. Compose forwards it from `.env` to `gateway`. #### `FLUXER_ERLANG_DIRTY_CPU_SCHEDULERS` -Defaults to two thirds of the scheduler count, rounded up. Dirty CPU scheduler count. Positive integer, passed as `+SDcpu N:N`. Same fallback rule as `FLUXER_ERLANG_SCHEDULERS`. +Defaults to two thirds of the scheduler count, rounded up. Dirty CPU scheduler count. Positive integer, passed as `+SDcpu N:N`. Same fallback rule as `FLUXER_ERLANG_SCHEDULERS`. Compose forwards it from `.env` to `gateway`. #### `FLUXER_ERLANG_SCHEDULERS_MIN` -Default `2`. The floor of the scheduler clamp. Compose forwards it to `gateway`. +Default `2`. The floor of the scheduler clamp. Compose forwards it from `.env` to `gateway`. #### `FLUXER_ERLANG_SCHEDULERS_MAX` -Default `16`. The ceiling of the scheduler clamp. Compose forwards it to `gateway`, and `.env.example` ships both names commented out. +Default `16`. The ceiling of the scheduler clamp. Compose forwards it from `.env` to `gateway`, and `.env.example` ships both names commented out. The Gateway entrypoint derives the scheduler counts before the BEAM starts. It reads the container CPU quota, clamps the result between `FLUXER_ERLANG_SCHEDULERS_MIN` and `FLUXER_ERLANG_SCHEDULERS_MAX`, exports the answer as `FLUXER_ERLANG_SCHEDULERS`, and derives `FLUXER_ERLANG_DIRTY_CPU_SCHEDULERS` from it. `vm.args.src` then substitutes both into `+S` and `+SDcpu`. -The clamp bounds reach the Gateway from `.env`. To pin the count, set `FLUXER_ERLANG_SCHEDULERS` on the `gateway` service in `docker-compose.yml`, which skips the clamp. +The clamp bounds reach the Gateway from `.env`. To pin the count, set `FLUXER_ERLANG_SCHEDULERS` in `.env`, which skips the clamp. The Gateway protocol version is `1`. The Gateway answers any other value in `?v=` with 101, then a close frame reading `Invalid API version`. [Gateway overview](/gateway/overview/) has the close code. -The Gateway reads every `FLUXER_GATEWAY_` name straight from the environment, so any of them works once it is in the `gateway` container environment. The bundled Compose file passes only the names it lists, so add any other name through a Compose override file. +The Gateway reads every `FLUXER_GATEWAY_` name straight from the environment. Compose forwards each one a single-node stack can use. The role, the clustering names and `FLUXER_GATEWAY_API_RPC_ENDPOINT` need a Compose override file. ## App proxy settings @@ -1500,19 +1502,19 @@ The Gateway reads every `FLUXER_GATEWAY_` name straight from the environment, so #### `FLUXER_APP_PROXY_HOST` -Default `0.0.0.0`. The bind address. Compose sets it explicitly. +Default `0.0.0.0`. The bind address. The image sets the same value, and Compose does not set it. #### `FLUXER_APP_PROXY_PORT` -Default `8080`. The listen port. Compose sets it explicitly. +Default `8080`. The listen port. Compose sets `8080`, which the edge proxies to. #### `FLUXER_STATIC_DIR` -Default `./static`. Where the client's SPA bundle lives, unrelated to `static-proxy`. +Default `./static`. Where the client's SPA bundle lives, unrelated to `static-proxy`. A path inside the image, so Compose does not forward it. #### `FLUXER_APP_PROXY_INDEX_UPSTREAM_URL` -No default. An upstream to fetch `index.html` from. Leave unset for the shipped image. +No default. An upstream to fetch `index.html` from. Leave unset for the shipped image. Compose forwards it from `.env`. #### `DISCOVERY_UPSTREAM_URL` @@ -1520,51 +1522,51 @@ Default `http://localhost:8088/api/.well-known/fluxer`. Where the bootstrap disc #### `DISCOVERY_REFRESH_INTERVAL_MS` -Default `60000`. How often discovery is refetched. Unprefixed name. +Default `60000`. How often discovery is refetched. Unprefixed name. Compose forwards it from `.env`. #### `PUBLIC_BOOTSTRAP_API_ENDPOINT` -Default `/api`. The API path put in the page bootstrap. +Default `/api`. The API path put in the page bootstrap. Compose sets `/api`, the path the edge routes to the API. #### `PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT` -No default. The absolute API URL in the bootstrap. Compose builds it from the public origin. +No default. The absolute API URL in the bootstrap. Compose forwards it from `.env`, and builds it from the public origin when it is unset. #### `FLUXER_APP_PROXY_SAME_ORIGIN_HOSTS` -Default empty. Hostnames, comma separated, without a scheme, port or path. When a request's `Host` is in the list, the page bootstrap points the web app and its API at that host, as `https://` and `https:///api`. Every other request gets the discovery values unchanged. Set it when the web app is served on more than one hostname and each hostname routes `/api` to the API. Invalid entries are logged and dropped. +Default empty. Hostnames, comma separated, without a scheme, port or path. When a request's `Host` is in the list, the page bootstrap points the web app and its API at that host, as `https://` and `https:///api`. Every other request gets the discovery values unchanged. Set it when the web app is served on more than one hostname and each hostname routes `/api` to the API. Invalid entries are logged and dropped. Compose forwards it from `.env`. #### `FLUXER_APP_PROXY_MANIFEST_SCOPE_EXTENSIONS` -Default empty. HTTPS origins, comma separated, without a path, query or credentials. `app-proxy` writes each one into the `scope_extensions` list of the web app manifest, so an installed web app keeps other origins inside its window. Empty leaves the list empty. Each listed origin must serve `/.well-known/web-app-origin-association` naming this web app, or browsers ignore the entry. Invalid entries are logged and dropped. +Default empty. HTTPS origins, comma separated, without a path, query or credentials. `app-proxy` writes each one into the `scope_extensions` list of the web app manifest, so an installed web app keeps other origins inside its window. Empty leaves the list empty. Each listed origin must serve `/.well-known/web-app-origin-association` naming this web app, or browsers ignore the entry. Invalid entries are logged and dropped. Compose forwards it from `.env`. ## Admin settings -`admin` serves the dashboard at `/admin`. All are optional. +`admin` serves the dashboard at `FLUXER_ADMIN_BASE_PATH`, `/admin` by default. All are optional. #### `FLUXER_ADMIN_HOST` -Default `0.0.0.0`. The bind address. Compose sets it explicitly. +Default `0.0.0.0`. The bind address. Compose does not set it, so the default applies. #### `FLUXER_ADMIN_PORT` -Default `3020`. The listen port. The image sets 8080 and Compose sets 8080. +Default `3020`. The listen port. The image sets 8080 and Compose sets 8080, which the edge proxies to. #### `FLUXER_ADMIN_BASE_PATH` -Defaults to empty in Rust, `/admin` in Node. The path the dashboard is mounted under. Normalised to a leading slash with no trailing slash. The service serves at root and re-prefixes every emitted URL with this. +Default empty. The path the dashboard is mounted under. Write it with a leading slash and no trailing slash, such as `/panel`. The edge and the Compose-built `FLUXER_ADMIN_ENDPOINT` use the value as written, and only `admin` itself normalises it. The service serves at root and re-prefixes every emitted URL with this. Read by `admin` and the edge. Compose forwards it from `.env` with a default of `/admin`, hands the same value to the edge, which routes that path to `admin`, and builds the default `FLUXER_ADMIN_ENDPOINT` from it. A change moves the admin OAuth redirect with it, and takes `docker compose up -d`. #### `FLUXER_ADMIN_OAUTH_REDIRECT_URI` -Defaults to the admin endpoint plus `/oauth2_callback`. The OAuth2 redirect. Must equal what the API derives from `FLUXER_ADMIN_ENDPOINT`. The API does not read this name. +Defaults to the admin endpoint plus `/oauth2_callback`. The OAuth2 redirect. Must equal what the API derives from `FLUXER_ADMIN_ENDPOINT`, so Compose does not set it. The API does not read this name. -#### `FLUXER_RELEASE_CHANNEL` +#### `FLUXER_ADMIN_OAUTH_CLIENT_ID` -Default `stable`. The reported channel. `RELEASE_CHANNEL` is preferred over it. +Default `1234567890123456789`. The OAuth2 client id of `admin`. Must equal the admin application id built into the API, so Compose does not set it. #### `FLUXER_BUILD_VERSION` -Defaults to the crate version. The reported build. `BUILD_VERSION` is preferred over it. +Defaults to the crate version. The reported build of `admin` and `app-proxy`. `BUILD_VERSION` is preferred over it. Compose does not set it. ## Content Security Policy @@ -1592,15 +1594,15 @@ Read by Docker Compose to select the proxy overlay. #### `FLUXER_EDGE_SITE_ADDRESS` -Read by the edge container only, and only in the bundled layout. The overlay overwrites it with `:8080`. +Read by the edge container only, and only in the bundled layout. Both overlays overwrite it, `docker-compose.proxy.yml` with `:8080` and `tunnel.compose.yml` with `:80`. #### `FLUXER_CADDY_SITE_ADDRESS` Read by Docker Compose as the default for `FLUXER_EDGE_SITE_ADDRESS` when that name is unset. -#### `FLUXER_EDGE_TRUSTED_PROXIES` +#### `FLUXER_EDGE_TRUSTED_PROXIES` and `FLUXER_EDGE_ENCODE` -Read by the edge container only. It takes effect on `docker compose up -d edge`. +Read by the edge container only. They take effect on `docker compose up -d edge`. #### `FLUXER_EDGE_BIND` @@ -1614,6 +1616,10 @@ Used as the host side of the edge's published ports. Image name selection. +#### The third-party image names + +Image name selection for the bundled services, listed under [Images](#images). + #### `POSTGRES_PASSWORD` Becomes `FLUXER_POSTGRES_PASSWORD` and the Postgres image's own password. @@ -1624,43 +1630,71 @@ Becomes `FLUXER_SEARCH_API_KEY` and the Meilisearch image's own key. #### `FLUXER_S3_ACCESS_KEY` and `FLUXER_S3_SECRET_KEY` -Become `FLUXER_S3_ACCESS_KEY_ID` and `FLUXER_S3_SECRET_ACCESS_KEY`, and the `AWS_` pair. +Become `FLUXER_S3_ACCESS_KEY_ID` and `FLUXER_S3_SECRET_ACCESS_KEY`, and the identity `seaweedfs-init` installs. #### `LIVEKIT_API_KEY` and `LIVEKIT_API_SECRET` Become `FLUXER_LIVEKIT_API_KEY` and `FLUXER_LIVEKIT_API_SECRET`, and LiveKit's own `LIVEKIT_KEYS`. +#### The extra CA bundle + +`FLUXER_NODE_EXTRA_CA_CERTS` becomes `NODE_EXTRA_CA_CERTS` in `api` and `worker`. + +#### The LiveKit configuration names + +`FLUXER_LIVEKIT_LOG_LEVEL`, `FLUXER_LIVEKIT_TCP_PORT`, `FLUXER_LIVEKIT_UDP_PORT`, `FLUXER_LIVEKIT_USE_EXTERNAL_IP`, `FLUXER_LIVEKIT_NODE_IP`, `FLUXER_LIVEKIT_STUN_PRIMARY` and `FLUXER_LIVEKIT_STUN_SECONDARY` are written into `LIVEKIT_CONFIG`, and the two ports also into the published port mappings. + +#### The per-service pool sizes + +`FLUXER_API_POSTGRES_MAX_CONNECTIONS`, `FLUXER_WORKER_POSTGRES_MAX_CONNECTIONS`, `FLUXER_USERS_SHARD_POSTGRES_MAX_CONNECTIONS` and `FLUXER_MESSAGES_SHARD_POSTGRES_MAX_CONNECTIONS` each become `FLUXER_POSTGRES_MAX_CONNECTIONS` in their own service. + +#### Memory, tuning, health checks and restarts + +The names under [Resources](#resources) and [Health checks and restarts](#health-checks-and-restarts) are read by Docker Compose, which hands them to the engine or to the bundled services under their own names. + ## Keys Compose does not forward -`.env.example` lists the variables forwarded by `docker-compose.yml`. Add any setting below through a Compose override file that puts it in the relevant service's environment. +Every other name on this page takes effect from `.env`, except the names Compose fills from another name, listed first below. The other names below stay out of it for the reason given. Add one through a Compose override file that puts it in the relevant service's environment, which [Keep a local compose change](/operator/upgrading/#keep-a-local-compose-change) describes. -#### `FLUXER_AUTH_BLUESKY_` and the names under it +#### Names Compose fills from another name -See [Bluesky connections](#bluesky-connections) for configuration. +Compose writes each of these from the `.env` name given, so setting the name itself in `.env` has no effect. `FLUXER_POSTGRES_PASSWORD` comes from `POSTGRES_PASSWORD`. `FLUXER_POSTGRES_MAX_CONNECTIONS` comes from `FLUXER_API_POSTGRES_MAX_CONNECTIONS`, `FLUXER_WORKER_POSTGRES_MAX_CONNECTIONS`, `FLUXER_USERS_SHARD_POSTGRES_MAX_CONNECTIONS` or `FLUXER_MESSAGES_SHARD_POSTGRES_MAX_CONNECTIONS`, one per service. `FLUXER_S3_ACCESS_KEY_ID` comes from `FLUXER_S3_ACCESS_KEY`, and `FLUXER_S3_SECRET_ACCESS_KEY` from `FLUXER_S3_SECRET_KEY`. `FLUXER_SEARCH_API_KEY` comes from `MEILI_MASTER_KEY`. `FLUXER_LIVEKIT_API_KEY` comes from `LIVEKIT_API_KEY`, and `FLUXER_LIVEKIT_API_SECRET` from `LIVEKIT_API_SECRET`. `FLUXER_BASE_DOMAIN` comes from `FLUXER_DOMAIN`. -#### `FLUXER_PUSH_APNS_` and `FLUXER_PUSH_FCM_` +#### Fixed wiring -Mobile push cannot be configured at all from the example. `api` and `push` both read these names. An override has to reach both. +`FLUXER_ENV`, `FLUXER_SELF_HOSTED`, `FLUXER_DATABASE_BACKEND`, the listen ports `FLUXER_API_PORT`, `FLUXER_GATEWAY_PORT`, `FLUXER_MEDIA_PROXY_PORT`, `FLUXER_APP_PROXY_PORT` and `FLUXER_ADMIN_PORT`, the internal endpoints `FLUXER_INTERNAL_API_ENDPOINT`, `FLUXER_INTERNAL_MEDIA_PROXY_ENDPOINT` and `FLUXER_MEDIA_PROXY_ENDPOINT`, the `FLUXER_API_ENDPOINT` of `admin`, `FLUXER_SVC_NAME`, `FLUXER_SVC_MODE`, `FLUXER_SVC_SHARD_ID`, `FLUXER_SVC_SHARD_COUNT`, `FLUXER_API_WORKER_MODE`, `FLUXER_API_WORKER_ENABLE_CRON_SCHEDULER`, `FLUXER_MEDIA_PROXY_MODE`, `FLUXER_MEDIA_PROXY_STORAGE_BACKEND`, `DISCOVERY_UPSTREAM_URL` and `PUBLIC_BOOTSTRAP_API_ENDPOINT`. Compose sets each to a fixed value that the edge, the health checks or another service depend on. A different value gives no working stack. -#### `RUST_LOG`, `LOG_LEVEL` and `LOGGER_LEVEL` +#### Names the stack has no use for -The only way to change log verbosity. +`FLUXER_MEDIA_PROXY_HOST`, `FLUXER_APP_PROXY_HOST`, `FLUXER_ADMIN_HOST` and `FLUXER_PUSH_SERVICE_HOST`, whose default already listens on every interface. `FLUXER_PUSH_SERVICE_PORT`, whose default the health check of `push` reads. `FLUXER_SVC_LISTEN_HOST` and `FLUXER_SVC_PORT`, which the health checks address. `FLUXER_API_WORKER_LANE` and `FLUXER_API_WORKER_TASK`, which only the single-lane worker modes read. `FLUXER_MEDIA_PROXY_STORAGE_ROOT`, which only the local backend reads. `FLUXER_S3_BUCKET_STATIC`, which only the `static` mode of `media-proxy` reads. `FLUXER_GATEWAY_ROLE`, the `FLUXER_GATEWAY_CLUSTER_` names, `FLUXER_ERLANG_NODE_NAME` and `FLUXER_ERLANG_DIST_PORT`, since the stack runs one Gateway node. `FLUXER_GATEWAY_API_RPC_ENDPOINT`, which the Gateway derives from the internal API address. `FLUXER_SNOWFLAKE_SERVICE_SUBJECT`, `FLUXER_USERS_SERVICE_SUBJECT` and `FLUXER_GIF_SERVICE_SUBJECT`, which have to match the subjects the internal services listen on. `FLUXER_ADMIN_OAUTH_REDIRECT_URI`, which has to match what the API derives. `FLUXER_ADMIN_OAUTH_CLIENT_ID`, which has to match the admin application id built into the API. The relay-mode names of `push`, `FLUXER_PUSH_RELAY_MAX_CONCURRENT`, `FLUXER_PUSH_RELAY_MAX_BODY_BYTES`, `FLUXER_PUSH_RELAY_TRUSTED_PROXY_HOPS`, `FLUXER_PUSH_RELAY_SOURCE_BUCKET_ENABLED` and the `_ENTRIES`, `_PER_MINUTE` and `_BURST` names under `FLUXER_PUSH_RELAY_TOKEN_BUCKET` and `FLUXER_PUSH_RELAY_SOURCE_BUCKET`, since the stack runs `push` in delivery mode. `FLUXER_STATIC_DIR`, a path inside the image. `FLUXER_DOCS_ENDPOINT`, which only the docs server reads. -#### `FLUXER_APP_PRODUCT_NAME` and the branding URLs +#### Allocator and image library tuning -Branding is otherwise admin-dashboard only. +`LD_PRELOAD` and `MALLOC_CONF` in `media-proxy` and `app-proxy`, and `VIPS_CONCURRENCY`, `VIPS_DISC_THRESHOLD` and `G_MESSAGES_DEBUG` in `media-proxy`. Each image sets them. `LD_PRELOAD` loads jemalloc and `MALLOC_CONF` tunes it. `VIPS_CONCURRENCY` stays at `1` on purpose, and `FLUXER_MEDIA_PROXY_MAX_NATIVE_TRANSFORMS` is the setting for image work in parallel. `VIPS_DISC_THRESHOLD`, `1g` in the image, is the image size above which libvips decodes to disk instead of memory. Lower it when a small `FLUXER_MEDIA_PROXY_MEMORY_LIMIT` runs out of memory. -#### `FLUXER_INSTANCE_SETUP_CONFIGURED` +#### Build and platform metadata -Supplies the initial setup state on a self-hosted instance, until the first write of the stored `app_public_config` row takes over. +`BUILD_VERSION`, `FLUXER_BUILD_VERSION` and `RELEASE_CHANNEL` describe the shipped build. `POD_NAME` and `HOSTNAME` come from the platform. -#### Every Media Proxy performance knob +#### Test and development switches -No example coverage at all. +`FLUXER_TEST_MODE_ENABLED`, `FLUXER_TEST_HARNESS_TOKEN`, `FLUXER_DISABLE_RATE_LIMITS`, `FLUXER_VALIDATE_RESPONSES` and `FLUXER_RELAX_REGISTRATION_RATE_LIMITS`. None of them belongs on a production instance. + +#### The Cassandra names + +Every `FLUXER_CASSANDRA_` name. The stack runs Postgres. + +#### Older names of a forwarded setting + +`FLUXER_NATS_CORE_URL`, `FLUXER_RISK_IPINFO_API_KEY`, `KLIPY_API_KEY` and `YOUTUBE_API_KEY`. A service reads each only when the name it replaces is unset or empty, and Compose forwards that name instead. + +#### Individual Stripe price names + +The `FLUXER_STRIPE_PRICE_MONTHLY_`, `FLUXER_STRIPE_PRICE_YEARLY_`, `FLUXER_STRIPE_PRICE_GIFT_1_MONTH_` and `FLUXER_STRIPE_PRICE_GIFT_1_YEAR_` names, one per currency. `FLUXER_STRIPE_PRICES` holds the same prices in one JSON object, and the admin dashboard can replace them. ## Runtime settings in the admin dashboard -The admin dashboard at `/admin` stores these in the database. They apply without recreating containers, and a value set here wins over the matching environment variable. +The admin dashboard at `FLUXER_ADMIN_BASE_PATH` stores these in the database. They apply without recreating containers, and a value set here wins over the matching environment variable. #### Instance Config, Public App Identity @@ -1742,9 +1776,11 @@ The stack runs its containers on one Docker bridge network, which is private to | seaweedfs-init | chrislusf/seaweedfs:4.47 | Creates the buckets and the S3 identity, then exits | | livekit | livekit/livekit-server:v1.12.0 | Voice and video | +The third-party images above are the defaults of the names under [Images](#images). + The edge and LiveKit are the only services that publish ports. The edge publishes 80/tcp, 443/tcp, 443/udp, or one plain-HTTP port under the overlay. LiveKit publishes 7881/tcp and 7882/udp. Everything else is reachable only over the bridge network. -`api` is the one service an operator configures directly, through the shared environment block. `worker`, `gateway`, `app-proxy`, `media-proxy`, and `push` are touched rarely, `worker` for lane concurrency, `app-proxy` for CSP extras, `media-proxy` for transform limits, and `push` for queue capacity and send concurrency. The edge takes only the `FLUXER_EDGE_` variables, `postgres` only the password, `meilisearch` only the master key, `valkey` only the `FLUXER_VALKEY_` tuning values, and `livekit` only the key pair and the port variables. `static-proxy` reads no environment variables, and the remaining services need none. +Most settings reach `api`, `worker`, `gateway`, `media-proxy`, `push`, `admin` and the internal services through one shared environment block, so one line in `.env` reaches every one of them that reads it. `app-proxy` takes its own list, since it reads the public address, the client IP settings, the GeoIP database and its own names. The edge takes the `FLUXER_EDGE_` variables and `FLUXER_ADMIN_BASE_PATH`. The third-party services take their image name, their memory limit, and their own tuning: `postgres` the password, database, role and `FLUXER_POSTGRES_` server settings, `meilisearch` the master key and `FLUXER_MEILISEARCH_` values, `valkey` the `FLUXER_VALKEY_` values, `seaweedfs` its heap ceiling and telemetry switch, and `livekit` the key pair, the port, STUN and log level variables. `static-proxy` reads no environment variables. The internal services each run a router, which takes requests and holds no state, and one shard, which holds the caches and the database connections. The shipped stack fixes `FLUXER_SVC_SHARD_COUNT` at `1`. @@ -1782,6 +1818,14 @@ Default `768mb`. The ceiling for `meilisearch`. Must stay well above `FLUXER_MEI Default `384mb`. Becomes `MEILI_MAX_INDEXING_MEMORY`. The memory the indexer may use for one batch. Lower it whenever you lower `FLUXER_MEILISEARCH_MEMORY_LIMIT`. +#### `FLUXER_MEILISEARCH_ENV` + +Default `production`. Becomes `MEILI_ENV`. `production` requires the master key on every request. + +#### `FLUXER_MEILISEARCH_NO_ANALYTICS` + +Default `true`. Becomes `MEILI_NO_ANALYTICS`, which keeps Meilisearch from sending usage data. + #### `FLUXER_SEAWEEDFS_MEMORY_LIMIT` Default `2gb`. The ceiling for `seaweedfs`. One process runs the master, the volume server and the S3 gateway. The peak is the parts of one upload in flight at once, which the client uploads in parallel: a 500 MB attachment is 20 parts of 25 MB. @@ -1790,6 +1834,10 @@ Default `2gb`. The ceiling for `seaweedfs`. One process runs the master, the vol Default `1536MiB`. The heap ceiling the Go runtime collects against. Go cannot see the container limit, so without this value an upload burst grows the heap past `FLUXER_SEAWEEDFS_MEMORY_LIMIT` and the kernel OOM-kills the container mid-upload with exit 137. Raising `FLUXER_SEAWEEDFS_MEMORY_LIMIT` on its own does not fix that, because the runtime grows to fill whatever it is given. Keep this near three quarters of the container limit and move the two together. +#### `FLUXER_SEAWEEDFS_TELEMETRY` + +Default `false`. Becomes the `-master.telemetry` flag of `seaweedfs`, which reports usage to the SeaweedFS project when on. + #### `FLUXER_SEAWEEDFS_INIT_MEMORY_LIMIT` Default `128mb`. The ceiling for `seaweedfs-init`. A one-shot container that exits, so it never overlaps steady state. @@ -1898,6 +1946,14 @@ No default. The V8 old-space ceiling for `api`, in MB. Appended to `NODE_OPTIONS No default. The V8 old-space ceiling for `worker`, in MB. Same rule against `FLUXER_WORKER_MEMORY_LIMIT`. +#### `FLUXER_API_NODE_OPTIONS` + +No default. Extra Node flags for `api`, such as `--dns-result-order=ipv4first`. Compose reads it from `.env` and appends it to `NODE_OPTIONS` after `--enable-source-maps` and the heap flag, only when non-empty. + +#### `FLUXER_WORKER_NODE_OPTIONS` + +No default. Extra Node flags for `worker`. Same rule as `FLUXER_API_NODE_OPTIONS`. + The names on the Postgres command line tune the bundled server. Keep them consistent with `FLUXER_POSTGRES_MEMORY_LIMIT`. All are optional. #### `FLUXER_POSTGRES_SERVER_MAX_CONNECTIONS` @@ -1928,9 +1984,28 @@ Default `128MB`. The budget for each autovacuum worker. Postgres runs three work Default `1gb`. The shared memory of the `postgres` container, used by parallel queries and parallel maintenance. Keep it well above `FLUXER_POSTGRES_MAINTENANCE_WORK_MEM`. A manual VACUUM sizes its shared memory from that budget, and fails with `could not resize shared memory segment` when it does not fit. -The WAL settings `min_wal_size=512MB`, `max_wal_size=2GB` and `wal_buffers=16MB` are fixed on the command line, with no variable of their own. +The rest of the Postgres command line has one variable per setting. -The bundled Valkey uses persistent storage. These settings control its memory limit and behaviour when full. +- `FLUXER_POSTGRES_RANDOM_PAGE_COST`: default `1.1`, sets `random_page_cost`. +- `FLUXER_POSTGRES_EFFECTIVE_IO_CONCURRENCY`: default `200`, sets `effective_io_concurrency`. +- `FLUXER_POSTGRES_DEFAULT_STATISTICS_TARGET`: default `200`, sets `default_statistics_target`. +- `FLUXER_POSTGRES_JIT`: default `off`, sets `jit`. +- `FLUXER_POSTGRES_MIN_WAL_SIZE`: default `512MB`, sets `min_wal_size`. +- `FLUXER_POSTGRES_MAX_WAL_SIZE`: default `2GB`, sets `max_wal_size`. +- `FLUXER_POSTGRES_CHECKPOINT_COMPLETION_TARGET`: default `0.9`, sets `checkpoint_completion_target`. +- `FLUXER_POSTGRES_WAL_BUFFERS`: default `16MB`, sets `wal_buffers`. +- `FLUXER_POSTGRES_WAL_COMPRESSION`: default `zstd`, sets `wal_compression`. +- `FLUXER_POSTGRES_BGWRITER_DELAY`: default `50ms`, sets `bgwriter_delay`. +- `FLUXER_POSTGRES_BGWRITER_LRU_MAXPAGES`: default `1000`, sets `bgwriter_lru_maxpages`. +- `FLUXER_POSTGRES_AUTOVACUUM_VACUUM_SCALE_FACTOR`: default `0.05`, sets `autovacuum_vacuum_scale_factor`. +- `FLUXER_POSTGRES_AUTOVACUUM_ANALYZE_SCALE_FACTOR`: default `0.02`, sets `autovacuum_analyze_scale_factor`. +- `FLUXER_POSTGRES_AUTOVACUUM_VACUUM_COST_LIMIT`: default `2000`, sets `autovacuum_vacuum_cost_limit`. +- `FLUXER_POSTGRES_TRACK_IO_TIMING`: default `on`, sets `track_io_timing`. +- `FLUXER_POSTGRES_SHARED_PRELOAD_LIBRARIES`: default `pg_stat_statements`, sets `shared_preload_libraries`. + +`shared_preload_libraries` is read only when the server starts. Add a library only if the image ships it, or Postgres refuses to start. + +The bundled Valkey uses persistent storage. These settings control its memory limit, its behaviour when full, and how often it writes to disk. #### `FLUXER_VALKEY_MAXMEMORY` @@ -1940,6 +2015,10 @@ Default `192mb`. The dataset ceiling. Bounds stored keys only. Client buffers, r Default `noeviction`. An over-limit write returns an OOM error. Keep this policy to protect deletion queues and other shared state. See [Volumes and buckets](#volumes-and-buckets) for recovery implications. +#### `FLUXER_VALKEY_APPENDFSYNC` + +Default `everysec`. How often Valkey flushes its append-only file to disk. `always`, `everysec` or `no`. `everysec` can lose the last second of writes in a crash. + No service sets a CPU limit, a CPU reservation or `cpu_shares`, so every container sees the host's full CPU count. Bound the Gateway's scheduler count with `FLUXER_ERLANG_SCHEDULERS_MIN` and `FLUXER_ERLANG_SCHEDULERS_MAX`, or pin it with `FLUXER_ERLANG_SCHEDULERS` and `FLUXER_ERLANG_DIRTY_CPU_SCHEDULERS` from [Gateway settings](#gateway-settings). On a host smaller than the 16 GB the defaults assume, lower the large ceilings and the Postgres tuning together. The block below is the 8 GB profile. @@ -1992,11 +2071,29 @@ FLUXER_GIFS_SHARD_CACHE_MAX_BYTES=67108864 At 4 GB the api heap ceiling is 396 MB and the worker heap ceiling is 332 MB. That covers chat. A large attachment and a search reindex at the same time exceed those heap ceilings. Media transforms above roughly 20 MB start failing in `media-proxy`, Meilisearch indexes a backlog more slowly, and a busy voice room is the first thing to drop. Keep `FLUXER_POSTGRES_SERVER_MAX_CONNECTIONS` at or above 110. +## Health checks and restarts + +Compose reads these, and no Fluxer service does. The health checks decide when a dependent service may start and when Docker marks a container unhealthy. All are optional. + +- `FLUXER_RESTART_POLICY`: default `unless-stopped`, for every long-running service. `seaweedfs-init` never restarts. +- `FLUXER_HEALTHCHECK_INTERVAL`: default `10s`, for every check. +- `FLUXER_HEALTHCHECK_TIMEOUT`: default `5s`, for every check. +- `FLUXER_HEALTHCHECK_RETRIES`: default `10`, for `edge`, `postgres`, `valkey`, `nats`, `meilisearch`, `livekit` and `static-proxy`. +- `FLUXER_APP_HEALTHCHECK_RETRIES`: default `30`, for the internal services, `api`, `gateway`, `push` and `admin`. +- `FLUXER_APP_HEALTHCHECK_START_PERIOD`: default `90s`, for `api`, `worker` and `gateway`. +- `FLUXER_SVC_HEALTHCHECK_START_PERIOD`: default `60s`, for the internal services, `push` and `admin`. +- `FLUXER_WORKER_HEALTHCHECK_RETRIES`: default `3`, for `worker`. Its check reads a heartbeat file, and a stale one means the worker is stuck. +- `FLUXER_SEAWEEDFS_HEALTHCHECK_RETRIES`: default `20`, for `seaweedfs`. +- `FLUXER_SEAWEEDFS_HEALTHCHECK_START_PERIOD`: default `60s`, for `seaweedfs`. `api`, `worker` and `media-proxy` wait on `seaweedfs-init`, which waits on this check. +- `FLUXER_SEAWEEDFS_INIT_ATTEMPTS`: default `60`. Compose forwards it from `.env` to `seaweedfs-init`, which checks and creates the buckets that many times, 2 seconds apart, before it gives up. Raise it when `docker compose up` fails on `seaweedfs-init` on a slow disk. + +Durations take the Compose form, such as `90s` or `2m`. Raise the retries or start period on a slow host where a service is still starting when its check gives up. + ## Routing The edge is the only HTTP entry point, and every route is served from the one public hostname. It rewrites each path before handing it to an upstream. [What the single port routes](/operator/reverse-proxy/#what-the-single-port-routes) has the path table, and [What every proxy must do](/operator/reverse-proxy/#what-every-proxy-must-do) has the requirements for anything in front of the edge. -None of the variables on this page change that routing. The `Caddyfile` is a bind mount, so an edit to it takes `docker compose restart edge`. +`FLUXER_ADMIN_BASE_PATH` moves the admin path, and no other variable on this page changes that routing. The `Caddyfile` is a bind mount, so an edit to it takes `docker compose restart edge`. CORS origins are exactly the app endpoint, the `FLUXER_APP_ORIGIN_ALIASES` origins, and the marketing endpoint. Serving the client from a hostname other than `FLUXER_DOMAIN` requires overriding `FLUXER_APP_ENDPOINT`. @@ -2020,14 +2117,13 @@ Startup refuses an existing `JOBS` or `JOBS_DLQ` stream with the wrong name, sub Volume names are prefixed with the Compose project name, so `postgres-data` is `fluxer_postgres-data` on the host. -`seaweedfs-init` creates these buckets and exits. +`seaweedfs-init` creates these buckets and exits. Each bucket takes its name from its variable in `.env`, and the table gives the default. -| Bucket | Holds | -| --- | --- | -| fluxer | Avatars, guild and entity assets, themes, entrance sounds, memes, and processed attachments | -| fluxer-uploads | Raw attachment uploads, before processing | -| fluxer-downloads | Desktop client build artefacts | -| fluxer-reports | Abuse report evidence, and NCMEC payloads where that integration is on | -| fluxer-harvests | User and guild data archives | +| Bucket | Set by | Holds | +| --- | --- | --- | +| fluxer | `FLUXER_S3_BUCKET_CDN` | Avatars, guild and entity assets, themes, entrance sounds, memes, and processed attachments | +| fluxer-uploads | `FLUXER_S3_BUCKET_UPLOADS` | Raw attachment uploads, before processing | +| fluxer-reports | `FLUXER_S3_BUCKET_REPORTS` | Abuse report evidence, and NCMEC payloads where that integration is on | +| fluxer-harvests | `FLUXER_S3_BUCKET_HARVESTS` | User and guild data archives | `FLUXER_S3_BUCKET_STATIC` names an optional bucket. The shipped stack does not use or create it. diff --git a/fluxer_docs/src/content/docs/operator/get-started.mdx b/fluxer_docs/src/content/docs/operator/get-started.mdx index 0044569af..72528e099 100644 --- a/fluxer_docs/src/content/docs/operator/get-started.mdx +++ b/fluxer_docs/src/content/docs/operator/get-started.mdx @@ -238,12 +238,12 @@ The desktop client opens the hosted web app for its release channel, so reach yo Recovery requires `.env`, a database dump and a backup of `seaweedfs-data`, which holds uploads, avatars, reports and harvests. See [Volumes and buckets](/operator/configuration/#volumes-and-buckets) for everything that needs backing up. -The dump costs no downtime, so run it on a schedule while the stack serves: +The dump costs no downtime, so run it on a schedule while the stack serves. It runs inside the `postgres` container and takes the role and database from that container's environment, which follows `FLUXER_POSTGRES_USERNAME` and `FLUXER_POSTGRES_DATABASE`. It is for the bundled database. A database outside the stack is backed up with its own tools: ```bash cd ~/fluxer mkdir -p backups -docker compose exec -T postgres pg_dump -U fluxer -d fluxer --format=custom \ +docker compose exec -T postgres sh -c 'pg_dump -U $POSTGRES_USER -d $POSTGRES_DB --format=custom' \ > "backups/fluxer-$(date -u +%Y%m%dT%H%M%SZ).dump" ``` @@ -251,7 +251,7 @@ PowerShell writes UTF-16 through `>`, which corrupts a binary dump. On Windows, ```powershell mkdir -Force backups -docker compose exec -T postgres pg_dump -U fluxer -d fluxer --format=custom -f /tmp/fluxer.dump +docker compose exec -T postgres sh -c 'pg_dump -U $POSTGRES_USER -d $POSTGRES_DB --format=custom -f /tmp/fluxer.dump' docker compose cp postgres:/tmp/fluxer.dump backups/fluxer.dump docker compose exec -T postgres rm /tmp/fluxer.dump ``` diff --git a/fluxer_docs/src/content/docs/operator/reverse-proxy.mdx b/fluxer_docs/src/content/docs/operator/reverse-proxy.mdx index e4cbbdc59..b996e033b 100644 --- a/fluxer_docs/src/content/docs/operator/reverse-proxy.mdx +++ b/fluxer_docs/src/content/docs/operator/reverse-proxy.mdx @@ -396,7 +396,7 @@ Forward every path and query string unchanged. The edge handles routing: | `/.well-known/assetlinks.json` | Android app association | | `/version.json` | Client version metadata | -All other paths serve the web app. +All other paths serve the web app. The admin path follows `FLUXER_ADMIN_BASE_PATH`, `/admin` by default. Pass the query string on `/gateway` through untouched. Clients always send `?v=`, `?encoding=`, `?compress=` and `?stream=`, and `1` is the only version the Gateway accepts. diff --git a/fluxer_docs/src/content/docs/operator/upgrading.mdx b/fluxer_docs/src/content/docs/operator/upgrading.mdx index 42081cabc..4088fe45d 100644 --- a/fluxer_docs/src/content/docs/operator/upgrading.mdx +++ b/fluxer_docs/src/content/docs/operator/upgrading.mdx @@ -25,6 +25,8 @@ The images come from `FLUXER_IMAGE_TAG` in `.env`. The stack files come from a g The repository holds no ref by the name of a pinned image tag. A release tags each image on its own, as `fluxer-api@2026.813.205040`, so pass `--ref` with that tag or with the commit it points at. Without that override the download fails with exit 4. +The stack files on `main` can run ahead of the `v1` images. The current `docker-compose.yml` passes an optional setting with no Compose default through empty when `.env` leaves it out, and only images built from the same change or later read an empty value as unset. Older images reject some of those empty values and `api`, `worker` and `media-proxy` fail to start. Refresh the stack files only once the images the tag names are at least as new, or pin both with `--ref`. + ## What the script does `--update` runs these steps in this order, and stops on the first one that fails: @@ -195,6 +197,10 @@ Within minutes of the upgrade, `worker` starts deleting existing data that is pa Instances on Cassandra already behave this way. +## Check the svc request ceiling + +`docker-compose.yml` used to pass `FLUXER_SVC_MAX_CONCURRENT_REQUESTS` only to the `users` and `messages` routers and shards. It now reaches every svc container, so a value set in `.env` also caps `snowflakes`, `gifs` and `unfurl`. Unset, `snowflakes` allows `320` requests in flight, `messages` `192` and the rest `64`. An instance that sets the name either raises it to at least `320` or removes it from `.env`, then runs `docker compose up -d`. + ## Add the passkey columns on Cassandra An instance on the Postgres backend needs nothing here. An instance on Cassandra or Scylla adds two columns to `webauthn_credentials` before it starts the new `api` and `worker` images. Replace `fluxer` with the value of `FLUXER_CASSANDRA_KEYSPACE`. @@ -229,7 +235,7 @@ Nothing else is copied. The dump covers `postgres-data`. The other volumes eithe Put a dump on a timer as well. On Linux and macOS, this crontab line writes one a night and keeps two weeks of them: ```cron -15 3 * * * cd /srv/fluxer && docker compose exec -T postgres pg_dump -U fluxer -d fluxer --format=custom > "backups/fluxer-$(date -u +\%Y\%m\%dT\%H\%M\%SZ).dump" && find backups -name 'fluxer-*.dump' -mtime +14 -delete +15 3 * * * cd /srv/fluxer && docker compose exec -T postgres sh -c 'pg_dump -U $POSTGRES_USER -d $POSTGRES_DB --format=custom' > "backups/fluxer-$(date -u +\%Y\%m\%dT\%H\%M\%SZ).dump" && find backups -name 'fluxer-*.dump' -mtime +14 -delete ``` `/srv/fluxer` stands for the directory holding `.env`. Write it as an absolute path, because cron does not expand `~`. An unescaped `%` in a crontab is a newline, which is why every one above has a backslash. @@ -269,7 +275,7 @@ The `seaweedfs-data` volume stays until you delete it. Copy its objects to the n The other bundled services have no shipped overlay. Take one out with an override file listed in `COMPOSE_FILE`, which [Keep a local compose change](#keep-a-local-compose-change) describes, rather than by editing `docker-compose.yml`, which the Place step replaces on every upgrade. -The installer backs up only the bundled database and object store. Arrange separate backups for external stores before upgrading, and keep them with the release's backup record. +The installer backs up only the bundled database and object store. It skips the database dump when the stack defines no `postgres` service, and when `FLUXER_POSTGRES_HOST` or the host in `FLUXER_POSTGRES_URL` names anything other than `postgres`. The bundled service is idle in that shape, and its data directory still holds the role and database it was first started with, so the by-hand dump and restore commands on this page do not work against it either. Arrange separate backups for external stores before upgrading, and keep them with the release's backup record. ## Roll back @@ -295,12 +301,12 @@ The database restores from the custom-format dump: ```bash docker compose stop docker compose up -d --wait postgres -docker compose exec -T postgres pg_restore -U fluxer -d fluxer --clean --if-exists \ +docker compose exec -T postgres sh -c 'pg_restore -U $POSTGRES_USER -d $POSTGRES_DB --clean --if-exists' \ < backups/record-20260831T120000Z/fluxer.dump docker compose up -d ``` -`--clean` prints notices about objects that do not exist yet, which is expected against a fresh directory. +`--clean` prints notices about objects that do not exist yet, which is expected against a fresh directory. The role and database come from the `postgres` container's environment, which follows `FLUXER_POSTGRES_USERNAME` and `FLUXER_POSTGRES_DATABASE`, the same way the installer's dump reads them. These commands are for the bundled database. A database outside the stack restores with its own tools. PowerShell has no `<` redirection. On Windows, copy the dump into the container and name it as a file: @@ -308,7 +314,7 @@ PowerShell has no `<` redirection. On Windows, copy the dump into the container docker compose stop docker compose up -d --wait postgres docker compose cp backups\record-20260831T120000Z\fluxer.dump postgres:/tmp/fluxer.dump -docker compose exec -T postgres pg_restore -U fluxer -d fluxer --clean --if-exists /tmp/fluxer.dump +docker compose exec -T postgres sh -c 'pg_restore -U $POSTGRES_USER -d $POSTGRES_DB --clean --if-exists /tmp/fluxer.dump' docker compose exec -T postgres rm /tmp/fluxer.dump docker compose up -d ``` @@ -327,12 +333,12 @@ For a `seaweedfs-data.tar`, write `tar xf` in place of `tar xzf`. `tar` extracts ## Move to a new Postgres major version -`docker-compose.yml` pins `postgres:16-alpine`. A newer major does not read the data directory an older major wrote, so the data moves across through a dump. The upgrade refuses when the refreshed compose file changes that pin, because the move destroys the volume holding the database. +`docker-compose.yml` runs `postgres:16-alpine` unless `FLUXER_POSTGRES_IMAGE` in `.env` names another image. A newer major does not read the data directory an older major wrote, so the data moves across through a dump. The upgrade refuses when the refreshed compose file would run a different major from the current one, because the move destroys the volume holding the database. It reads the current major from the image the old file names, and takes `FLUXER_POSTGRES_IMAGE` into account only for a file that reads it. Take the dump while the old major is still serving, then stop everything and drop the volume: ```bash -docker compose exec -T postgres pg_dump -U fluxer -d fluxer --format=custom \ +docker compose exec -T postgres sh -c 'pg_dump -U $POSTGRES_USER -d $POSTGRES_DB --format=custom' \ > backups/pre-major.dump docker compose down docker volume rm fluxer_postgres-data @@ -342,16 +348,18 @@ On Windows, run `pg_dump` inside the container with `-f /tmp/pre-major.dump` and Once that volume is removed, the dump is the only copy of the database. -Put the new `postgres` tag in `docker-compose.yml`, start the database on its own, and restore into the empty directory: +`FLUXER_POSTGRES_IMAGE` takes effect only once `docker-compose.yml` reads it. An instance on older stack files runs `sh install.sh --update` first, or edits the tag in its `docker-compose.yml` as before. + +Set `FLUXER_POSTGRES_IMAGE` in `.env` to the new tag, such as `postgres:17-alpine`, start the database on its own, and restore into the empty directory: ```bash docker compose up -d --wait postgres -docker compose exec -T postgres pg_restore -U fluxer -d fluxer --clean --if-exists \ +docker compose exec -T postgres sh -c 'pg_restore -U $POSTGRES_USER -d $POSTGRES_DB --clean --if-exists' \ < backups/pre-major.dump docker compose up -d ``` -Run `sh install.sh --update` afterwards to finish the move on the refreshed files. +Run `sh install.sh --update` afterwards to finish the move on the refreshed files. Remove the `FLUXER_POSTGRES_IMAGE` line once the refreshed `docker-compose.yml` runs the same major by default. ## Pin a release @@ -359,11 +367,11 @@ Run `sh install.sh --update` afterwards to finish the move on the refreshed file A pinned instance also pins its stack files. [Match the images to the stack files](#match-the-images-to-the-stack-files) has the `--ref` a pinned tag needs. -The tag applies only to the Fluxer images. `caddy`, `postgres`, `valkey`, `nats`, `meilisearch`, `seaweedfs` and `livekit` are pinned in `docker-compose.yml`, and they move only when an upgrade refreshes that file. +The tag applies only to the Fluxer images. `caddy`, `postgres`, `valkey`, `nats`, `meilisearch`, `seaweedfs` and `livekit` take their tags from `docker-compose.yml`, and move when an upgrade refreshes that file. An image name in `.env`, such as `FLUXER_POSTGRES_IMAGE`, holds one of them in place instead. [Images](/operator/configuration/#images) lists the names. ## Change the domain or the passkey relying party -`FLUXER_DOMAIN` supplies the default `FLUXER_PASSKEY_RP_ID`, which is the domain a passkey is tied to. A passkey works only under the identifier it was created with. +`FLUXER_DOMAIN` supplies the default `FLUXER_PASSKEY_RP_ID`, which is the domain a passkey is tied to. `FLUXER_PUBLIC_ORIGIN` does not move it. A passkey works only under the identifier it was created with. :::danger[Changing either value invalidates every passkey] Passkeys registered under the old `FLUXER_PASSKEY_RP_ID` stop working and cannot be recovered. Every user has to register a new one, so keep another sign-in method working before you change it. diff --git a/fluxer_docs/src/installer/install.ps1 b/fluxer_docs/src/installer/install.ps1 index 0c6e2916a..30efdadde 100644 --- a/fluxer_docs/src/installer/install.ps1 +++ b/fluxer_docs/src/installer/install.ps1 @@ -435,8 +435,8 @@ function Get-FluxerRefForTag([string]$Tag) { } # The images come from FLUXER_IMAGE_TAG and the stack files come from a git ref. A release tags its -# images and its commit with the same CalVer string, so a pinned tag names the commit that carries -# its compose files. The moving tags v1 and latest track main. +# images and its commit with the same CalVer string, so a pinned tag names the commit that holds its +# compose files. The moving tags v1 and latest map to main, which can run ahead of the v1 images. function Assert-FluxerDerivedRef([string]$Value, [string]$EnvPath) { if ($Value.Length -eq 0) { Stop-Fluxer "$EnvPath declares no FLUXER_IMAGE_TAG, so no ref can be derived. Pass -Ref." $FluxerExitRefused @@ -631,8 +631,10 @@ function Remove-FluxerStagingDirectory([string]$Path) { # Invoke-WebRequest -Uri https://raw.githubusercontent.com/fluxerapp/fluxer/main/deploy/self-hosting/docker-compose.yml -OutFile docker-compose.yml # # The files come from a git ref and the images come from FLUXER_IMAGE_TAG. The ref is derived from -# the tag unless -Ref names one, which is the pairing rule that stops a compose file from asking for -# a variable the running images do not read. +# the tag unless -Ref names one. A pinned CalVer tag names the commit its images were built from, so +# its compose file asks only for variables those images read. The moving tags v1 and latest map to +# main, and main's compose file can run ahead of the v1 images until the image builds are dispatched +# again. # # Everything lands in a staging directory first, so a failed download leaves the working directory # on the set it already had, and so the upgrade can compare old against new before replacing. @@ -995,15 +997,34 @@ function Get-FluxerRunningImageId($Running, [string]$Reference) { return '' } -function Get-FluxerPostgresMajor([string]$Path) { +function Get-FluxerPostgresMajor([string]$Path, [string]$EnvPath) { if (-not (Test-Path -LiteralPath $Path)) { return '' } + $image = '' foreach ($line in [System.IO.File]::ReadAllText($Path).Split("`n")) { - if ($line -match '^\s*image:\s*postgres:(\d+)') { - return $Matches[1] + if ($line -match '^\s*image:\s*(postgres:\S*)') { + $image = $Matches[1] + break } } + if ($image.Length -eq 0) { + foreach ($line in [System.IO.File]::ReadAllText($Path).Split("`n")) { + if ($line -match '^\s*image:\s*\$\{FLUXER_POSTGRES_IMAGE:-([^}]*)\}') { + $image = $Matches[1] + $configured = Get-FluxerComposeValue $EnvPath 'FLUXER_POSTGRES_IMAGE' + if ($configured.Length -gt 0) { + $image = $configured + } + break + } + } + } + $name = ($image -split '@')[0] + $name = ($name -split '/')[-1] + if ($name -match ':(\d+)[^:]*$') { + return $Matches[1] + } return '' } @@ -1297,9 +1318,12 @@ function Test-FluxerDumpHeader([string]$Path) { # schema change. # # By hand: -# docker compose exec -T postgres pg_dump -U fluxer -d fluxer --format=custom > backups\fluxer.dump +# docker compose exec -T postgres sh -c 'pg_dump -U $POSTGRES_USER -d $POSTGRES_DB --format=custom -f /tmp/fluxer.dump' +# docker compose cp postgres:/tmp/fluxer.dump backups\fluxer.dump +# docker compose exec -T postgres rm /tmp/fluxer.dump # -# The database and the role are both named fluxer and are fixed in docker-compose.yml. Keep -T. +# The postgres container's POSTGRES_USER and POSTGRES_DB follow FLUXER_POSTGRES_USERNAME and +# FLUXER_POSTGRES_DATABASE, so the command needs no names. Keep -T. # Without it Docker attaches a terminal to the command and the dump arrives corrupted, which is # why the first five bytes are checked against the custom-format magic rather than only the size. # @@ -1327,11 +1351,44 @@ function Test-FluxerStackDefinesService([string]$Name, [string]$TargetDir) { return $script:FluxerStackServices -contains $Name } +# The bundled postgres container takes POSTGRES_USER and POSTGRES_DB from FLUXER_POSTGRES_USERNAME +# and FLUXER_POSTGRES_DATABASE, and the image applies them only to an empty data directory. When +# those names point the apps at a database outside the stack, the bundled directory still holds the +# role and database it was first started with, so a dump that reads the container env asks for a +# role that is not there. The bundled service is idle in that shape and the dump skips it. +function Test-FluxerPostgresExternal([string]$TargetDir) { + $envPath = Join-Path $TargetDir '.env' + $pgHost = Get-FluxerComposeValue $envPath 'FLUXER_POSTGRES_HOST' + if ($pgHost.Length -gt 0 -and $pgHost -ne 'postgres') { + return $true + } + $url = Get-FluxerComposeValue $envPath 'FLUXER_POSTGRES_URL' + if ($url.Length -eq 0) { + return $false + } + $urlHost = $url + $scheme = $urlHost.IndexOf('://') + if ($scheme -ge 0) { + $urlHost = $urlHost.Substring($scheme + 3) + } + $urlHost = ($urlHost -split '[/?]', 2)[0] + $at = $urlHost.LastIndexOf('@') + if ($at -ge 0) { + $urlHost = $urlHost.Substring($at + 1) + } + $urlHost = ($urlHost -split ':', 2)[0] + return $urlHost -ne 'postgres' +} + function Backup-FluxerDatabase([string]$Record, [string]$TargetDir) { if (-not (Test-FluxerStackDefinesService 'postgres' $TargetDir)) { Write-FluxerLine 'Skipping the database dump. This stack defines no postgres service, so its database runs outside the stack and only the operator of that database can dump it.' return } + if (Test-FluxerPostgresExternal $TargetDir) { + Write-FluxerLine 'Skipping the database dump. FLUXER_POSTGRES_HOST or FLUXER_POSTGRES_URL points the stack at a database outside it, so the bundled postgres service is idle and only the operator of that database can dump it.' + return + } if (-not (Test-FluxerPostgresRunning)) { Write-FluxerLine 'Postgres is not running. Starting it for the dump.' if ((Invoke-FluxerDocker @('compose', 'up', '-d', '--wait', 'postgres')) -ne 0) { @@ -1340,7 +1397,7 @@ function Backup-FluxerDatabase([string]$Record, [string]$TargetDir) { } $dump = Join-Path $Record $FluxerDumpFile Write-FluxerLine 'Dumping the database.' - $code = Invoke-FluxerDockerToFile @('compose', 'exec', '-T', 'postgres', 'pg_dump', '-U', 'fluxer', '-d', 'fluxer', '--format=custom') $dump $TargetDir + $code = Invoke-FluxerDockerToFile @('compose', 'exec', '-T', 'postgres', 'sh', '-c', '"exec pg_dump -U $POSTGRES_USER -d $POSTGRES_DB --format=custom"') $dump $TargetDir if ($code -ne 0) { Remove-FluxerTemporary $dump Stop-Fluxer 'pg_dump failed. The instance is untouched.' $FluxerExitBackup @@ -1462,8 +1519,9 @@ function Backup-FluxerInstance([string]$Record, [string]$TargetDir, [string]$Pro # The refreshed file is still staged when this runs, so a refusal here leaves the instance exactly # as it was. function Assert-FluxerPostgresMajor([string]$TargetDir, [string]$StagingDir) { - $old = Get-FluxerPostgresMajor (Join-Path $TargetDir $script:FluxerComposeBase) - $new = Get-FluxerPostgresMajor (Join-Path $StagingDir 'docker-compose.yml') + $envPath = Join-Path $TargetDir '.env' + $old = Get-FluxerPostgresMajor (Join-Path $TargetDir $script:FluxerComposeBase) $envPath + $new = Get-FluxerPostgresMajor (Join-Path $StagingDir 'docker-compose.yml') $envPath if ($old.Length -eq 0 -or $new.Length -eq 0 -or $old -eq $new) { return } @@ -1569,8 +1627,8 @@ function Show-FluxerUpdatePlan([string]$TargetDir, [string]$EnvPath, [string]$Ba if ($changed -eq 0) { Write-FluxerLine " Note: ref $Ref moves no stack file" } - $old = Get-FluxerPostgresMajor (Join-Path $TargetDir $script:FluxerComposeBase) - $new = Get-FluxerPostgresMajor (Join-Path $staging 'docker-compose.yml') + $old = Get-FluxerPostgresMajor (Join-Path $TargetDir $script:FluxerComposeBase) $EnvPath + $new = Get-FluxerPostgresMajor (Join-Path $staging 'docker-compose.yml') $EnvPath if ($old.Length -gt 0 -and $new.Length -gt 0 -and $old -ne $new) { Write-FluxerLine " Refusal: postgres moves from $old to $new, which this script does not do" Write-FluxerLine ' Outcome: the run stops at that refusal and changes nothing' @@ -1823,6 +1881,14 @@ function Get-FluxerEnvScalar([string]$EnvPath, [string]$Name) { return $raw } +function Get-FluxerComposeValue([string]$EnvPath, [string]$Name) { + $value = [string][Environment]::GetEnvironmentVariable($Name) + if ($value.Length -eq 0 -and (Test-Path -LiteralPath $EnvPath)) { + $value = Get-FluxerEnvScalar $EnvPath $Name + } + return $value +} + function Get-FluxerComposeSetting([string]$EnvPath) { $value = '' $source = 'the environment' diff --git a/fluxer_docs/src/installer/install.sh b/fluxer_docs/src/installer/install.sh index 153d53b82..3e3040ae1 100644 --- a/fluxer_docs/src/installer/install.sh +++ b/fluxer_docs/src/installer/install.sh @@ -711,8 +711,8 @@ fluxer_ref_for_tag() { # The images come from FLUXER_IMAGE_TAG and the stack files come from a git ref. # A release tags its images and its commit with the same CalVer string, so a -# pinned tag names the commit that carries its compose files. The moving tags v1 -# and latest track main. +# pinned tag names the commit that holds its compose files. The moving tags v1 +# and latest map to main, which can run ahead of the v1 images. fluxer_resolve_ref() { [ -z "$opt_ref" ] || return 0 if [ "$opt_update" -eq 1 ] || [ "$opt_rollback" -eq 1 ]; then @@ -763,9 +763,11 @@ fluxer_open_scratch() { # curl -fsSL https://raw.githubusercontent.com/fluxerapp/fluxer/main/deploy/self-hosting/docker-compose.yml -o docker-compose.yml # # The files come from a git ref and the images come from FLUXER_IMAGE_TAG. The -# ref is derived from the tag unless --ref names one, which is the pairing rule -# that stops a compose file from asking for a variable the running images do not -# read, or from pinning a service image the release never built. +# ref is derived from the tag unless --ref names one. A pinned CalVer tag names +# the commit its images were built from, so its compose file asks only for +# variables those images read and pins only images the release built. The moving +# tags v1 and latest map to main, and main's compose file can run ahead of the v1 +# images until the image builds are dispatched again. fluxer_fetch_stack() { fluxer_say "Downloading the stack files from ref $opt_ref." fluxer_stack_files > "$fluxer_scratch/files" @@ -1237,6 +1239,14 @@ fluxer_env_scalar() { printf '%s' "$fluxer_scalar" } +fluxer_compose_value() { + eval "fluxer_cv=\${$1:-}" + if [ -z "$fluxer_cv" ]; then + fluxer_cv=$(fluxer_env_scalar "$1") + fi + printf '%s' "$fluxer_cv" +} + fluxer_read_compose_setting() { fluxer_compose_file=${COMPOSE_FILE:-} fluxer_compose_from="the environment" @@ -1469,12 +1479,13 @@ fluxer_postgres_running() { # the pull is the only way back across a schema change. # # By hand: -# docker compose exec -T postgres pg_dump -U fluxer -d fluxer --format=custom > backups/fluxer.dump +# docker compose exec -T postgres sh -c 'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom' > backups/fluxer.dump # -# The database and the role are both named fluxer and are fixed in -# docker-compose.yml. Keep -T. Without it Docker attaches a terminal to the -# command and the dump arrives corrupted, which is why the first five bytes are -# checked against the custom-format magic below rather than only the size. +# The postgres container's POSTGRES_USER and POSTGRES_DB follow +# FLUXER_POSTGRES_USERNAME and FLUXER_POSTGRES_DATABASE, so the command needs no +# names. Keep -T. Without it Docker attaches a terminal to the command and the +# dump arrives corrupted, which is why the first five bytes are checked against +# the custom-format magic below rather than only the size. # # The dump runs against the live stack. pg_dump reads inside one transaction, so # it sees a consistent database without stopping anything. The volume copy below @@ -1501,11 +1512,39 @@ $(fluxer_compose_error ' ')" grep -qxF "$1" "$fluxer_scratch/all-services" } +# The bundled postgres container takes POSTGRES_USER and POSTGRES_DB from +# FLUXER_POSTGRES_USERNAME and FLUXER_POSTGRES_DATABASE, and the image applies +# them only to an empty data directory. When those names point the apps at a +# database outside the stack, the bundled directory still holds the role and +# database it was first started with, so a dump that reads the container env asks +# for a role that is not there. The bundled service is idle in that shape and the +# dump skips it. +# +# By hand: +# grep -E '^FLUXER_POSTGRES_(HOST|URL)=' .env +fluxer_postgres_external() { + fluxer_pg_host=$(fluxer_compose_value FLUXER_POSTGRES_HOST) + if [ -n "$fluxer_pg_host" ] && [ "$fluxer_pg_host" != postgres ]; then + return 0 + fi + fluxer_pg_url=$(fluxer_compose_value FLUXER_POSTGRES_URL) + [ -n "$fluxer_pg_url" ] || return 1 + fluxer_pg_url_host=${fluxer_pg_url#*://} + fluxer_pg_url_host=${fluxer_pg_url_host%%[/?]*} + fluxer_pg_url_host=${fluxer_pg_url_host##*@} + fluxer_pg_url_host=${fluxer_pg_url_host%%:*} + [ "$fluxer_pg_url_host" != postgres ] +} + fluxer_dump_postgres() { if ! fluxer_stack_defines_service postgres; then fluxer_say 'Skipping the database dump. This stack defines no postgres service, so its database runs outside the stack and only the operator of that database can dump it.' return 0 fi + if fluxer_postgres_external; then + fluxer_say 'Skipping the database dump. FLUXER_POSTGRES_HOST or FLUXER_POSTGRES_URL points the stack at a database outside it, so the bundled postgres service is idle and only the operator of that database can dump it.' + return 0 + fi if ! fluxer_postgres_running; then fluxer_say 'Postgres is not running. Starting it for the dump.' if ! $fluxer_engine compose up -d --wait postgres; then @@ -1514,7 +1553,7 @@ fluxer_dump_postgres() { fi fluxer_dump_path="$fluxer_record/$FLUXER_DUMP_FILE" fluxer_say 'Dumping the database.' - if ! $fluxer_engine compose exec -T postgres pg_dump -U fluxer -d fluxer --format=custom > "$fluxer_dump_path"; then + if ! $fluxer_engine compose exec -T postgres sh -c 'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom' > "$fluxer_dump_path"; then rm -f "$fluxer_dump_path" fluxer_fail 7 'pg_dump failed. The instance is untouched.' fi @@ -1627,7 +1666,17 @@ fluxer_backup() { } fluxer_postgres_major() { - sed -n 's/^[[:space:]]*image:[[:space:]]*postgres:\([0-9][0-9]*\).*/\1/p' "$1" | head -n 1 + fluxer_pg_image=$(sed -n 's/^[[:space:]]*image:[[:space:]]*\(postgres:[^[:space:]]*\).*/\1/p' "$1" | head -n 1) + if [ -z "$fluxer_pg_image" ]; then + fluxer_pg_image=$(sed -n 's/^[[:space:]]*image:[[:space:]]*${FLUXER_POSTGRES_IMAGE:-\([^}]*\)}.*/\1/p' "$1" | head -n 1) + if [ -n "$fluxer_pg_image" ] && [ -n "$(fluxer_compose_value FLUXER_POSTGRES_IMAGE)" ]; then + fluxer_pg_image=$(fluxer_compose_value FLUXER_POSTGRES_IMAGE) + fi + fi + fluxer_pg_image=${fluxer_pg_image%%@*} + case ${fluxer_pg_image##*/} in + *:*) printf '%s\n' "${fluxer_pg_image##*:}" | sed -n 's/^\([0-9][0-9]*\).*/\1/p' ;; + esac } # A newer Postgres major does not read the data directory an older major wrote, diff --git a/fluxer_gateway/Dockerfile b/fluxer_gateway/Dockerfile index c0c845b20..cc54a5eb2 100644 --- a/fluxer_gateway/Dockerfile +++ b/fluxer_gateway/Dockerfile @@ -1,7 +1,6 @@ # SPDX-License-Identifier: AGPL-3.0-or-later FROM erlang:28-slim AS build -ARG LOGGER_LEVEL=info ARG RUST_TOOLCHAIN=1.98.1 ENV PATH="/root/.cargo/bin:${PATH}" @@ -15,7 +14,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ g++ \ libc6-dev \ - gettext-base \ ca-certificates \ && rm -rf /var/lib/apt/lists/* @@ -29,7 +27,7 @@ COPY . . WORKDIR /usr/src/app/fluxer_gateway RUN rebar3 compile --deps_only -RUN LOGGER_LEVEL=${LOGGER_LEVEL} envsubst < config/sys.config.template > config/sys.config && \ +RUN cp config/sys.config.template config/sys.config && \ rebar3 as prod release FROM erlang:28-slim diff --git a/fluxer_gateway/scripts/docker_entrypoint.sh b/fluxer_gateway/scripts/docker_entrypoint.sh index 482f651a1..eba970084 100755 --- a/fluxer_gateway/scripts/docker_entrypoint.sh +++ b/fluxer_gateway/scripts/docker_entrypoint.sh @@ -50,8 +50,8 @@ clamp_int() { echo "$value" } -: "${FLUXER_ERLANG_SCHEDULERS_MIN:=2}" -: "${FLUXER_ERLANG_SCHEDULERS_MAX:=16}" +is_positive_int "${FLUXER_ERLANG_SCHEDULERS_MIN:-}" || FLUXER_ERLANG_SCHEDULERS_MIN=2 +is_positive_int "${FLUXER_ERLANG_SCHEDULERS_MAX:-}" || FLUXER_ERLANG_SCHEDULERS_MAX=16 : "${FLUXER_ERLANG_NODE_NAME:=fluxer_gateway@127.0.0.1}" : "${FLUXER_ERLANG_DIST_PORT:=8081}" diff --git a/fluxer_gateway/src/gateway/fluxer_gateway_config.erl b/fluxer_gateway/src/gateway/fluxer_gateway_config.erl index 2b22346ae..25664477f 100644 --- a/fluxer_gateway/src/gateway/fluxer_gateway_config.erl +++ b/fluxer_gateway/src/gateway/fluxer_gateway_config.erl @@ -3,7 +3,7 @@ -module(fluxer_gateway_config). -typing([eqwalizer]). --export([load/0, build_config/1, optional_string/1]). +-export([load/0, build_config/1, optional_string/1, env_value/1]). -export_type([config/0]). -type config() :: map(). @@ -31,8 +31,7 @@ env_config() -> <<"internal">> => env_internal_config(), <<"public">> => env_public_config(), <<"proxy">> => env_proxy_config(), - <<"services">> => env_services_config(), - <<"telemetry">> => env_telemetry_config() + <<"services">> => env_services_config() }. -spec env_internal_config() -> map(). @@ -82,7 +81,7 @@ env_gateway_base_config() -> <<"push_outbox_request_timeout_ms">> => env_int( "FLUXER_GATEWAY_PUSH_OUTBOX_REQUEST_TIMEOUT_MS", 100000 ), - <<"logger_level">> => env_binary("FLUXER_GATEWAY_LOGGER_LEVEL", <<"info">>), + <<"logger_level">> => env_optional_binary("FLUXER_GATEWAY_LOGGER_LEVEL"), <<"api_rpc_endpoint">> => env_optional_binary("FLUXER_GATEWAY_API_RPC_ENDPOINT"), <<"cluster_enabled">> => env_bool("FLUXER_GATEWAY_CLUSTER_ENABLED", false), <<"cluster_discovery_dns_name">> => env_optional_binary( @@ -109,7 +108,6 @@ env_gateway_base_config() -> <<"presence_push_buffer_max_bytes">> => env_int( "FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_BYTES", 1048576 ), - <<"shutdown_drain_wait_ms">> => env_int("FLUXER_GATEWAY_SHUTDOWN_DRAIN_WAIT_MS", 5000), <<"gateway_http_rpc_max_concurrency">> => env_int( "FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY", 512 ), @@ -131,19 +129,11 @@ env_nats_config() -> <<"auth_token">> => env_string("FLUXER_NATS_AUTH_TOKEN", "") }. --spec env_telemetry_config() -> map(). -env_telemetry_config() -> - #{ - <<"enabled">> => env_bool("FLUXER_TELEMETRY_ENABLED", false), - <<"environment">> => env_string("FLUXER_ENV", "development") - }. - -spec build_config(map()) -> config(). build_config(RawConfig) -> Service = get_map(RawConfig, [<<"services">>, <<"gateway">>]), Internal = get_map(RawConfig, [<<"internal">>]), Nats = get_map(RawConfig, [<<"services">>, <<"nats">>]), - Telemetry = get_map(RawConfig, [<<"telemetry">>]), Proxy = get_map(RawConfig, [<<"proxy">>]), Public = get_map(RawConfig, [<<"public">>]), lists:foldl(fun maps:merge/2, #{}, [ @@ -152,7 +142,7 @@ build_config(RawConfig) -> build_sharding_config(Service), build_http_config(Service), build_cluster_config(Service, Public), - build_misc_config(Service, Telemetry) + build_misc_config(Service) ]). -spec build_core_config(map(), map(), map(), map()) -> config(). @@ -178,9 +168,6 @@ build_core_config(Service, Internal, Nats, Proxy) -> build_push_config(Service, Public) -> #{ push_enabled => get_bool(Service, <<"push_enabled">>, true), - push_user_guild_settings_cache_mb => - get_int(Service, <<"push_user_guild_settings_cache_mb">>, 1024), - push_blocked_ids_cache_mb => get_int(Service, <<"push_blocked_ids_cache_mb">>, 1024), static_cdn_endpoint => public_endpoint( get_binary(Service, <<"static_cdn_endpoint">>, <<"http://localhost:8088">>), Public ), @@ -208,8 +195,7 @@ build_sharding_config(Service) -> guild_counts_cache_shards => get_optional_int(Service, <<"guild_counts_cache_shards">>), guild_shards => get_optional_int(Service, <<"guild_shards">>), session_shards => get_optional_int(Service, <<"session_shards">>), - session_connect_max_queue => get_int(Service, <<"session_connect_max_queue">>, 1024), - shutdown_drain_wait_ms => get_int(Service, <<"shutdown_drain_wait_ms">>, 5000) + session_connect_max_queue => get_int(Service, <<"session_connect_max_queue">>, 1024) }. -spec build_http_config(map()) -> config(). @@ -247,19 +233,15 @@ build_cluster_config(Service, Public) -> ) }. --spec build_misc_config(map(), map()) -> config(). -build_misc_config(Service, Telemetry) -> +-spec build_misc_config(map()) -> config(). +build_misc_config(Service) -> #{ handoff_enable_event_pause => get_bool( Service, <<"handoff_enable_event_pause">>, false ), voice_state_counts_sync_interval_ms => get_int(Service, <<"voice_state_counts_sync_interval_ms">>, 30000), - logger_level => get_log_level(Service, <<"logger_level">>, warning), - telemetry => #{ - enabled => get_bool(Telemetry, <<"enabled">>, true), - environment => get_string(Telemetry, <<"environment">>, "development") - } + logger_level => get_log_level(Service, <<"logger_level">>, info) }. -spec get_map(map(), [binary()]) -> map(). @@ -271,21 +253,33 @@ get_map(Map, Keys) -> -spec env_string(string(), string()) -> string(). env_string(Name, Default) -> - case os:getenv(Name) of - false -> Default; - "" -> Default; + case env_value(Name) of + undefined -> Default; Value -> Value end. +-spec env_value(string()) -> string() | undefined. +env_value(Name) -> + case os:getenv(Name) of + false -> undefined; + Value -> non_blank(Value) + end. + +-spec non_blank(string()) -> string() | undefined. +non_blank(Value) -> + case string:trim(Value) of + "" -> undefined; + _ -> Value + end. + -spec env_binary(string(), binary()) -> binary(). env_binary(Name, Default) -> characters_to_binary_or_default(env_string(Name, binary_to_list(Default)), Default). -spec env_optional_binary(string()) -> binary() | undefined. env_optional_binary(Name) -> - case os:getenv(Name) of - false -> undefined; - "" -> undefined; + case env_value(Name) of + undefined -> undefined; Value -> characters_to_binary_or_default(Value, undefined) end. @@ -300,27 +294,17 @@ characters_to_binary_or_default(Value, Default) -> -spec env_int(string(), integer()) -> integer(). env_int(Name, Default) -> - parse_env_int(Name, os:getenv(Name), Default). + case env_value(Name) of + undefined -> Default; + Value -> parse_int(Name, Value) + end. --spec parse_env_int(string(), false | string(), integer()) -> integer(). -parse_env_int(_Name, false, Default) -> - Default; -parse_env_int(_Name, "", Default) -> - Default; -parse_env_int(Name, Value, Default) -> - parse_int(Name, Value, Default). - --spec parse_int(string(), string(), integer()) -> integer(). -parse_int(Name, Value, Default) -> - case string:trim(Value) of - "" -> - Default; - Trimmed -> - try list_to_integer(Trimmed) of - Parsed -> Parsed - catch - error:badarg -> erlang:error({invalid_integer_env, Name, Value}) - end +-spec parse_int(string(), string()) -> integer(). +parse_int(Name, Value) -> + try list_to_integer(string:trim(Value)) of + Parsed -> Parsed + catch + error:badarg -> erlang:error({invalid_integer_env, Name, Value}) end. -spec env_bool(string(), boolean()) -> boolean(). diff --git a/fluxer_gateway/src/gateway/fluxer_gateway_env.erl b/fluxer_gateway/src/gateway/fluxer_gateway_env.erl index fd963217f..e1aeed7b5 100644 --- a/fluxer_gateway/src/gateway/fluxer_gateway_env.erl +++ b/fluxer_gateway/src/gateway/fluxer_gateway_env.erl @@ -60,36 +60,16 @@ build_config() -> -spec apply_system_config(config()) -> ok. apply_system_config(Config) -> - apply_logger_config(Config), - store_environment(Config). - --spec apply_logger_config(config()) -> ok. -apply_logger_config(Config) -> LoggerLevel = resolve_logger_level(Config), _ = logger:set_primary_config(level, LoggerLevel), _ = logger:set_handler_config(default, level, LoggerLevel), ok. --spec store_environment(config()) -> ok. -store_environment(Config) -> - Telemetry = maps:get(telemetry, Config, #{}), - Environment = maps:get(environment, Telemetry, <<"unknown">>), - EnvBin = ensure_binary(Environment), - persistent_term:put({fluxer_config, environment}, EnvBin), - ok. - --spec ensure_binary(term()) -> binary(). -ensure_binary(Value) when is_binary(Value) -> Value; -ensure_binary(Value) when is_list(Value) -> characters_to_binary_or_unknown(Value); -ensure_binary(Value) when is_atom(Value) -> atom_to_binary(Value, utf8); -ensure_binary(_) -> <<"unknown">>. - -spec resolve_logger_level(config()) -> logger_level(). resolve_logger_level(Config) -> Default = normalize_logger_level(maps:get(logger_level, Config, info)), - case os:getenv("LOGGER_LEVEL") of - false -> Default; - "" -> Default; + case fluxer_gateway_config:env_value("LOGGER_LEVEL") of + undefined -> Default; Value -> parse_logger_level(Value, Default) end. @@ -117,7 +97,3 @@ normalize_logger_level(critical) -> critical; normalize_logger_level(alert) -> alert; normalize_logger_level(emergency) -> emergency; normalize_logger_level(_) -> info. - --spec characters_to_binary_or_unknown(term()) -> binary(). -characters_to_binary_or_unknown(Value) -> - type_conv:ensure_binary(Value, <<"unknown">>). diff --git a/fluxer_gateway/src/gateway/gateway_build_info.erl b/fluxer_gateway/src/gateway/gateway_build_info.erl index 3432d48eb..66cd5c7f8 100644 --- a/fluxer_gateway/src/gateway/gateway_build_info.erl +++ b/fluxer_gateway/src/gateway/gateway_build_info.erl @@ -9,13 +9,9 @@ -spec version() -> binary(). version() -> - case os:getenv("BUILD_VERSION") of - false -> - <<"dev">>; - "" -> - <<"dev">>; - Value -> - ensure_binary(Value) + case fluxer_gateway_config:env_value("BUILD_VERSION") of + undefined -> <<"dev">>; + Value -> ensure_binary(Value) end. -spec version_headers(#{binary() => binary()}) -> #{binary() => binary()}. diff --git a/fluxer_gateway/src/gateway/gateway_timings.erl b/fluxer_gateway/src/gateway/gateway_timings.erl index 17af4878c..9e336a6e4 100644 --- a/fluxer_gateway/src/gateway/gateway_timings.erl +++ b/fluxer_gateway/src/gateway/gateway_timings.erl @@ -523,8 +523,8 @@ pod_name() -> first_runtime_name([], Fallback) -> Fallback; first_runtime_name([Name | Rest], Fallback) -> - case os:getenv(Name) of - false -> first_runtime_name(Rest, Fallback); + case fluxer_gateway_config:env_value(Name) of + undefined -> first_runtime_name(Rest, Fallback); Value -> normalize_key(Value) end. diff --git a/fluxer_gateway/test/fluxer_gateway_config_tests.erl b/fluxer_gateway/test/fluxer_gateway_config_tests.erl index e13281ac1..79a963498 100644 --- a/fluxer_gateway/test/fluxer_gateway_config_tests.erl +++ b/fluxer_gateway/test/fluxer_gateway_config_tests.erl @@ -117,14 +117,12 @@ presence_push_buffer_env_defaults_test() -> env_only_http_runtime_config_test() -> with_envs( [ - {"FLUXER_GATEWAY_SHUTDOWN_DRAIN_WAIT_MS", "1234"}, {"FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY", "42"}, {"FLUXER_GATEWAY_HTTP_FAILURE_THRESHOLD", "9"}, {"FLUXER_GATEWAY_HTTP_RECOVERY_TIMEOUT_MS", "6000"} ], fun() -> Config = fluxer_gateway_config:load(), - ?assertEqual(1234, maps:get(shutdown_drain_wait_ms, Config)), ?assertEqual(42, maps:get(gateway_http_rpc_max_concurrency, Config)), ?assertEqual(9, maps:get(gateway_http_failure_threshold, Config)), ?assertEqual(6000, maps:get(gateway_http_recovery_timeout_ms, Config)) @@ -158,6 +156,32 @@ env_int_falls_back_to_the_default_for_an_empty_value_test() -> ?assertEqual(512, maps:get(gateway_http_rpc_max_concurrency, Config)) end). +blank_env_values_fall_back_to_the_defaults_test() -> + with_envs( + [ + {"FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY", " "}, + {"FLUXER_CLIENT_IP_HEADER_NAME", " "}, + {"FLUXER_NATS_URL", "\t"}, + {"FLUXER_GATEWAY_API_RPC_ENDPOINT", " "}, + {"FLUXER_GATEWAY_LOGGER_LEVEL", " "}, + {"FLUXER_GATEWAY_PUSH_ENABLED", " "} + ], + fun() -> + Config = fluxer_gateway_config:load(), + ?assertEqual(512, maps:get(gateway_http_rpc_max_concurrency, Config)), + ?assertEqual(<<"x-forwarded-for">>, maps:get(client_ip_header, Config)), + ?assertEqual("nats://nats:4222", maps:get(nats_core_url, Config)), + ?assertEqual(undefined, maps:get(api_rpc_endpoint, Config)), + ?assertEqual(info, maps:get(logger_level, Config)), + ?assertEqual(true, maps:get(push_enabled, Config)) + end + ). + +logger_level_env_test() -> + with_env("FLUXER_GATEWAY_LOGGER_LEVEL", "Debug", fun() -> + ?assertEqual(debug, maps:get(logger_level, fluxer_gateway_config:load())) + end). + rpc_concurrency_key_defaults_test() -> Config = fluxer_gateway_config:build_config(#{}), ?assertEqual(512, maps:get(gateway_nats_rpc_max_handlers, Config)), diff --git a/fluxer_gifs/src/media_proxy.rs b/fluxer_gifs/src/media_proxy.rs index 9bf094316..2195ca636 100644 --- a/fluxer_gifs/src/media_proxy.rs +++ b/fluxer_gifs/src/media_proxy.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +use fluxer_svc::config::optional_env; use url::Url; #[derive(Clone)] @@ -25,19 +26,17 @@ impl MediaProxyUrlBuilder { } pub fn from_env() -> anyhow::Result { - let endpoint = std::env::var("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT") - .or_else(|_| std::env::var("FLUXER_MEDIA_ENDPOINT")) - .unwrap_or_default(); - if endpoint.trim().is_empty() { + let Some(endpoint) = optional_env("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT") + .or_else(|| optional_env("FLUXER_MEDIA_ENDPOINT")) + else { anyhow::bail!( "gifs shard requires FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT or FLUXER_MEDIA_ENDPOINT" ); - } + }; - let secret_key = std::env::var("FLUXER_MEDIA_PROXY_SECRET_KEY").unwrap_or_default(); - if secret_key.trim().is_empty() { + let Some(secret_key) = optional_env("FLUXER_MEDIA_PROXY_SECRET_KEY") else { anyhow::bail!("gifs shard requires FLUXER_MEDIA_PROXY_SECRET_KEY"); - } + }; let endpoint = fluxer_common::config::normalize_public_endpoint_from_env( endpoint.trim_end_matches('/'), diff --git a/fluxer_media_proxy/Cargo.toml b/fluxer_media_proxy/Cargo.toml index 9f4259b72..2408c4a27 100644 --- a/fluxer_media_proxy/Cargo.toml +++ b/fluxer_media_proxy/Cargo.toml @@ -49,7 +49,7 @@ thiserror = "2.0.20" tokio = {version = "1.53.1", features = ["fs", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time"]} tokio-util = {version = "0.7.19", features = ["io"]} tracing = "0.1.44" -tracing-subscriber = {version = "0.3.23", features = ["env-filter", "fmt", "json"]} +tracing-subscriber = {version = "0.3.23", features = ["fmt", "json"]} url = "2.5.8" wyhash = "0.6.0" diff --git a/fluxer_media_proxy/src/config/mod.rs b/fluxer_media_proxy/src/config/mod.rs index 1b40109cb..ff971261c 100644 --- a/fluxer_media_proxy/src/config/mod.rs +++ b/fluxer_media_proxy/src/config/mod.rs @@ -8,7 +8,7 @@ use crate::constants; use crate::secret::{SecretBytes, SecretString}; use http::HeaderValue; use parse::{ - EnvMap, decode_upload_relay_secret, default_native_transform_concurrency, non_empty, + EnvMap, decode_upload_relay_secret, default_native_transform_concurrency, parse_allowed_origins, parse_attachment_url_secrets, parse_bool, parse_bucket_style, parse_f32, parse_mode_env, parse_policy_mode, parse_storage_backend, parse_u16, parse_u64, parse_usize, validate_read_endpoint, @@ -128,7 +128,6 @@ pub struct UploadRelayConfig { #[derive(Clone, Debug)] pub struct Config { - pub node_env: String, pub bind_host: String, pub port: u16, pub(crate) secret_key: SecretString, @@ -174,7 +173,6 @@ impl Config { })?; Ok(Self { - node_env: env.get("NODE_ENV").unwrap_or("development").to_owned(), bind_host: env .get("FLUXER_MEDIA_PROXY_HOST") .unwrap_or("0.0.0.0") @@ -185,15 +183,16 @@ impl Config { 8080, )?, secret_key, - public_endpoint: non_empty(env.get("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT")).map( - |endpoint| { + public_endpoint: env + .get("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT") + .map(str::trim) + .map(|endpoint| { fluxer_common::config::normalize_public_endpoint( endpoint.trim_end_matches('/'), &public_base_domain, public_port, ) - }, - ), + }), mode, read_only: parse_bool( "FLUXER_MEDIA_PROXY_READ_ONLY", @@ -234,12 +233,16 @@ impl StorageConfig { .get("FLUXER_S3_BUCKET_CDN") .map(ToOwned::to_owned) .unwrap_or_else(|| "cdn".to_owned()); - let s3_read_endpoint = non_empty(env.get("FLUXER_S3_READ_ENDPOINT")); + let s3_read_endpoint = env + .get("FLUXER_S3_READ_ENDPOINT") + .map(|v| v.trim().to_owned()); if let Some(endpoint) = s3_read_endpoint.as_deref() { validate_read_endpoint(endpoint)?; } - let s3_read_bucket = - non_empty(env.get("FLUXER_S3_READ_BUCKET")).unwrap_or_else(|| bucket_cdn.clone()); + let s3_read_bucket = env + .get("FLUXER_S3_READ_BUCKET") + .map(|v| v.trim().to_owned()) + .unwrap_or_else(|| bucket_cdn.clone()); let s3_read_bucket_style = parse_bucket_style(env.get("FLUXER_S3_READ_BUCKET_STYLE"))? .unwrap_or(if s3_force_path_style { BucketStyle::Path @@ -248,7 +251,7 @@ impl StorageConfig { }); let s3_read_signed = parse_bool( "FLUXER_S3_READ_SIGNED", - non_empty(env.get("FLUXER_S3_READ_SIGNED")).as_deref(), + env.get("FLUXER_S3_READ_SIGNED").map(str::trim), )? .unwrap_or(false); Ok(Self { diff --git a/fluxer_media_proxy/src/config/parse.rs b/fluxer_media_proxy/src/config/parse.rs index 40527de0e..b11b5298e 100644 --- a/fluxer_media_proxy/src/config/parse.rs +++ b/fluxer_media_proxy/src/config/parse.rs @@ -26,6 +26,7 @@ impl EnvMap { self.0 .iter() .find_map(|(k, v)| (k == key).then_some(v.as_str())) + .filter(|v| !v.trim().is_empty()) } } @@ -150,14 +151,8 @@ fn is_origin_host(host: url::Host<&str>) -> bool { } } -pub(super) fn non_empty(raw: Option<&str>) -> Option { - raw.map(str::trim) - .filter(|value| !value.is_empty()) - .map(ToOwned::to_owned) -} - pub(super) fn parse_bucket_style(raw: Option<&str>) -> anyhow::Result> { - let Some(raw) = non_empty(raw) else { + let Some(raw) = raw.map(str::trim) else { return Ok(None); }; match raw.to_ascii_lowercase().as_str() { @@ -210,7 +205,7 @@ pub(super) fn decode_upload_relay_secret( raw: Option<&str>, mode: DeploymentMode, ) -> anyhow::Result { - let Some(raw) = raw.map(str::trim).filter(|s| !s.is_empty()) else { + let Some(raw) = raw.map(str::trim) else { anyhow::ensure!( !mode.serves_upload_relay(), "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 is required in upload and relay modes" diff --git a/fluxer_media_proxy/src/config/tests/attachment_signature.rs b/fluxer_media_proxy/src/config/tests/attachment_signature.rs index 47fd4e6b4..795d37fec 100644 --- a/fluxer_media_proxy/src/config/tests/attachment_signature.rs +++ b/fluxer_media_proxy/src/config/tests/attachment_signature.rs @@ -53,7 +53,7 @@ fn the_signature_mode_parses_every_variant_case_insensitively() { #[test] fn rejects_an_unknown_signature_mode() { - for raw in ["strict", "true", "1", "", "deny"] { + for raw in ["strict", "true", "1", "deny"] { let err = Config::load_from_iter(env_with(&[(MODE_KEY, raw), (SECRETS_KEY, FIRST)])).unwrap_err(); assert_eq!( diff --git a/fluxer_media_proxy/src/config/tests/cors.rs b/fluxer_media_proxy/src/config/tests/cors.rs index 34e735851..1dee2e946 100644 --- a/fluxer_media_proxy/src/config/tests/cors.rs +++ b/fluxer_media_proxy/src/config/tests/cors.rs @@ -45,7 +45,7 @@ fn cors_mode_parses_every_variant_case_insensitively() { #[test] fn rejects_an_unknown_cors_mode() { - for raw in ["strict", "true", "1", ""] { + for raw in ["strict", "true", "1"] { let err = Config::load_from_iter(env_with(&[ ("FLUXER_MEDIA_PROXY_CORS_MODE", raw), (ORIGINS_KEY, "https://web.fluxer.app"), diff --git a/fluxer_media_proxy/src/config/tests/mod.rs b/fluxer_media_proxy/src/config/tests/mod.rs index d155d6ae1..6be5f1df8 100644 --- a/fluxer_media_proxy/src/config/tests/mod.rs +++ b/fluxer_media_proxy/src/config/tests/mod.rs @@ -28,7 +28,6 @@ fn env_with<'a>(extra: &[(&'a str, &'a str)]) -> Vec<(&'a str, &'a str)> { fn with_shared_runtime_env(release: &[(&str, &str)]) -> Vec<(String, String)> { [ - ("NODE_ENV", "production"), ("FLUXER_ENV", "production"), ("FLUXER_MEDIA_PROXY_SECRET_KEY", "shared-runtime-secret"), ("FLUXER_S3_ENDPOINT", "https://ewr1.vultrobjects.com"), @@ -68,6 +67,36 @@ fn default_config_matches_media_service() { ); } +#[test] +fn blank_values_fall_back_to_the_defaults() { + let cfg = Config::load_from_iter(env_with(&[ + ("FLUXER_MEDIA_PROXY_MODE", ""), + ("FLUXER_MEDIA_PROXY_HOST", " "), + ("FLUXER_MEDIA_PROXY_PORT", ""), + ("FLUXER_MEDIA_PROXY_READ_ONLY", ""), + ("FLUXER_MEDIA_PROXY_STORAGE_BACKEND", ""), + ("FLUXER_MEDIA_PROXY_NSFW_THRESHOLD", " "), + ("FLUXER_MEDIA_PROXY_CORS_MODE", ""), + ("FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE", " "), + ("FLUXER_S3_REGION", ""), + ("FLUXER_S3_BUCKET_CDN", ""), + ])) + .unwrap(); + assert_eq!(DeploymentMode::Mp, cfg.mode); + assert_eq!("0.0.0.0", cfg.bind_host); + assert_eq!(8080, cfg.port); + assert!(!cfg.read_only); + assert_eq!(StorageBackend::Local, cfg.storage.backend); + assert_eq!(0.85, cfg.media.nsfw_threshold); + assert_eq!(PolicyMode::Off, cfg.cors.mode); + assert_eq!(PolicyMode::Off, cfg.attachment_signature.mode); + assert_eq!("us-east-1", cfg.storage.s3_region); + assert_eq!("cdn", cfg.storage.bucket_cdn); + + let err = Config::load_from_iter([("FLUXER_MEDIA_PROXY_SECRET_KEY", " ")]).unwrap_err(); + assert!(err.to_string().contains("FLUXER_MEDIA_PROXY_SECRET_KEY")); +} + #[test] fn canonical_media_proxy_env_overrides_apply() { let cfg = Config::load_from_iter([ @@ -282,7 +311,6 @@ fn production_media_proxy_release_env_loads() { ])) .unwrap(); - assert_eq!("production", cfg.node_env); assert_eq!(DeploymentMode::Mp, cfg.mode); assert_eq!(PolicyMode::Enforce, cfg.cors.mode); assert_eq!( diff --git a/fluxer_media_proxy/src/healthcheck.rs b/fluxer_media_proxy/src/healthcheck.rs index 41d10ce5a..66e14e3ef 100644 --- a/fluxer_media_proxy/src/healthcheck.rs +++ b/fluxer_media_proxy/src/healthcheck.rs @@ -2,24 +2,20 @@ use anyhow::Context as _; use std::{ - env, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, time::Duration, }; pub async fn run() -> anyhow::Result<()> { let addr = target( - env::var("FLUXER_MEDIA_PROXY_HOST").ok().as_deref(), - env::var("FLUXER_MEDIA_PROXY_PORT").ok().as_deref(), + fluxer_common::config::env_value("FLUXER_MEDIA_PROXY_HOST").as_deref(), + fluxer_common::config::env_value("FLUXER_MEDIA_PROXY_PORT").as_deref(), )?; probe(addr).await } fn target(host: Option<&str>, port: Option<&str>) -> anyhow::Result { - let host = host - .map(str::trim) - .filter(|host| !host.is_empty()) - .unwrap_or("127.0.0.1"); + let host = host.map(str::trim).unwrap_or("127.0.0.1"); let ip = host .parse::() .with_context(|| format!("FLUXER_MEDIA_PROXY_HOST is not an IP address: {host}"))?; @@ -28,7 +24,7 @@ fn target(host: Option<&str>, port: Option<&str>) -> anyhow::Result IpAddr::V6(ip) if ip.is_unspecified() => IpAddr::V6(Ipv6Addr::LOCALHOST), ip => ip, }; - let port = match port.map(str::trim).filter(|port| !port.is_empty()) { + let port = match port.map(str::trim) { Some(port) => port .parse::() .with_context(|| format!("FLUXER_MEDIA_PROXY_PORT is not a port number: {port}"))?, @@ -72,7 +68,7 @@ mod tests { ); assert_eq!( "[::1]:8080".parse::().unwrap(), - target(Some("::"), Some("")).unwrap() + target(Some("::"), None).unwrap() ); assert!(target(Some("0.0.0.0"), Some("nope")).is_err()); } diff --git a/fluxer_media_proxy/src/main.rs b/fluxer_media_proxy/src/main.rs index 87f73f50f..9c1b6b0db 100644 --- a/fluxer_media_proxy/src/main.rs +++ b/fluxer_media_proxy/src/main.rs @@ -2,7 +2,7 @@ use clap::Parser; use fluxer_media_proxy::{cli, healthcheck, run}; -use tracing_subscriber::{EnvFilter, layer::SubscriberExt, util::SubscriberInitExt}; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; #[tokio::main(flavor = "multi_thread")] async fn main() -> anyhow::Result<()> { @@ -12,7 +12,7 @@ async fn main() -> anyhow::Result<()> { } tracing_subscriber::registry() - .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) + .with(fluxer_common::config::env_filter("info")) .with(tracing_subscriber::fmt::layer().json()) .init(); diff --git a/fluxer_media_proxy/src/server/middleware.rs b/fluxer_media_proxy/src/server/middleware.rs index 1acf3bf3e..2c8043d33 100644 --- a/fluxer_media_proxy/src/server/middleware.rs +++ b/fluxer_media_proxy/src/server/middleware.rs @@ -125,10 +125,7 @@ pub(in crate::server) fn build_version() -> &'static str { static BUILD_VERSION: OnceLock = OnceLock::new(); BUILD_VERSION .get_or_init(|| { - std::env::var("BUILD_VERSION") - .ok() - .filter(|value| !value.trim().is_empty()) - .unwrap_or_else(|| "dev".to_owned()) + fluxer_common::config::env_value("BUILD_VERSION").unwrap_or_else(|| "dev".to_owned()) }) .as_str() } diff --git a/fluxer_media_proxy/src/storage/tests/mod.rs b/fluxer_media_proxy/src/storage/tests/mod.rs index aa3e3a781..7f05d3777 100644 --- a/fluxer_media_proxy/src/storage/tests/mod.rs +++ b/fluxer_media_proxy/src/storage/tests/mod.rs @@ -36,7 +36,6 @@ pub(crate) type CapturedRequest = (Method, http::Uri, HeaderMap, Bytes); fn test_config(root: &Path) -> Config { Config { - node_env: "test".to_owned(), bind_host: "127.0.0.1".to_owned(), port: 0, secret_key: SecretString::new("secret".to_owned()), diff --git a/fluxer_push/Cargo.toml b/fluxer_push/Cargo.toml index 335e502cc..76ab2acfc 100644 --- a/fluxer_push/Cargo.toml +++ b/fluxer_push/Cargo.toml @@ -34,5 +34,4 @@ sha2 = "0.11.0" thiserror = "2.0.20" tokio = { version = "1.53.1", features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] } tracing = "0.1.44" -tracing-subscriber = { version = "0.3.23", features = ["env-filter", "fmt", "json"] } url = "2.5.8" diff --git a/fluxer_push/src/healthcheck.rs b/fluxer_push/src/healthcheck.rs index 03970b5b7..89843b48b 100644 --- a/fluxer_push/src/healthcheck.rs +++ b/fluxer_push/src/healthcheck.rs @@ -3,25 +3,21 @@ use crate::config::Mode; use anyhow::Context as _; use std::{ - env, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, time::Duration, }; pub async fn run(mode: Mode) -> anyhow::Result<()> { let addr = target( - env::var("FLUXER_PUSH_SERVICE_HOST").ok().as_deref(), - env::var("FLUXER_PUSH_SERVICE_PORT").ok().as_deref(), + fluxer_svc::config::optional_env("FLUXER_PUSH_SERVICE_HOST").as_deref(), + fluxer_svc::config::optional_env("FLUXER_PUSH_SERVICE_PORT").as_deref(), mode, )?; probe(addr).await } fn target(host: Option<&str>, port: Option<&str>, mode: Mode) -> anyhow::Result { - let host = host - .map(str::trim) - .filter(|host| !host.is_empty()) - .unwrap_or("127.0.0.1"); + let host = host.map(str::trim).unwrap_or("127.0.0.1"); let ip = host .parse::() .with_context(|| format!("FLUXER_PUSH_SERVICE_HOST is not an IP address: {host}"))?; @@ -30,7 +26,7 @@ fn target(host: Option<&str>, port: Option<&str>, mode: Mode) -> anyhow::Result< IpAddr::V6(ip) if ip.is_unspecified() => IpAddr::V6(Ipv6Addr::LOCALHOST), ip => ip, }; - let port = match port.map(str::trim).filter(|port| !port.is_empty()) { + let port = match port.map(str::trim) { Some(port) => port .parse::() .with_context(|| format!("FLUXER_PUSH_SERVICE_PORT is not a port number: {port}"))?, diff --git a/fluxer_push/src/main.rs b/fluxer_push/src/main.rs index ebe611745..ed770e8c3 100644 --- a/fluxer_push/src/main.rs +++ b/fluxer_push/src/main.rs @@ -2,7 +2,6 @@ use clap::Parser; use fluxer_push::{cli, healthcheck, run}; -use tracing_subscriber::{EnvFilter, layer::SubscriberExt, util::SubscriberInitExt}; #[tokio::main(flavor = "multi_thread")] async fn main() -> anyhow::Result<()> { @@ -11,10 +10,7 @@ async fn main() -> anyhow::Result<()> { return healthcheck::run(args.mode).await; } - tracing_subscriber::registry() - .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) - .with(tracing_subscriber::fmt::layer().json()) - .init(); + fluxer_svc::init_tracing(); let cfg = cli::load_config(&args)?; run(cfg).await diff --git a/fluxer_svc/src/config.rs b/fluxer_svc/src/config.rs index b2d22d003..508e43844 100644 --- a/fluxer_svc/src/config.rs +++ b/fluxer_svc/src/config.rs @@ -20,7 +20,6 @@ pub struct ServiceConfig { pub nats_auth_token: Option, pub cache_max_entries: u64, pub cache_ttl: Duration, - pub cache_hard_ttl: Duration, pub max_concurrent_requests: usize, pub scylla_hosts: Vec, pub scylla_keyspace: String, @@ -114,12 +113,6 @@ impl ServiceConfig { .transpose()? .unwrap_or(30_000); - let cache_hard_ttl_ms = optional_from(&get, "FLUXER_SVC_CACHE_HARD_TTL_MS") - .map(|v| v.parse::()) - .transpose()? - .unwrap_or(600_000) - .max(cache_ttl_ms); - let cassandra_port = optional_from(&get, "FLUXER_CASSANDRA_PORT") .map(|v| v.parse::()) .transpose()? @@ -175,7 +168,6 @@ impl ServiceConfig { .transpose()? .unwrap_or(100_000), cache_ttl: Duration::from_millis(cache_ttl_ms), - cache_hard_ttl: Duration::from_millis(cache_hard_ttl_ms), max_concurrent_requests, scylla_hosts, scylla_keyspace: optional_from(&get, "FLUXER_CASSANDRA_KEYSPACE") @@ -218,7 +210,7 @@ fn optional_from(get: &F, name: &str) -> Option where F: Fn(&str) -> Option, { - get(name).filter(|v| !v.is_empty()) + get(name).filter(|v| !v.trim().is_empty()) } pub fn parse_hosts(hosts: &str) -> Vec { @@ -359,6 +351,33 @@ mod tests { assert!(cfg.postgres_prepared_statements); } + #[test] + fn blank_values_fall_back_to_the_defaults() { + let cfg = config_from_pairs(&[ + ("FLUXER_SVC_NAME", "messages"), + ("FLUXER_SVC_MODE", ""), + ("FLUXER_SVC_PORT", " "), + ("FLUXER_SVC_MAX_CONCURRENT_REQUESTS", ""), + ("FLUXER_NATS_AUTH_TOKEN", " "), + ("FLUXER_POSTGRES_HOST", ""), + ("FLUXER_POSTGRES_PASSWORD", " "), + ("FLUXER_POSTGRES_MAX_CONNECTIONS", ""), + ("FLUXER_POSTGRES_PREPARED_STATEMENTS", " "), + ]); + + assert_eq!(Mode::Router, cfg.mode); + assert_eq!(8090, cfg.listen_addr.port()); + assert_eq!( + MESSAGES_MAX_CONCURRENT_REQUESTS, + cfg.max_concurrent_requests + ); + assert_eq!(None, cfg.nats_auth_token); + assert_eq!("127.0.0.1", cfg.postgres_host); + assert_eq!(Some("fluxer".to_owned()), cfg.postgres_password); + assert_eq!(20, cfg.postgres_max_connections); + assert!(cfg.postgres_prepared_statements); + } + #[test] fn reads_postgres_database_env() { let cfg = config_from_pairs(&[ diff --git a/fluxer_svc/src/lib.rs b/fluxer_svc/src/lib.rs index 123ff13a5..eeaafd52d 100644 --- a/fluxer_svc/src/lib.rs +++ b/fluxer_svc/src/lib.rs @@ -20,7 +20,9 @@ pub fn init_tracing() { tracing_subscriber::fmt() .json() .with_env_filter( - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), + config::optional_env("RUST_LOG") + .and_then(|filter| EnvFilter::try_new(filter).ok()) + .unwrap_or_else(|| EnvFilter::new("info")), ) .try_init() .ok(); diff --git a/fluxer_svc/src/router.rs b/fluxer_svc/src/router.rs index 14e29651c..4c4cbfca7 100644 --- a/fluxer_svc/src/router.rs +++ b/fluxer_svc/src/router.rs @@ -942,7 +942,6 @@ mod tests { nats_auth_token: None, cache_max_entries: 100, cache_ttl: Duration::from_secs(30), - cache_hard_ttl: Duration::from_secs(600), max_concurrent_requests, scylla_hosts: Vec::new(), scylla_keyspace: "fluxer".to_owned(), diff --git a/fluxer_svc/src/server.rs b/fluxer_svc/src/server.rs index 512835e4c..cd93b63bb 100644 --- a/fluxer_svc/src/server.rs +++ b/fluxer_svc/src/server.rs @@ -81,10 +81,7 @@ fn build_version() -> &'static str { static BUILD_VERSION: OnceLock = OnceLock::new(); BUILD_VERSION .get_or_init(|| { - std::env::var("BUILD_VERSION") - .ok() - .filter(|v| !v.trim().is_empty()) - .unwrap_or_else(|| "dev".to_owned()) + crate::config::optional_env("BUILD_VERSION").unwrap_or_else(|| "dev".to_owned()) }) .as_str() } diff --git a/fluxer_svc/src/shard.rs b/fluxer_svc/src/shard.rs index 99d519ee2..d45ac26f6 100644 --- a/fluxer_svc/src/shard.rs +++ b/fluxer_svc/src/shard.rs @@ -436,7 +436,6 @@ mod tests { nats_auth_token: None, cache_max_entries: 100, cache_ttl: Duration::from_secs(30), - cache_hard_ttl: Duration::from_secs(600), max_concurrent_requests, scylla_hosts: Vec::new(), scylla_keyspace: "fluxer".to_owned(), diff --git a/fluxer_unfurl/src/resolvers/klipy.rs b/fluxer_unfurl/src/resolvers/klipy.rs index 4386d4ec6..0c8cc2e52 100644 --- a/fluxer_unfurl/src/resolvers/klipy.rs +++ b/fluxer_unfurl/src/resolvers/klipy.rs @@ -4,6 +4,7 @@ use super::{ResolveContext, Resolver, ResolverResult}; use crate::http_fetch; use crate::media_proxy::{MediaMetadata, embed_media_flags}; use crate::types::{EmbedMedia, EmbedProvider, MessageEmbed}; +use fluxer_svc::config::optional_env; use std::future::Future; use std::pin::Pin; use std::time::Duration; @@ -136,14 +137,7 @@ fn klipy_resource(kind: &str) -> &'static str { } fn klipy_api_key() -> Option { - std::env::var("FLUXER_KLIPY_API_KEY") - .ok() - .filter(|key| !key.is_empty()) - .or_else(|| { - std::env::var("KLIPY_API_KEY") - .ok() - .filter(|key| !key.is_empty()) - }) + optional_env("FLUXER_KLIPY_API_KEY").or_else(|| optional_env("KLIPY_API_KEY")) } async fn resolve_media_via_api( diff --git a/fluxer_unfurl/src/resolvers/youtube.rs b/fluxer_unfurl/src/resolvers/youtube.rs index 16e6f167d..634116e2a 100644 --- a/fluxer_unfurl/src/resolvers/youtube.rs +++ b/fluxer_unfurl/src/resolvers/youtube.rs @@ -5,8 +5,8 @@ use crate::http_fetch; use crate::media_proxy::embed_media_flags; use crate::text_limits; use crate::types::{EmbedAuthor, EmbedMedia, EmbedProvider, MessageEmbed}; +use fluxer_svc::config::optional_env; use serde::Deserialize; -use std::env; use std::future::Future; use std::pin::Pin; use std::time::Duration; @@ -252,14 +252,7 @@ struct YouTubeThumbnail { } fn youtube_api_key() -> Option { - env::var("FLUXER_YOUTUBE_API_KEY") - .ok() - .filter(|key| !key.is_empty()) - .or_else(|| { - env::var("YOUTUBE_API_KEY") - .ok() - .filter(|key| !key.is_empty()) - }) + optional_env("FLUXER_YOUTUBE_API_KEY").or_else(|| optional_env("YOUTUBE_API_KEY")) } fn build_youtube_api_url(video_id: &str, api_key: &str) -> anyhow::Result { diff --git a/fluxer_unfurl/src/shard_impl.rs b/fluxer_unfurl/src/shard_impl.rs index 91e749f68..eff43cb28 100644 --- a/fluxer_unfurl/src/shard_impl.rs +++ b/fluxer_unfurl/src/shard_impl.rs @@ -8,6 +8,7 @@ use crate::embed_normalizer::normalize_embeds; use crate::media_proxy::MediaProxyClient; use crate::resolvers::{self, ResolveContext, ResolverResult}; use crate::types::{InvalidatedResponse, NsfwMode, UnfurlRequest, UnfurlResponse, UnfurlResult}; +use fluxer_svc::config::optional_env; use fluxer_svc::shard::ShardService; use moka::future::Cache; use std::sync::Arc; @@ -32,19 +33,13 @@ impl UnfurlShard { let resolvers = resolvers::build_resolver_chain(); - let media_proxy_endpoint = std::env::var("FLUXER_MEDIA_PROXY_ENDPOINT") - .ok() - .filter(|v| !v.is_empty()); - let media_proxy_secret = std::env::var("FLUXER_MEDIA_PROXY_SECRET_KEY") - .ok() - .filter(|v| !v.is_empty()); - let media_proxy_public_endpoint = std::env::var("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT") - .ok() - .filter(|v| !v.is_empty()) + let media_proxy_endpoint = optional_env("FLUXER_MEDIA_PROXY_ENDPOINT"); + let media_proxy_secret = optional_env("FLUXER_MEDIA_PROXY_SECRET_KEY"); + let media_proxy_public_endpoint = optional_env("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT") .map(|v| fluxer_common::config::normalize_public_endpoint_from_env(&v)); let static_cdn_endpoint = fluxer_common::config::normalize_public_endpoint_from_env( - &std::env::var("FLUXER_UNFURL_STATIC_CDN_ENDPOINT") - .or_else(|_| std::env::var("FLUXER_STATIC_CDN_ENDPOINT")) + &optional_env("FLUXER_UNFURL_STATIC_CDN_ENDPOINT") + .or_else(|| optional_env("FLUXER_STATIC_CDN_ENDPOINT")) .unwrap_or_default(), ); let (media_proxy_endpoint, media_proxy_secret) = match ( diff --git a/packages/config/src/ConfigLoader.ts b/packages/config/src/ConfigLoader.ts index bf2c203ad..78b58828c 100644 --- a/packages/config/src/ConfigLoader.ts +++ b/packages/config/src/ConfigLoader.ts @@ -2,7 +2,7 @@ import {createECDH} from 'node:crypto'; import {isConfigObject} from '@fluxer/config/src/config_loader/ConfigObject'; -import {buildNamedFluxerEnvOverrides} from '@fluxer/config/src/config_loader/EnvironmentOverrides'; +import {buildNamedFluxerEnvOverrides, readEnvValue} from '@fluxer/config/src/config_loader/EnvironmentOverrides'; import { buildUrl, type DerivedEndpoints, @@ -31,9 +31,7 @@ function defaultConfig(): MasterConfig { base_domain: '', public_origin: '', public_scheme: 'http', - internal_scheme: 'http', public_port: 8088, - internal_port: 8088, static_cdn_domain: '', invite_domain: '', gift_domain: '', @@ -46,18 +44,13 @@ function defaultConfig(): MasterConfig { media: '', static_cdn: '', admin: '', - docs: '', marketing: '', invite: '', gift: '', }, internal: { kv: 'redis://localhost:6379/0', - kv_provider: 'redis', kv_mode: 'standalone', - kv_cluster_nodes: [], - kv_cluster_nat_map: {}, - api: 'http://127.0.0.1:8080', media_proxy: 'http://127.0.0.1:8082', }, database: { @@ -109,18 +102,6 @@ function defaultConfig(): MasterConfig { presigned_harvest_downloads_enabled: true, unfurl_ignored_hosts: [], app_origin_aliases: [], - embeds: { - oembed_html_enabled: false, - oembed_html_allow_untrusted_on_self_hosted: false, - oembed_html_allowed_hosts: [], - cache_default_ttl_seconds: 86_400, - cache_max_ttl_seconds: 604_800, - cache_min_ttl_seconds: 300, - cache_respect_remote_ttl: true, - }, - content_moderation: { - nsfw_threshold: 0.7, - }, storage_change_feed: { enabled: false, stream: 'STORAGE_CHANGES', @@ -132,10 +113,7 @@ function defaultConfig(): MasterConfig { auth_token: '', }, media_proxy: { - host: '0.0.0.0', - port: 8082, secret_key: '', - mode: 'upload', upload_relay: { endpoint: 'http://localhost:8088/media', secret_base64: '', @@ -148,19 +126,12 @@ function defaultConfig(): MasterConfig { }, }, gateway: { - port: 8771, rpc_auth_token: '', }, admin: { - port: 3020, - base_path: '/admin', secret_key_base: '', oauth_client_secret: '', }, - app_proxy: { - port: 8773, - assets_dir: 'fluxer_app/dist', - }, }, auth: { sudo_mode_secret: '', @@ -200,7 +171,6 @@ function defaultConfig(): MasterConfig { api_secret: '', url: '', internal_url: '', - webhook_url: '', }, search: { engine: 'elasticsearch', @@ -253,10 +223,6 @@ function defaultConfig(): MasterConfig { enabled: false, apps: [], }, - fcm: { - enabled: false, - apps: [], - }, }, }, instance: { @@ -314,13 +280,10 @@ function requireString(value: string | undefined, envName: string): void { } } -function validateUploadRelaySecret(value: string, mode: string): void { +function validateUploadRelaySecret(value: string): void { const trimmed = value.trim(); if (trimmed.length === 0) { - if (mode === 'upload') { - throw new Error('FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 is required in upload mode'); - } - return; + throw new Error('FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 is required'); } if (!/^[A-Za-z0-9+/]+={0,2}$/u.test(trimmed)) { throw new Error('FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 must be base64'); @@ -537,9 +500,7 @@ function validatePublicEndpoints(endpoints: DerivedEndpoints): void { function normalizeConfig(config: MasterConfig): MasterConfig { assertOneOf(config.env, ['development', 'production', 'test'], 'FLUXER_ENV'); assertOneOf(config.domain.public_scheme, ['http', 'https'], 'FLUXER_PUBLIC_SCHEME'); - assertOneOf(config.domain.internal_scheme, ['http', 'https'], 'FLUXER_INTERNAL_SCHEME'); assertOneOf(config.database.backend, ['postgres', 'cassandra'], 'FLUXER_DATABASE_BACKEND'); - assertOneOf(config.internal.kv_provider, ['redis'], 'FLUXER_KV_PROVIDER'); assertOneOf(config.internal.kv_mode, ['standalone', 'cluster'], 'FLUXER_KV_MODE'); assertOneOf(config.integrations.email.provider, ['smtp', 'none'], 'FLUXER_EMAIL_PROVIDER'); assertOneOf(config.integrations.search.engine, ['elasticsearch', 'meilisearch'], 'FLUXER_SEARCH_ENGINE'); @@ -563,7 +524,7 @@ function normalizeConfig(config: MasterConfig): MasterConfig { requireString(config.s3?.access_key_id, 'FLUXER_S3_ACCESS_KEY_ID'); requireString(config.s3?.secret_access_key, 'FLUXER_S3_SECRET_ACCESS_KEY'); requireString(config.services.media_proxy.secret_key, 'FLUXER_MEDIA_PROXY_SECRET_KEY'); - validateUploadRelaySecret(config.services.media_proxy.upload_relay.secret_base64, config.services.media_proxy.mode); + validateUploadRelaySecret(config.services.media_proxy.upload_relay.secret_base64); validateAttachmentUrlSecrets(config.services.media_proxy.attachment_urls.secrets_base64); requireString(config.services.admin.secret_key_base, 'FLUXER_ADMIN_SECRET_KEY_BASE'); requireString(config.services.admin.oauth_client_secret, 'FLUXER_ADMIN_OAUTH_CLIENT_SECRET'); @@ -622,10 +583,6 @@ function applyPublicPort(config: MasterConfig, endpoints: DerivedEndpoints): Mas endpoint: normalize(config.services.media_proxy.upload_relay.endpoint), }, }, - gateway: { - ...config.services.gateway, - media_proxy_endpoint: normalizeOptional(config.services.gateway.media_proxy_endpoint), - }, }, auth: { ...config.auth, @@ -703,7 +660,10 @@ export async function loadConfig(): Promise { const endpoints = {...derived, ...(normalized.endpoint_overrides ?? {})}; validatePublicEndpoints(endpoints); const withPublicPort = applyPublicPort(normalized, endpoints); - normalizePasskeys(withPublicPort, process.env.FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS === undefined); + normalizePasskeys( + withPublicPort, + readEnvValue(process.env, 'FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS') === undefined, + ); cachedConfig = withPublicPort; return cachedConfig; } diff --git a/packages/config/src/EndpointDerivation.ts b/packages/config/src/EndpointDerivation.ts index 8bef09ada..8013f7cdc 100644 --- a/packages/config/src/EndpointDerivation.ts +++ b/packages/config/src/EndpointDerivation.ts @@ -1,13 +1,9 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -const DOCS_ENDPOINT = 'https://fluxer.dev'; - export interface DomainConfig { base_domain: string; public_scheme: 'http' | 'https'; - internal_scheme: 'http' | 'https'; public_port?: number; - internal_port?: number; static_cdn_domain?: string; invite_domain?: string; gift_domain?: string; @@ -29,7 +25,6 @@ export interface DerivedEndpoints { media: string; static_cdn: string; admin: string; - docs: string; marketing: string; invite: string; gift: string; @@ -129,7 +124,6 @@ export function deriveDomain( | 'media' | 'static_cdn' | 'admin' - | 'docs' | 'marketing' | 'invite' | 'gift', @@ -160,7 +154,6 @@ export function deriveEndpointsFromDomain(config: DomainConfig): DerivedEndpoint ? buildUrl('https', deriveDomain('static_cdn', config), undefined) : buildUrl(public_scheme, deriveDomain('static_cdn', config), public_port), admin: buildUrl(public_scheme, deriveDomain('admin', config), public_port, '/admin'), - docs: DOCS_ENDPOINT, marketing: buildUrl(public_scheme, deriveDomain('marketing', config), public_port, '/marketing'), invite: buildUrl(public_scheme, deriveDomain('invite', config), public_port, '/invite'), gift: buildUrl(public_scheme, deriveDomain('gift', config), public_port, '/gift'), diff --git a/packages/config/src/MasterConfig.ts b/packages/config/src/MasterConfig.ts index e33307a22..b87ae40d7 100644 --- a/packages/config/src/MasterConfig.ts +++ b/packages/config/src/MasterConfig.ts @@ -26,9 +26,7 @@ export interface MasterConfig { base_domain: string; public_origin: string; public_scheme: PublicScheme; - internal_scheme: PublicScheme; public_port: number; - internal_port: number; static_cdn_domain: string; invite_domain: string; gift_domain: string; @@ -37,12 +35,7 @@ export interface MasterConfig { endpoints: DerivedEndpoints; internal: { kv: string; - kv_provider: 'redis'; kv_mode: 'standalone' | 'cluster'; - kv_cluster_nodes: Array<{host: string; port: number}>; - kv_cluster_nat_map: Record; - api: string; - gateway?: string; media_proxy: string; }; database: { @@ -95,18 +88,6 @@ export interface MasterConfig { presigned_harvest_downloads_enabled: boolean; unfurl_ignored_hosts: Array; app_origin_aliases: Array; - embeds: { - oembed_html_enabled: boolean; - oembed_html_allow_untrusted_on_self_hosted: boolean; - oembed_html_allowed_hosts: Array; - cache_default_ttl_seconds: number; - cache_max_ttl_seconds: number; - cache_min_ttl_seconds: number; - cache_respect_remote_ttl: boolean; - }; - content_moderation?: { - nsfw_threshold?: number; - }; worker?: { mode?: 'all_lanes' | 'single_lane' | 'single_task'; lane?: 'realtime' | 'unfurl' | 'lifecycle' | 'batch'; @@ -131,10 +112,7 @@ export interface MasterConfig { auth_token?: string; }; media_proxy: { - host: string; - port: number; secret_key: string; - mode: string; upload_relay: { endpoint: string; secret_base64: string; @@ -147,21 +125,12 @@ export interface MasterConfig { }; }; gateway: { - port: number; rpc_auth_token?: string; - media_proxy_endpoint?: string; - api_rpc_endpoint?: string; }; admin: { - port: number; - base_path: string; secret_key_base: string; oauth_client_secret: string; }; - app_proxy: { - port: number; - assets_dir: string; - }; }; auth: { sudo_mode_secret: string; @@ -213,7 +182,6 @@ export interface MasterConfig { api_secret: string; url: string; internal_url: string; - webhook_url: string; default_region?: { id: string; name: string; @@ -280,27 +248,10 @@ export interface MasterConfig { key_id?: string; private_key?: string; private_key_path?: string; - default_environment?: 'production' | 'development'; apps?: Array<{ app_id?: string; topic?: string; environment?: 'production' | 'development'; - project_id?: string; - }>; - }; - fcm: { - enabled: boolean; - project_id?: string; - client_email?: string; - private_key?: string; - private_key_path?: string; - service_account_json_path?: string; - token_uri?: string; - apps?: Array<{ - app_id?: string; - topic?: string; - environment?: 'production' | 'development'; - project_id?: string; }>; }; }; diff --git a/packages/config/src/__tests__/ConfigLoader.test.ts b/packages/config/src/__tests__/ConfigLoader.test.ts index 9913301f8..784bb3671 100644 --- a/packages/config/src/__tests__/ConfigLoader.test.ts +++ b/packages/config/src/__tests__/ConfigLoader.test.ts @@ -22,8 +22,6 @@ const MINIMAL_ENV: Record = { FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64: 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=', FLUXER_ADMIN_SECRET_KEY_BASE: 'test-admin-secret', FLUXER_ADMIN_OAUTH_CLIENT_SECRET: 'test-admin-oauth-secret', - FLUXER_APP_PROXY_PORT: '8773', - FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: 'http://127.0.0.1:8088/media', FLUXER_GATEWAY_RPC_AUTH_TOKEN: 'test-gateway-token', FLUXER_SUDO_MODE_SECRET: 'test-sudo-secret', FLUXER_CONNECTION_INITIATION_SECRET: 'test-connection-secret', @@ -243,9 +241,9 @@ describe('ConfigLoader', () => { }, ); - test('rejects an empty client API endpoint override', async () => { + test('an empty client API endpoint override falls back to the derived endpoint', async () => { stubMinimalEnv({FLUXER_API_CLIENT_ENDPOINT: ''}); - await expect(loadConfig()).rejects.toThrow('FLUXER_API_CLIENT_ENDPOINT is required'); + expect((await loadConfig()).endpoints.api_client).toBe('http://localhost:8088/api'); }); test('defaults the passkey relying party to the deployment domain', async () => { @@ -260,17 +258,36 @@ describe('ConfigLoader', () => { expect(config.auth.passkeys.rp_id).toBe('chat.example.com'); }); - test('uses only the app origin when the operator clears the default list', async () => { + test('a blank origin list keeps the built-in origins', async () => { stubMinimalEnv({ FLUXER_BASE_DOMAIN: 'chat.example.com', FLUXER_PUBLIC_SCHEME: 'https', FLUXER_PUBLIC_PORT: '443', - FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS: '', + FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS: ' ', }); const config = await loadConfig(); - expect(config.auth.passkeys.additional_allowed_origins).toEqual(['https://chat.example.com']); + expect(config.auth.passkeys.additional_allowed_origins).toContain('https://web.fluxer.app'); + expect(config.auth.passkeys.additional_allowed_origins).toContain('https://chat.example.com'); + }); + + test('blank values fall back to the code defaults', async () => { + stubMinimalEnv({ + FLUXER_API_PORT: '', + FLUXER_EMAIL_FROM_NAME: '', + FLUXER_KV_URL: ' ', + FLUXER_S3_FORCE_PATH_STYLE: '', + FLUXER_API_STORAGE_CHANGE_FEED_SKIP_BUCKETS: '', + }); + + const config = await loadConfig(); + + expect(config.services.api.port).toBe(8080); + expect(config.integrations.email.from_name).toBe('Fluxer'); + expect(config.internal.kv).toBe('redis://localhost:6379/0'); + expect(config.s3?.force_path_style).toBe(false); + expect(config.services.api.storage_change_feed?.skip_buckets).toBeUndefined(); }); test('keeps explicit passkey relying party values', async () => { @@ -665,13 +682,11 @@ describe('ConfigLoader', () => { expect((await loadConfig()).services.media_proxy.upload_relay.max_body_bytes).toBe(524_288_000); }); - test('rejects a missing upload relay secret in upload mode', async () => { + test('rejects a missing upload relay secret', async () => { stubMinimalEnv(); vi.stubEnv('FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64', ''); - await expect(loadConfig()).rejects.toThrow( - 'FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 is required in upload mode', - ); + await expect(loadConfig()).rejects.toThrow('FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 is required'); }); test('rejects a non-base64 upload relay secret', async () => { @@ -686,15 +701,6 @@ describe('ConfigLoader', () => { ); }); - test('leaves the upload relay secret optional outside upload mode', async () => { - stubMinimalEnv({FLUXER_MEDIA_PROXY_MODE: 'mp'}); - vi.stubEnv('FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64', ''); - - const config = await loadConfig(); - - expect(config.services.media_proxy.upload_relay.secret_base64).toBe(''); - }); - test('rejects a VAPID public key that is not a 65-byte uncompressed point', async () => { const {privateKey} = generateVapidPair(); stubMinimalEnv({ @@ -753,7 +759,6 @@ describe('ConfigLoader', () => { test('inserts the public port into every other public url the config carries', async () => { stubMinimalEnv({ - FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: 'http://localhost/media', FLUXER_S3_PUBLIC_ENDPOINT: 'http://localhost/s3', FLUXER_EMAIL_APP_BASE_URL: 'http://localhost', FLUXER_AUTH_BLUESKY_CLIENT_URI: 'http://localhost', @@ -764,7 +769,6 @@ describe('ConfigLoader', () => { const config = await loadConfig(); - expect(config.services.gateway.media_proxy_endpoint).toBe('http://localhost:8088/media'); expect(config.s3?.presigned_url_base).toBe('http://localhost:8088/s3'); expect(config.integrations.email.app_base_url).toBe('http://localhost:8088'); expect(config.auth.bluesky.client_uri).toBe('http://localhost:8088'); diff --git a/packages/config/src/__tests__/EndpointDerivation.test.ts b/packages/config/src/__tests__/EndpointDerivation.test.ts index 9fb247a3f..02d6fd7c2 100644 --- a/packages/config/src/__tests__/EndpointDerivation.test.ts +++ b/packages/config/src/__tests__/EndpointDerivation.test.ts @@ -34,7 +34,6 @@ describe('deriveDomain', () => { const baseConfig: DomainConfig = { base_domain: 'fluxer.dev', public_scheme: 'https', - internal_scheme: 'http', }; test.each([ 'api', @@ -44,7 +43,6 @@ describe('deriveDomain', () => { 'media', 'static_cdn', 'admin', - 'docs', 'marketing', 'invite', 'gift', @@ -72,9 +70,7 @@ const endpointScenarios: Array = [ config: { base_domain: 'localhost', public_scheme: 'http', - internal_scheme: 'http', public_port: 8088, - internal_port: 8088, }, expected: { api: 'http://localhost:8088/api', @@ -84,7 +80,6 @@ const endpointScenarios: Array = [ media: 'http://localhost:8088/media', static_cdn: 'http://localhost:8088', admin: 'http://localhost:8088/admin', - docs: 'https://fluxer.dev', marketing: 'http://localhost:8088/marketing', invite: 'http://localhost:8088/invite', gift: 'http://localhost:8088/gift', @@ -95,9 +90,7 @@ const endpointScenarios: Array = [ config: { base_domain: 'fluxer.app', public_scheme: 'https', - internal_scheme: 'http', public_port: 443, - internal_port: 8080, }, expected: { api: 'https://fluxer.app/api', @@ -107,7 +100,6 @@ const endpointScenarios: Array = [ media: 'https://fluxer.app/media', static_cdn: 'https://fluxer.app', admin: 'https://fluxer.app/admin', - docs: 'https://fluxer.dev', marketing: 'https://fluxer.app/marketing', invite: 'https://fluxer.app/invite', gift: 'https://fluxer.app/gift', @@ -118,9 +110,7 @@ const endpointScenarios: Array = [ config: { base_domain: 'staging.fluxer.dev', public_scheme: 'https', - internal_scheme: 'http', public_port: 8443, - internal_port: 8080, }, expected: { api: 'https://staging.fluxer.dev:8443/api', @@ -130,7 +120,6 @@ const endpointScenarios: Array = [ media: 'https://staging.fluxer.dev:8443/media', static_cdn: 'https://staging.fluxer.dev:8443', admin: 'https://staging.fluxer.dev:8443/admin', - docs: 'https://fluxer.dev', marketing: 'https://staging.fluxer.dev:8443/marketing', invite: 'https://staging.fluxer.dev:8443/invite', gift: 'https://staging.fluxer.dev:8443/gift', @@ -141,7 +130,6 @@ const endpointScenarios: Array = [ config: { base_domain: 'fluxer.app', public_scheme: 'https', - internal_scheme: 'http', public_port: 443, static_cdn_domain: 'cdn.fluxer.app', }, @@ -153,7 +141,6 @@ const endpointScenarios: Array = [ media: 'https://fluxer.app/media', static_cdn: 'https://cdn.fluxer.app', admin: 'https://fluxer.app/admin', - docs: 'https://fluxer.dev', marketing: 'https://fluxer.app/marketing', invite: 'https://fluxer.app/invite', gift: 'https://fluxer.app/gift', @@ -164,7 +151,6 @@ const endpointScenarios: Array = [ config: { base_domain: 'fluxer.app', public_scheme: 'https', - internal_scheme: 'http', public_port: 443, invite_domain: 'fluxer.gg', gift_domain: 'fluxer.gift', @@ -177,7 +163,6 @@ const endpointScenarios: Array = [ media: 'https://fluxer.app/media', static_cdn: 'https://fluxer.app', admin: 'https://fluxer.app/admin', - docs: 'https://fluxer.dev', marketing: 'https://fluxer.app/marketing', invite: 'https://fluxer.gg/invite', gift: 'https://fluxer.gift/gift', @@ -188,7 +173,6 @@ const endpointScenarios: Array = [ config: { base_domain: 'canary.fluxer.app', public_scheme: 'https', - internal_scheme: 'http', public_port: 443, static_cdn_domain: 'cdn-canary.fluxer.app', }, @@ -200,7 +184,6 @@ const endpointScenarios: Array = [ media: 'https://canary.fluxer.app/media', static_cdn: 'https://cdn-canary.fluxer.app', admin: 'https://canary.fluxer.app/admin', - docs: 'https://fluxer.dev', marketing: 'https://canary.fluxer.app/marketing', invite: 'https://canary.fluxer.app/invite', gift: 'https://canary.fluxer.app/gift', @@ -211,7 +194,6 @@ const endpointScenarios: Array = [ config: { base_domain: 'example.com', public_scheme: 'http', - internal_scheme: 'http', public_port: 80, }, expected: { @@ -222,7 +204,6 @@ const endpointScenarios: Array = [ media: 'http://example.com/media', static_cdn: 'http://example.com', admin: 'http://example.com/admin', - docs: 'https://fluxer.dev', marketing: 'http://example.com/marketing', invite: 'http://example.com/invite', gift: 'http://example.com/gift', @@ -233,7 +214,6 @@ const endpointScenarios: Array = [ config: { base_domain: 'example.com', public_scheme: 'https', - internal_scheme: 'http', }, expected: { api: 'https://example.com/api', @@ -243,7 +223,6 @@ const endpointScenarios: Array = [ media: 'https://example.com/media', static_cdn: 'https://example.com', admin: 'https://example.com/admin', - docs: 'https://fluxer.dev', marketing: 'https://example.com/marketing', invite: 'https://example.com/invite', gift: 'https://example.com/gift', @@ -254,7 +233,6 @@ const endpointScenarios: Array = [ config: { base_domain: '127.0.0.1', public_scheme: 'http', - internal_scheme: 'http', public_port: 8088, }, expected: { @@ -265,7 +243,6 @@ const endpointScenarios: Array = [ media: 'http://127.0.0.1:8088/media', static_cdn: 'http://127.0.0.1:8088', admin: 'http://127.0.0.1:8088/admin', - docs: 'https://fluxer.dev', marketing: 'http://127.0.0.1:8088/marketing', invite: 'http://127.0.0.1:8088/invite', gift: 'http://127.0.0.1:8088/gift', @@ -276,7 +253,6 @@ const endpointScenarios: Array = [ config: { base_domain: 'localhost', public_scheme: 'http', - internal_scheme: 'http', public_port: 8088, static_cdn_domain: 'cdn.example.com', }, @@ -288,7 +264,6 @@ const endpointScenarios: Array = [ media: 'http://localhost:8088/media', static_cdn: 'https://cdn.example.com', admin: 'http://localhost:8088/admin', - docs: 'https://fluxer.dev', marketing: 'http://localhost:8088/marketing', invite: 'http://localhost:8088/invite', gift: 'http://localhost:8088/gift', @@ -469,10 +444,7 @@ describe('endpoints derived from a public origin', () => { test('an origin with a non-standard port ports every derived endpoint', () => { const origin = parsePublicOrigin('https://chat.example.com:29080'); assert.ok(origin); - const endpoints = deriveEndpointsFromDomain({ - ...origin, - internal_scheme: 'http', - }); + const endpoints = deriveEndpointsFromDomain(origin); expect(endpoints.api_client).toBe('https://chat.example.com:29080/api'); expect(endpoints.app).toBe('https://chat.example.com:29080'); expect(endpoints.gateway).toBe('wss://chat.example.com:29080/gateway'); @@ -481,10 +453,7 @@ describe('endpoints derived from a public origin', () => { test('an origin written with an explicit :443 derives portless endpoints', () => { const origin = parsePublicOrigin('https://chat.example.com:443'); assert.ok(origin); - const endpoints = deriveEndpointsFromDomain({ - ...origin, - internal_scheme: 'http', - }); + const endpoints = deriveEndpointsFromDomain(origin); expect(endpoints.admin).toBe('https://chat.example.com/admin'); expect(endpoints.app).toBe('https://chat.example.com'); expect(endpoints.gateway).toBe('wss://chat.example.com/gateway'); diff --git a/packages/config/src/__tests__/EnvironmentOverrides.test.ts b/packages/config/src/__tests__/EnvironmentOverrides.test.ts index 9881f52f4..a19f192a6 100644 --- a/packages/config/src/__tests__/EnvironmentOverrides.test.ts +++ b/packages/config/src/__tests__/EnvironmentOverrides.test.ts @@ -1,45 +1,8 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import { - buildNamedFluxerEnvOverrides, - parseEnvValue, - setNestedValue, -} from '@fluxer/config/src/config_loader/EnvironmentOverrides'; +import {buildNamedFluxerEnvOverrides, setNestedValue} from '@fluxer/config/src/config_loader/EnvironmentOverrides'; import {describe, expect, test} from 'vitest'; -describe('parseEnvValue', () => { - test('parses boolean true', () => { - expect(parseEnvValue('true')).toBe(true); - expect(parseEnvValue(' true ')).toBe(true); - }); - test('parses boolean false', () => { - expect(parseEnvValue('false')).toBe(false); - expect(parseEnvValue(' false ')).toBe(false); - }); - test('parses integers', () => { - expect(parseEnvValue('42')).toBe(42); - expect(parseEnvValue('-7')).toBe(-7); - expect(parseEnvValue('0')).toBe(0); - }); - test('parses floats', () => { - expect(parseEnvValue('3.14')).toBe(3.14); - expect(parseEnvValue('-0.5')).toBe(-0.5); - }); - test('parses JSON objects', () => { - expect(parseEnvValue('{"key": "value"}')).toEqual({key: 'value'}); - }); - test('parses JSON arrays', () => { - expect(parseEnvValue('[1, 2, 3]')).toEqual([1, 2, 3]); - }); - test('rejects invalid JSON-like values', () => { - expect(() => parseEnvValue('{not json}')).toThrow('must be valid JSON'); - }); - test('returns raw string for plain strings', () => { - expect(parseEnvValue('hello')).toBe('hello'); - expect(parseEnvValue('localhost')).toBe('localhost'); - }); -}); - describe('setNestedValue', () => { test('sets a top-level key', () => { const target: Record = {}; @@ -81,14 +44,13 @@ describe('setNestedValue', () => { }); describe('buildNamedFluxerEnvOverrides', () => { - test('builds canonical split env overrides and preserves empty strings', () => { + test('builds canonical split env overrides', () => { const overrides = buildNamedFluxerEnvOverrides({ FLUXER_BASE_DOMAIN: 'canonical.example', FLUXER_API_ENDPOINT: 'https://canonical.example/api', FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS: 'https://a.example, https://b.example', FLUXER_S3_FORCE_PATH_STYLE: 'true', FLUXER_AUTH_BLUESKY_KEYS: '[{"kid":"key-1","private_key_path":"/etc/fluxer/keys/bluesky.pem"}]', - FLUXER_ADMIN_BASE_PATH: '', FLUXER_STRIPE_PRICE_MONTHLY_USD: 'price_monthly_usd', }); @@ -100,17 +62,46 @@ describe('buildNamedFluxerEnvOverrides', () => { bluesky: {keys: [{kid: 'key-1', private_key_path: '/etc/fluxer/keys/bluesky.pem'}]}, }, s3: {force_path_style: true}, - services: { - admin: {base_path: ''}, - }, integrations: {stripe: {prices: {monthly_usd: 'price_monthly_usd'}}}, }); }); - test('maps the internal scheme and KV provider names', () => { - expect(buildNamedFluxerEnvOverrides({FLUXER_INTERNAL_SCHEME: 'https', FLUXER_KV_PROVIDER: 'redis'})).toMatchObject({ - domain: {internal_scheme: 'https'}, - internal: {kv_provider: 'redis'}, + test.each(['', ' ', '\t\n'])('treats %j as unset for every value type', (blank) => { + expect( + buildNamedFluxerEnvOverrides({ + FLUXER_BASE_DOMAIN: blank, + FLUXER_API_PORT: blank, + FLUXER_S3_FORCE_PATH_STYLE: blank, + FLUXER_API_IP_BAN_EXEMPT_IPS: blank, + FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS: blank, + FLUXER_LIVEKIT_DEFAULT_REGION: blank, + FLUXER_AUTH_BLUESKY_KEYS: blank, + FLUXER_EMAIL_FROM_NAME: blank, + }), + ).toEqual({}); + }); + + test('a blank canonical name falls through to its alias', () => { + expect( + buildNamedFluxerEnvOverrides({ + FLUXER_NATS_URL: '', + FLUXER_NATS_CORE_URL: 'nats://alias', + FLUXER_IPINFO_API_KEY: ' ', + FLUXER_RISK_IPINFO_API_KEY: 'alias-key', + }), + ).toMatchObject({ + services: {nats: {core_url: 'nats://alias'}}, + integrations: {ipinfo: {api_key: 'alias-key'}}, + }); + }); + + test('a blank alias is unset too', () => { + expect(buildNamedFluxerEnvOverrides({FLUXER_NATS_URL: '', FLUXER_NATS_CORE_URL: ' '})).toEqual({}); + }); + + test('none clears the storage change feed skip list', () => { + expect(buildNamedFluxerEnvOverrides({FLUXER_API_STORAGE_CHANGE_FEED_SKIP_BUCKETS: ' None '})).toEqual({ + services: {api: {storage_change_feed: {skip_buckets: []}}}, }); }); @@ -120,10 +111,6 @@ describe('buildNamedFluxerEnvOverrides', () => { ); }); - test('leaves the default in place for a blank integer override', () => { - expect(buildNamedFluxerEnvOverrides({FLUXER_API_PORT: ''})).toEqual({}); - }); - test('the canonical name wins over its alias regardless of declaration order', () => { expect( buildNamedFluxerEnvOverrides({ diff --git a/packages/config/src/__tests__/SelfHostingCompose.ts b/packages/config/src/__tests__/SelfHostingCompose.ts index e146743f5..4d59fb486 100644 --- a/packages/config/src/__tests__/SelfHostingCompose.ts +++ b/packages/config/src/__tests__/SelfHostingCompose.ts @@ -42,8 +42,6 @@ export function serviceEnvironment(name: string): Record { return value === undefined ? {} : environment(value, `services.${name}.environment`); } -export const sharedEnvironment = environment(compose['x-fluxer-env'], 'x-fluxer-env'); - export function serviceList(name: string, key: string): Array { const value = composeService(name)[key]; assert.ok(Array.isArray(value), `services.${name}.${key} must be a list`); diff --git a/packages/config/src/__tests__/SelfHostingComposeEnvironment.test.ts b/packages/config/src/__tests__/SelfHostingComposeEnvironment.test.ts index 76c088609..3b432d76e 100644 --- a/packages/config/src/__tests__/SelfHostingComposeEnvironment.test.ts +++ b/packages/config/src/__tests__/SelfHostingComposeEnvironment.test.ts @@ -1,28 +1,90 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import {serviceEnvironment, serviceNames, sharedEnvironment} from '@fluxer/config/src/__tests__/SelfHostingCompose'; +import {serviceEnvironment, serviceNames} from '@fluxer/config/src/__tests__/SelfHostingCompose'; +import {NAMED_FLUXER_ENV_NAMES} from '@fluxer/config/src/config_loader/EnvironmentOverrides'; import {describe, expect, test} from 'vitest'; -describe('the shipped compose stack wires every service it starts', () => { - test('no service sizes a pool for a connection it cannot make', () => { - const pooledServices = serviceNames.filter((name) => 'FLUXER_POSTGRES_MAX_CONNECTIONS' in serviceEnvironment(name)); - expect(pooledServices.length).toBeGreaterThan(0); - for (const name of pooledServices) { - expect(serviceEnvironment(name), name).toMatchObject({ - FLUXER_DATABASE_BACKEND: 'postgres', - FLUXER_POSTGRES_HOST: expect.stringMatching(/\S/u), - FLUXER_POSTGRES_PORT: expect.stringMatching(/\S/u), - FLUXER_POSTGRES_DATABASE: expect.stringMatching(/\S/u), - FLUXER_POSTGRES_USERNAME: expect.stringMatching(/\S/u), - FLUXER_POSTGRES_PASSWORD: expect.stringMatching(/\S/u), - }); - } +const API_SETTINGS_NOT_FORWARDED: Record = { + FLUXER_CASSANDRA_HOSTS: 'the stack runs Postgres only', + FLUXER_CASSANDRA_PORT: 'the stack runs Postgres only', + FLUXER_CASSANDRA_KEYSPACE: 'the stack runs Postgres only', + FLUXER_CASSANDRA_LOCAL_DC: 'the stack runs Postgres only', + FLUXER_CASSANDRA_USERNAME: 'the stack runs Postgres only', + FLUXER_CASSANDRA_PASSWORD: 'the stack runs Postgres only', + FLUXER_API_WORKER_MODE: 'the one worker container runs every lane', + FLUXER_API_WORKER_LANE: 'the one worker container runs every lane', + FLUXER_API_WORKER_TASK: 'the one worker container runs every lane', + FLUXER_API_WORKER_ENABLE_CRON_SCHEDULER: 'the one worker container hosts cron', + FLUXER_RELAX_REGISTRATION_RATE_LIMITS: 'test and development only', + FLUXER_DISABLE_RATE_LIMITS: 'test and development only', + FLUXER_TEST_MODE_ENABLED: 'test and development only', + FLUXER_TEST_HARNESS_TOKEN: 'test and development only', + FLUXER_VALIDATE_RESPONSES: 'test and development only', + ...Object.fromEntries( + ['MONTHLY', 'YEARLY', 'GIFT_1_MONTH', 'GIFT_1_YEAR'].flatMap((slot) => + ['USD', 'EUR', 'BRL', 'DKK', 'INR', 'NOK', 'PLN', 'SEK', 'TRY'].map((currency) => [ + `FLUXER_STRIPE_PRICE_${slot}_${currency}`, + 'FLUXER_STRIPE_PRICES or the dashboard sets prices', + ]), + ), + ), +}; + +const INPUT_NAMES: Record = { + FLUXER_BASE_DOMAIN: 'FLUXER_DOMAIN', + FLUXER_POSTGRES_PASSWORD: 'POSTGRES_PASSWORD', + FLUXER_SEARCH_API_KEY: 'MEILI_MASTER_KEY', + FLUXER_S3_ACCESS_KEY_ID: 'FLUXER_S3_ACCESS_KEY', + FLUXER_S3_SECRET_ACCESS_KEY: 'FLUXER_S3_SECRET_KEY', + FLUXER_LIVEKIT_API_KEY: 'LIVEKIT_API_KEY', + FLUXER_LIVEKIT_API_SECRET: 'LIVEKIT_API_SECRET', + POSTGRES_DB: 'FLUXER_POSTGRES_DATABASE', + POSTGRES_USER: 'FLUXER_POSTGRES_USERNAME', + MEILI_ENV: 'FLUXER_MEILISEARCH_ENV', + MEILI_NO_ANALYTICS: 'FLUXER_MEILISEARCH_NO_ANALYTICS', + MEILI_MAX_INDEXING_MEMORY: 'FLUXER_MEILISEARCH_MAX_INDEXING_MEMORY', + GOMEMLIMIT: 'FLUXER_SEAWEEDFS_GOMEMLIMIT', + LIVEKIT_KEYS: 'LIVEKIT_API_KEY', + NODE_EXTRA_CA_CERTS: 'FLUXER_NODE_EXTRA_CA_CERTS', +}; + +const OWN_POOL_SIZES = new Set(['api', 'worker', 'users-shard', 'messages-shard']); + +const INTERPOLATION = /^\$\{([A-Z][A-Z0-9_]*)/u; + +function inputName(service: string, key: string): string { + if (key === 'FLUXER_POSTGRES_MAX_CONNECTIONS' && OWN_POOL_SIZES.has(service)) { + return `FLUXER_${service.toUpperCase().replace('-', '_')}_POSTGRES_MAX_CONNECTIONS`; + } + return INPUT_NAMES[key] ?? key; +} + +function forwardedEntries(): Array<{service: string; key: string; name: string}> { + return serviceNames.flatMap((service) => + Object.entries(serviceEnvironment(service)).flatMap(([key, value]) => { + const match = INTERPOLATION.exec(value.trim()); + if (match == null) { + return []; + } + return [{service, key, name: match[1]}]; + }), + ); +} + +describe('the shipped compose stack forwards settings from .env', () => { + test('the api is handed every setting its config loader reads', () => { + const api = serviceEnvironment('api'); + const missing = NAMED_FLUXER_ENV_NAMES.filter((name) => !(name in api) && !(name in API_SETTINGS_NOT_FORWARDED)); + expect(missing).toEqual([]); + expect(Object.keys(API_SETTINGS_NOT_FORWARDED).filter((name) => !NAMED_FLUXER_ENV_NAMES.includes(name))).toEqual( + [], + ); }); - test('the shared block sets the client-IP trust the merged services read', () => { - expect(sharedEnvironment).toMatchObject({ - FLUXER_TRUST_CLIENT_IP_HEADER: `\${FLUXER_TRUST_CLIENT_IP_HEADER:-true}`, - FLUXER_CLIENT_IP_HEADER_NAME: `\${FLUXER_CLIENT_IP_HEADER_NAME:-x-forwarded-for}`, - }); + test('every forwarded setting is read from .env under the name the operator sets', () => { + const renamed = forwardedEntries() + .filter(({service, key, name}) => name !== inputName(service, key)) + .map(({service, key, name}) => `${service}.${key} reads ${name}`); + expect(renamed).toEqual([]); }); }); diff --git a/packages/config/src/__tests__/SelfHostingObjectStoreAuth.test.ts b/packages/config/src/__tests__/SelfHostingObjectStoreAuth.test.ts index 897240d28..8077e4e56 100644 --- a/packages/config/src/__tests__/SelfHostingObjectStoreAuth.test.ts +++ b/packages/config/src/__tests__/SelfHostingObjectStoreAuth.test.ts @@ -33,7 +33,7 @@ describe('the shipped object store checks the credentials the stack sends', () = }); test('media-proxy signs its reads, which the store now refuses to serve unsigned', () => { - expect(serviceEnvironment('media-proxy').FLUXER_S3_READ_SIGNED).toBe('true'); + expect(serviceEnvironment('media-proxy').FLUXER_S3_READ_SIGNED).toBe(`\${FLUXER_S3_READ_SIGNED:-true}`); }); test('every service that reaches the store waits for the identity to exist', () => { diff --git a/packages/config/src/config_loader/EnvironmentOverrides.ts b/packages/config/src/config_loader/EnvironmentOverrides.ts index 259c0fbe3..11f5cf595 100644 --- a/packages/config/src/config_loader/EnvironmentOverrides.ts +++ b/packages/config/src/config_loader/EnvironmentOverrides.ts @@ -17,7 +17,6 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record = { FLUXER_BASE_DOMAIN: {path: ['domain', 'base_domain']}, FLUXER_PUBLIC_ORIGIN: {path: ['domain', 'public_origin']}, FLUXER_PUBLIC_SCHEME: {path: ['domain', 'public_scheme']}, - FLUXER_INTERNAL_SCHEME: {path: ['domain', 'internal_scheme']}, FLUXER_PUBLIC_PORT: {path: ['domain', 'public_port'], parse: parseInteger}, FLUXER_STATIC_CDN_DOMAIN: {path: ['domain', 'static_cdn_domain']}, FLUXER_INVITE_DOMAIN: {path: ['domain', 'invite_domain']}, @@ -29,7 +28,6 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record = { FLUXER_MEDIA_ENDPOINT: {path: ['endpoint_overrides', 'media']}, FLUXER_STATIC_CDN_ENDPOINT: {path: ['endpoint_overrides', 'static_cdn']}, FLUXER_ADMIN_ENDPOINT: {path: ['endpoint_overrides', 'admin']}, - FLUXER_DOCS_ENDPOINT: {path: ['endpoint_overrides', 'docs']}, FLUXER_MARKETING_ENDPOINT: {path: ['endpoint_overrides', 'marketing']}, FLUXER_INVITE_ENDPOINT: {path: ['endpoint_overrides', 'invite']}, FLUXER_GIFT_ENDPOINT: {path: ['endpoint_overrides', 'gift']}, @@ -54,10 +52,7 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record = { FLUXER_POSTGRES_PREPARED_STATEMENTS: {path: ['database', 'postgres', 'prepared_statements'], parse: parseBoolean}, FLUXER_DATABASE_BACKEND: {path: ['database', 'backend']}, FLUXER_KV_URL: {path: ['internal', 'kv']}, - FLUXER_KV_PROVIDER: {path: ['internal', 'kv_provider']}, FLUXER_KV_MODE: {path: ['internal', 'kv_mode']}, - FLUXER_INTERNAL_API_ENDPOINT: {path: ['internal', 'api']}, - FLUXER_INTERNAL_GATEWAY_ENDPOINT: {path: ['internal', 'gateway']}, FLUXER_INTERNAL_MEDIA_PROXY_ENDPOINT: {path: ['internal', 'media_proxy']}, FLUXER_S3_ENDPOINT: {path: ['s3', 'endpoint']}, FLUXER_S3_PUBLIC_ENDPOINT: {path: ['s3', 'presigned_url_base']}, @@ -104,46 +99,11 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record = { FLUXER_API_STORAGE_CHANGE_FEED_STREAM: {path: ['services', 'api', 'storage_change_feed', 'stream']}, FLUXER_API_STORAGE_CHANGE_FEED_SKIP_BUCKETS: { path: ['services', 'api', 'storage_change_feed', 'skip_buckets'], - parse: parseCsv, + parse: parseBucketList, }, FLUXER_API_UNFURL_IGNORED_HOSTS: {path: ['services', 'api', 'unfurl_ignored_hosts'], parse: parseCsv}, FLUXER_APP_ORIGIN_ALIASES: {path: ['services', 'api', 'app_origin_aliases'], parse: parseCsv}, - FLUXER_API_EMBEDS_OEMBED_HTML_ENABLED: { - path: ['services', 'api', 'embeds', 'oembed_html_enabled'], - parse: parseBoolean, - }, - FLUXER_API_EMBEDS_OEMBED_HTML_ALLOW_UNTRUSTED_ON_SELF_HOSTED: { - path: ['services', 'api', 'embeds', 'oembed_html_allow_untrusted_on_self_hosted'], - parse: parseBoolean, - }, - FLUXER_API_EMBEDS_OEMBED_HTML_ALLOWED_HOSTS: { - path: ['services', 'api', 'embeds', 'oembed_html_allowed_hosts'], - parse: parseCsv, - }, - FLUXER_API_EMBEDS_CACHE_DEFAULT_TTL_SECONDS: { - path: ['services', 'api', 'embeds', 'cache_default_ttl_seconds'], - parse: parseInteger, - }, - FLUXER_API_EMBEDS_CACHE_MAX_TTL_SECONDS: { - path: ['services', 'api', 'embeds', 'cache_max_ttl_seconds'], - parse: parseInteger, - }, - FLUXER_API_EMBEDS_CACHE_MIN_TTL_SECONDS: { - path: ['services', 'api', 'embeds', 'cache_min_ttl_seconds'], - parse: parseInteger, - }, - FLUXER_API_EMBEDS_CACHE_RESPECT_REMOTE_TTL: { - path: ['services', 'api', 'embeds', 'cache_respect_remote_ttl'], - parse: parseBoolean, - }, - FLUXER_API_CONTENT_MODERATION_NSFW_THRESHOLD: { - path: ['services', 'api', 'content_moderation', 'nsfw_threshold'], - parse: parseEnvValue, - }, - FLUXER_MEDIA_PROXY_HOST: {path: ['services', 'media_proxy', 'host']}, - FLUXER_MEDIA_PROXY_PORT: {path: ['services', 'media_proxy', 'port'], parse: parseInteger}, FLUXER_MEDIA_PROXY_SECRET_KEY: {path: ['services', 'media_proxy', 'secret_key']}, - FLUXER_MEDIA_PROXY_MODE: {path: ['services', 'media_proxy', 'mode']}, FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT: {path: ['services', 'media_proxy', 'upload_relay', 'endpoint']}, FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64: {path: ['services', 'media_proxy', 'upload_relay', 'secret_base64']}, FLUXER_MEDIA_PROXY_UPLOAD_RELAY_MAX_BODY_BYTES: { @@ -162,47 +122,9 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record = { path: ['services', 'media_proxy', 'attachment_urls', 'secrets_base64'], parse: parseCsv, }, - FLUXER_ADMIN_PORT: {path: ['services', 'admin', 'port'], parse: parseInteger}, - FLUXER_ADMIN_BASE_PATH: {path: ['services', 'admin', 'base_path']}, FLUXER_ADMIN_SECRET_KEY_BASE: {path: ['services', 'admin', 'secret_key_base']}, FLUXER_ADMIN_OAUTH_CLIENT_SECRET: {path: ['services', 'admin', 'oauth_client_secret']}, - FLUXER_APP_PROXY_PORT: {path: ['services', 'app_proxy', 'port'], parse: parseInteger}, - FLUXER_STATIC_DIR: {path: ['services', 'app_proxy', 'assets_dir']}, - FLUXER_GATEWAY_PORT: {path: ['services', 'gateway', 'port'], parse: parseInteger}, - FLUXER_GATEWAY_ROLE: {path: ['services', 'gateway', 'gateway_role']}, - FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: {path: ['services', 'gateway', 'media_proxy_endpoint']}, - FLUXER_GATEWAY_API_RPC_ENDPOINT: {path: ['services', 'gateway', 'api_rpc_endpoint']}, FLUXER_GATEWAY_RPC_AUTH_TOKEN: {path: ['services', 'gateway', 'rpc_auth_token']}, - FLUXER_GATEWAY_LOGGER_LEVEL: {path: ['services', 'gateway', 'logger_level']}, - FLUXER_GATEWAY_HTTP_FAILURE_THRESHOLD: { - path: ['services', 'gateway', 'gateway_http_failure_threshold'], - parse: parseInteger, - }, - FLUXER_GATEWAY_HTTP_RECOVERY_TIMEOUT_MS: { - path: ['services', 'gateway', 'gateway_http_recovery_timeout_ms'], - parse: parseInteger, - }, - FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY: { - path: ['services', 'gateway', 'gateway_http_rpc_max_concurrency'], - parse: parseInteger, - }, - FLUXER_GATEWAY_NATS_RPC_MAX_HANDLERS: { - path: ['services', 'gateway', 'gateway_nats_rpc_max_handlers'], - parse: parseInteger, - }, - FLUXER_GATEWAY_SHUTDOWN_DRAIN_WAIT_MS: { - path: ['services', 'gateway', 'shutdown_drain_wait_ms'], - parse: parseInteger, - }, - FLUXER_GATEWAY_CLUSTER_ENABLED: {path: ['services', 'gateway', 'cluster_enabled'], parse: parseEnvValue}, - FLUXER_GATEWAY_CLUSTER_DISCOVERY_DNS_NAME: {path: ['services', 'gateway', 'cluster_discovery_dns_name']}, - FLUXER_GATEWAY_CLUSTER_DISCOVERY_NODE_BASENAME: { - path: ['services', 'gateway', 'cluster_discovery_node_basename'], - }, - FLUXER_GATEWAY_CLUSTER_DISCOVERY_POLL_INTERVAL_MS: { - path: ['services', 'gateway', 'cluster_discovery_poll_interval_ms'], - parse: parseInteger, - }, FLUXER_SUDO_MODE_SECRET: {path: ['auth', 'sudo_mode_secret']}, FLUXER_CONNECTION_INITIATION_SECRET: {path: ['auth', 'connection_initiation_secret']}, FLUXER_SSO_ALLOW_PRIVATE_ADDRESSES: {path: ['auth', 'sso_allow_private_addresses'], parse: parseBoolean}, @@ -238,7 +160,6 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record = { FLUXER_LIVEKIT_API_SECRET: {path: ['integrations', 'voice', 'api_secret']}, FLUXER_LIVEKIT_URL: {path: ['integrations', 'voice', 'url']}, FLUXER_LIVEKIT_INTERNAL_URL: {path: ['integrations', 'voice', 'internal_url']}, - FLUXER_LIVEKIT_WEBHOOK_URL: {path: ['integrations', 'voice', 'webhook_url']}, FLUXER_LIVEKIT_DEFAULT_REGION: {path: ['integrations', 'voice', 'default_region'], parse: parseJsonObject}, FLUXER_SEARCH_ENGINE: {path: ['integrations', 'search', 'engine']}, FLUXER_SEARCH_URL: {path: ['integrations', 'search', 'url']}, @@ -272,10 +193,6 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record = { FLUXER_STRIPE_PRICE_YEARLY_PLN: {path: ['integrations', 'stripe', 'prices', 'yearly_pln']}, FLUXER_STRIPE_PRICE_YEARLY_SEK: {path: ['integrations', 'stripe', 'prices', 'yearly_sek']}, FLUXER_STRIPE_PRICE_YEARLY_TRY: {path: ['integrations', 'stripe', 'prices', 'yearly_try']}, - FLUXER_STRIPE_PRICE_VISIONARY_USD: {path: ['integrations', 'stripe', 'prices', 'visionary_usd']}, - FLUXER_STRIPE_PRICE_VISIONARY_EUR: {path: ['integrations', 'stripe', 'prices', 'visionary_eur']}, - FLUXER_STRIPE_PRICE_GIFT_VISIONARY_USD: {path: ['integrations', 'stripe', 'prices', 'gift_visionary_usd']}, - FLUXER_STRIPE_PRICE_GIFT_VISIONARY_EUR: {path: ['integrations', 'stripe', 'prices', 'gift_visionary_eur']}, FLUXER_STRIPE_PRICE_GIFT_1_MONTH_USD: {path: ['integrations', 'stripe', 'prices', 'gift_1_month_usd']}, FLUXER_STRIPE_PRICE_GIFT_1_MONTH_EUR: {path: ['integrations', 'stripe', 'prices', 'gift_1_month_eur']}, FLUXER_STRIPE_PRICE_GIFT_1_MONTH_SEK: {path: ['integrations', 'stripe', 'prices', 'gift_1_month_sek']}, @@ -323,16 +240,7 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record = { FLUXER_PUSH_APNS_KEY_ID: {path: ['integrations', 'push', 'apns', 'key_id']}, FLUXER_PUSH_APNS_PRIVATE_KEY: {path: ['integrations', 'push', 'apns', 'private_key']}, FLUXER_PUSH_APNS_PRIVATE_KEY_PATH: {path: ['integrations', 'push', 'apns', 'private_key_path']}, - FLUXER_PUSH_APNS_DEFAULT_ENVIRONMENT: {path: ['integrations', 'push', 'apns', 'default_environment']}, FLUXER_PUSH_APNS_APPS: {path: ['integrations', 'push', 'apns', 'apps'], parse: parseJsonArray}, - FLUXER_PUSH_FCM_ENABLED: {path: ['integrations', 'push', 'fcm', 'enabled'], parse: parseBoolean}, - FLUXER_PUSH_FCM_PROJECT_ID: {path: ['integrations', 'push', 'fcm', 'project_id']}, - FLUXER_PUSH_FCM_CLIENT_EMAIL: {path: ['integrations', 'push', 'fcm', 'client_email']}, - FLUXER_PUSH_FCM_PRIVATE_KEY: {path: ['integrations', 'push', 'fcm', 'private_key']}, - FLUXER_PUSH_FCM_PRIVATE_KEY_PATH: {path: ['integrations', 'push', 'fcm', 'private_key_path']}, - FLUXER_PUSH_FCM_SERVICE_ACCOUNT_JSON_PATH: {path: ['integrations', 'push', 'fcm', 'service_account_json_path']}, - FLUXER_PUSH_FCM_TOKEN_URI: {path: ['integrations', 'push', 'fcm', 'token_uri']}, - FLUXER_PUSH_FCM_APPS: {path: ['integrations', 'push', 'fcm', 'apps'], parse: parseJsonArray}, FLUXER_SELF_HOSTED: {path: ['instance', 'self_hosted'], parse: parseBoolean}, FLUXER_AUTO_JOIN_INVITE_CODE: {path: ['instance', 'auto_join_invite_code']}, FLUXER_VISIONARIES_GUILD_ID: {path: ['instance', 'visionaries_guild_id']}, @@ -394,50 +302,27 @@ function parseBoolean(raw: string): boolean { } function parseJson(raw: string): unknown { - const trimmed = raw.trim(); - if (trimmed.length === 0) return undefined; try { - return JSON.parse(trimmed); + return JSON.parse(raw); } catch { throw new Error('must be valid JSON'); } } -function parseJsonObject(raw: string): ConfigObject | undefined { +function parseJsonObject(raw: string): ConfigObject { const value = parseJson(raw); - if (value === undefined || isConfigObject(value)) return value; + if (isConfigObject(value)) return value; throw new Error('must be a JSON object'); } -function parseJsonArray(raw: string): Array | undefined { +function parseJsonArray(raw: string): Array { const value = parseJson(raw); - if (value === undefined || Array.isArray(value)) return value; + if (Array.isArray(value)) return value; throw new Error('must be a JSON array'); } -export function parseEnvValue(raw: string): unknown { +function parseInteger(raw: string): number { const trimmed = raw.trim(); - const lower = trimmed.toLowerCase(); - if (lower === 'true' || lower === 'false') return parseBoolean(trimmed); - if (/^-?\d+$/.test(trimmed)) { - return parseInteger(trimmed); - } - if (/^-?\d+\.\d+$/.test(trimmed)) { - const value = Number(trimmed); - if (!Number.isFinite(value)) throw new Error('must be a finite number'); - return value; - } - if (trimmed.startsWith('{') || trimmed.startsWith('[')) { - return parseJson(trimmed); - } - return raw; -} - -function parseInteger(raw: string): number | undefined { - const trimmed = raw.trim(); - if (trimmed.length === 0) { - return undefined; - } if (!/^-?\d+$/.test(trimmed)) { throw new Error(`must be an integer, got ${JSON.stringify(raw)}`); } @@ -453,6 +338,10 @@ function parseCsv(raw: string): Array { .filter((part) => part.length > 0); } +function parseBucketList(raw: string): Array { + return raw.trim().toLowerCase() === 'none' ? [] : parseCsv(raw); +} + function parsePasskeyOrigins(raw: string): Array { if (/\p{Cc}/u.test(raw)) { throw new Error('must not contain control characters'); @@ -486,11 +375,18 @@ const NAMED_FLUXER_ENV_ALIASES: Record = { FLUXER_IPINFO_API_KEY: 'FLUXER_RISK_IPINFO_API_KEY', }; +export const NAMED_FLUXER_ENV_NAMES = Object.keys(NAMED_FLUXER_ENV_OVERRIDES); + +export function readEnvValue(env: NodeJS.ProcessEnv, name: string): string | undefined { + const value = env[name]; + return value === undefined || value.trim().length === 0 ? undefined : value; +} + export function buildNamedFluxerEnvOverrides(env: NodeJS.ProcessEnv): ConfigObject { const overrides: ConfigObject = {}; for (const [envKey, mapping] of Object.entries(NAMED_FLUXER_ENV_OVERRIDES)) { const alias = NAMED_FLUXER_ENV_ALIASES[envKey]; - const raw = env[envKey] ?? (alias === undefined ? undefined : env[alias]); + const raw = readEnvValue(env, envKey) ?? (alias === undefined ? undefined : readEnvValue(env, alias)); if (raw === undefined) { continue; } @@ -500,9 +396,6 @@ export function buildNamedFluxerEnvOverrides(env: NodeJS.ProcessEnv): ConfigObje } catch (error) { throw new Error(`${envKey} ${error instanceof Error ? error.message : String(error)}`); } - if (parsed === undefined) { - continue; - } setNestedValue(overrides, mapping.path, parsed); } return overrides; diff --git a/packages/constants/src/Timeouts.ts b/packages/constants/src/Timeouts.ts index 469ad762d..091481b81 100644 --- a/packages/constants/src/Timeouts.ts +++ b/packages/constants/src/Timeouts.ts @@ -1,5 +1,4 @@ // SPDX-License-Identifier: AGPL-3.0-or-later export const DEFAULT_KV_TIMEOUT_MS = 5000; -export const DEFAULT_HTTP_WORKER_TIMEOUT_MS = 30000; export const DEFAULT_SEARCH_CLIENT_TIMEOUT_MS = 30000; diff --git a/packages/errors/package.json b/packages/errors/package.json index 83d0c5116..e1d4d47ab 100644 --- a/packages/errors/package.json +++ b/packages/errors/package.json @@ -12,7 +12,6 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@fluxer/config": "workspace:*", "@fluxer/constants": "workspace:*", "@fluxer/hono_types": "workspace:*", "@fluxer/i18n": "workspace:*", diff --git a/packages/errors/src/domains/core/MaxBookmarksError.ts b/packages/errors/src/domains/core/MaxBookmarksError.ts index 0818921c4..6e03845ef 100644 --- a/packages/errors/src/domains/core/MaxBookmarksError.ts +++ b/packages/errors/src/domains/core/MaxBookmarksError.ts @@ -1,24 +1,15 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import {getConfig} from '@fluxer/config/src/ConfigLoader'; import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; import {BadRequestError} from '@fluxer/errors/src/domains/core/BadRequestError'; export class MaxBookmarksError extends BadRequestError { - constructor(params: { - maxBookmarks: number; - isPremium?: boolean; - }) { - const {maxBookmarks, isPremium} = params; - const config = getConfig(); - const selfHosted = 'self_hosted' in config ? config.self_hosted : false; + constructor(params: {maxBookmarks: number}) { + const {maxBookmarks} = params; super({ code: APIErrorCodes.MAX_BOOKMARKS, messageVariables: {count: maxBookmarks}, - data: { - max_bookmarks: maxBookmarks, - ...(selfHosted || isPremium === undefined ? {} : {is_premium: isPremium}), - }, + data: {max_bookmarks: maxBookmarks}, }); } } diff --git a/packages/logger/src/Logger.ts b/packages/logger/src/Logger.ts index 0a0bfcc23..847e5b813 100644 --- a/packages/logger/src/Logger.ts +++ b/packages/logger/src/Logger.ts @@ -20,7 +20,7 @@ function isPinoLevel(value: string): value is pino.Level { } function resolveEnvironment(options: LoggerOptions): string { - return options.environment ?? process.env.FLUXER_ENV ?? 'production'; + return options.environment ?? (process.env.FLUXER_ENV?.trim() || 'production'); } function resolveLevel(options: LoggerOptions, isDev: boolean): pino.Level { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index df8c47eeb..0cf2e7921 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1026,15 +1026,9 @@ importers: fluxer_api/pkgs/worker: dependencies: - '@fluxer/constants': - specifier: workspace:* - version: link:../../../packages/constants '@fluxer/logger': specifier: workspace:* version: link:../../../packages/logger - itty-time: - specifier: 'catalog:' - version: 2.0.2 devDependencies: '@types/node': specifier: 'catalog:' @@ -1969,9 +1963,6 @@ importers: packages/errors: dependencies: - '@fluxer/config': - specifier: workspace:* - version: link:../config '@fluxer/constants': specifier: workspace:* version: link:../constants diff --git a/tools/dev/src/dev.rs b/tools/dev/src/dev.rs index b0cc1b172..ef9f83e70 100644 --- a/tools/dev/src/dev.rs +++ b/tools/dev/src/dev.rs @@ -542,7 +542,6 @@ pub fn task_table() -> Result> { Some(format!("{public_url}/media")), ), ("FLUXER_STATIC_CDN_ENDPOINT".to_owned(), Some(public_url)), - ("RELEASE_CHANNEL".to_owned(), Some("canary".to_owned())), ], }); insert(DevTask { diff --git a/tools/dev/src/tunnel.rs b/tools/dev/src/tunnel.rs index 8dd3b9b27..36fe86a23 100644 --- a/tools/dev/src/tunnel.rs +++ b/tools/dev/src/tunnel.rs @@ -98,10 +98,6 @@ pub fn public_url_env(public_url: &str) -> Result> { "FLUXER_LIVEKIT_URL".to_owned(), format!("{gateway_base}/livekit"), ), - ( - "FLUXER_LIVEKIT_WEBHOOK_URL".to_owned(), - format!("{base}/api/webhooks/livekit"), - ), ( "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT".to_owned(), format!("{base}/media"),