diff --git a/deploy/self-hosting/.env.example b/deploy/self-hosting/.env.example index 2ee7a2a40..bc201ac98 100644 --- a/deploy/self-hosting/.env.example +++ b/deploy/self-hosting/.env.example @@ -1,14 +1,60 @@ +# Every variable docker-compose.yml reads from this file is named here: +# uncommented when it has no default, commented with its default when it has one. +# A name absent from this file is one Compose does not forward, and it reaches a +# service only through a Compose override file that adds it to that service's +# environment. packages/config/src/__tests__/DeployEnvCoverage.test.ts fails when +# a Compose edit forgets the matching line here. + FLUXER_DOMAIN=chat.example.com FLUXER_PUBLIC_SCHEME=https FLUXER_PUBLIC_PORT=443 -FLUXER_PUBLIC_ORIGIN=${FLUXER_PUBLIC_SCHEME}://${FLUXER_DOMAIN} -FLUXER_CADDY_SITE_ADDRESS=chat.example.com + +# How browsers reach this instance. +# +# Default: Fluxer binds 80 and 443 and gets its own Let's Encrypt certificate. +# Point DNS at this host and there is nothing else to configure. +# +# Behind your own reverse proxy (nginx, Traefik, HAProxy, Cloudflare Tunnel, +# another Caddy): uncomment COMPOSE_FILE below. Fluxer then serves plain HTTP on +# 127.0.0.1:8080 instead, and your proxy forwards everything to it. Keep +# FLUXER_PUBLIC_SCHEME and FLUXER_PUBLIC_PORT describing the PUBLIC address your +# proxy serves, not this local port. +#COMPOSE_FILE=docker-compose.yml:docker-compose.proxy.yml + +# Where the plain-HTTP port binds when the proxy overlay is in use. Leave it on +# loopback when the proxy runs on this host. Use 0.0.0.0:8080 only when the proxy +# is on another machine, and firewall the port to that machine. +#FLUXER_EDGE_BIND=127.0.0.1:8080 + +# Which upstream hops may set X-Forwarded-For. Fluxer rewrites the header from +# this to the real client address, so IP bans, rate limits and abuse detection +# see the caller rather than the proxy. The default covers proxies on private or +# loopback addresses, which is every same-host setup. Set it to your proxy's +# address if it reaches Fluxer from a public IP. +#FLUXER_EDGE_TRUSTED_PROXIES=private_ranges + +# The public origin browsers use, without a trailing slash. Derived from the three +# values above and correct for the usual https-on-443 setup, so leave it alone +# unless you serve Fluxer on a non-default port, where the port must appear here. +#FLUXER_PUBLIC_ORIGIN=https://chat.example.com + +# Overrides the address Fluxer's edge listens on. Honoured in the default mode +# only: docker-compose.proxy.yml sets the literal :8080 and Compose lets the last +# file win, so a value here is discarded under the proxy overlay with no warning. +# Set it only for an unusual default-mode layout, such as serving several +# hostnames or binding a non-default TLS port. +#FLUXER_EDGE_SITE_ADDRESS=chat.example.com + +# The old name for the value above. It is read only when +# FLUXER_EDGE_SITE_ADDRESS is unset, so an existing .env keeps the listener +# it already had. Rename it to FLUXER_EDGE_SITE_ADDRESS at your convenience. +#FLUXER_CADDY_SITE_ADDRESS= # FLUXER_PUBLIC_ORIGIN is the origin browsers see. It must carry the port # whenever FLUXER_PUBLIC_PORT is not the default for its scheme, because an # origin written with a default port never matches a browser Origin header. # Serving on any other port means setting all three, plus the published port -# below, and pointing FLUXER_CADDY_SITE_ADDRESS at the same scheme and host. +# below, and pointing FLUXER_EDGE_SITE_ADDRESS at the same scheme and host. # Compose expands this file from top to bottom, so FLUXER_PUBLIC_ORIGIN has to # stay below the two values it reads. Above them it silently expands to a bare # host with a trailing colon. @@ -48,6 +94,7 @@ FLUXER_S3_SECRET_KEY=CHANGE_ME FLUXER_SUDO_MODE_SECRET=CHANGE_ME FLUXER_CONNECTION_INITIATION_SECRET=CHANGE_ME FLUXER_GATEWAY_RPC_AUTH_TOKEN=CHANGE_ME +FLUXER_ERLANG_COOKIE=CHANGE_ME FLUXER_MEDIA_PROXY_SECRET_KEY=CHANGE_ME FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64=CHANGE_ME FLUXER_ADMIN_SECRET_KEY_BASE=CHANGE_ME @@ -55,7 +102,10 @@ FLUXER_ADMIN_OAUTH_CLIENT_SECRET=CHANGE_ME FLUXER_VAPID_PUBLIC_KEY=CHANGE_ME FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME -FLUXER_VAPID_EMAIL=admin@example.com + +# The VAPID contact address defaults to admin@ followed by FLUXER_DOMAIN. Set it +# only if that mailbox does not exist. +#FLUXER_VAPID_EMAIL=admin@chat.example.com # Passkeys follow FLUXER_DOMAIN by default. Set these only if browsers reach the # instance on a different host, and note that changing FLUXER_PASSKEY_RP_ID @@ -68,10 +118,22 @@ FLUXER_VAPID_EMAIL=admin@example.com # Extra Content-Security-Policy sources, appended to the built-in ones. Set these # only when a browser must reach an origin the defaults do not cover, such as a # voice server hosted on a domain other than FLUXER_DOMAIN. Separate several -# sources with spaces or commas. +# sources with spaces or commas. Every one of them is empty by default, and the +# three carrying a value below are illustrations, not defaults. +#FLUXER_CSP_EXTRA_DEFAULT_SRC= #FLUXER_CSP_EXTRA_CONNECT_SRC=wss://livekit.example.com:7881 #FLUXER_CSP_EXTRA_IMG_SRC=https://cdn.example.com +#FLUXER_CSP_EXTRA_MEDIA_SRC= +#FLUXER_CSP_EXTRA_FONT_SRC= #FLUXER_CSP_EXTRA_SCRIPT_SRC=https://analytics.example.com +#FLUXER_CSP_EXTRA_STYLE_SRC= +#FLUXER_CSP_EXTRA_FRAME_SRC= +#FLUXER_CSP_EXTRA_WORKER_SRC= +#FLUXER_CSP_EXTRA_MANIFEST_SRC= + +# One report-uri for Content-Security-Policy violation reports. Empty leaves the +# directive off the header. +#FLUXER_CSP_REPORT_URI= # Allow the SSO identity provider to resolve to a private or internal address. # Off by default: the API refuses to call non-public addresses so a misconfigured @@ -80,23 +142,21 @@ FLUXER_VAPID_EMAIL=admin@example.com # identity provider, and only when you trust everyone who can configure SSO. #FLUXER_SSO_ALLOW_PRIVATE_ADDRESSES=true +# Both reach LiveKit as LIVEKIT_KEYS and the webhook signing key, and the API as +# FLUXER_LIVEKIT_API_KEY and FLUXER_LIVEKIT_API_SECRET. Change them together. LIVEKIT_API_KEY=fluxer LIVEKIT_API_SECRET=CHANGE_ME -# Ports LiveKit publishes on the host for voice and video media. They take the -# same optional bind address as the Caddy ports above. This media does not pass -# through Caddy or through a tunnel, so it needs these ports reachable from -# clients. LiveKit advertises the port numbers from livekit.yaml, so publishing -# them on different host ports means changing that file too. +# The URL browsers use for voice signalling. Derived from FLUXER_PUBLIC_SCHEME, +# FLUXER_DOMAIN and FLUXER_PUBLIC_PORT as wss://host[:port]/livekit when empty. +# Set it only when LiveKit is served from another host. +#FLUXER_LIVEKIT_URL= + +# Media ports. LiveKit advertises these in ICE candidates, so the host must +# forward the same numbers. #FLUXER_LIVEKIT_TCP_PORT=7881 #FLUXER_LIVEKIT_UDP_PORT=7882 -# The voice server URL clients connect to. It defaults to FLUXER_PUBLIC_ORIGIN -# plus /livekit, which the bundled Caddy proxies to the LiveKit container. Set -# it only when LiveKit lives on its own host, and add that origin to -# FLUXER_CSP_EXTRA_CONNECT_SRC when you do. -#FLUXER_LIVEKIT_URL=wss://voice.example.com - FLUXER_KLIPY_API_KEY= FLUXER_EMAIL_ENABLED=false @@ -112,20 +172,50 @@ FLUXER_EMAIL_SMTP_SECURE=true FLUXER_CAPTCHA_ENABLED=false FLUXER_CAPTCHA_PROVIDER=none +FLUXER_CAPTCHA_HCAPTCHA_SITE_KEY= +FLUXER_CAPTCHA_HCAPTCHA_SECRET_KEY= +FLUXER_CAPTCHA_TURNSTILE_SITE_KEY= +FLUXER_CAPTCHA_TURNSTILE_SECRET_KEY= FLUXER_DISCOVERY_ENABLED=true -# Container memory. Every service limit and reservation below has a default that -# assumes a host with at least 16 GB of RAM. Limits are per-container ceilings, so -# their sum may exceed host RAM; the reservations are what protect the services -# whose death takes the whole instance down. Lower these on a smaller host. +# Container memory. The 25 limits sum to 16.75 GiB, which is a sum of ceilings and +# not an allocation, so the defaults fit a host with 8 GB and are sized for 16 GB. +# The four reservations are cgroup memory.low, which biases the kernel away from +# reclaiming from the services whose death takes the whole instance down. They do +# not reserve anything. Lower the limits on a smaller host. +#FLUXER_CADDY_MEMORY_LIMIT=256mb #FLUXER_POSTGRES_MEMORY_LIMIT=5gb #FLUXER_POSTGRES_MEMORY_RESERVATION=3gb +#FLUXER_VALKEY_MEMORY_LIMIT=256mb +#FLUXER_NATS_MEMORY_LIMIT=256mb +#FLUXER_MEILISEARCH_MEMORY_LIMIT=768mb +#FLUXER_SEAWEEDFS_MEMORY_LIMIT=512mb +#FLUXER_SEAWEEDFS_INIT_MEMORY_LIMIT=128mb +#FLUXER_LIVEKIT_MEMORY_LIMIT=512mb #FLUXER_API_MEMORY_LIMIT=2560mb #FLUXER_API_MEMORY_RESERVATION=1gb #FLUXER_WORKER_MEMORY_LIMIT=2560mb #FLUXER_WORKER_MEMORY_RESERVATION=1gb #FLUXER_GATEWAY_MEMORY_LIMIT=1gb -#FLUXER_MEILISEARCH_MEMORY_LIMIT=768mb +#FLUXER_GATEWAY_MEMORY_RESERVATION=384mb +#FLUXER_MEDIA_PROXY_MEMORY_LIMIT=512mb +#FLUXER_STATIC_PROXY_MEMORY_LIMIT=256mb +#FLUXER_APP_PROXY_MEMORY_LIMIT=256mb +#FLUXER_SNOWFLAKES_MEMORY_LIMIT=128mb +#FLUXER_SNOWFLAKES_SHARD_MEMORY_LIMIT=256mb +#FLUXER_USERS_MEMORY_LIMIT=128mb +#FLUXER_USERS_SHARD_MEMORY_LIMIT=256mb +#FLUXER_GIFS_MEMORY_LIMIT=128mb +#FLUXER_GIFS_SHARD_MEMORY_LIMIT=256mb +#FLUXER_MESSAGES_MEMORY_LIMIT=128mb +#FLUXER_MESSAGES_SHARD_MEMORY_LIMIT=256mb +#FLUXER_UNFURL_MEMORY_LIMIT=128mb +#FLUXER_UNFURL_SHARD_MEMORY_LIMIT=256mb +#FLUXER_ADMIN_MEMORY_LIMIT=256mb + +# Meilisearch indexing memory. Keep it well under FLUXER_MEILISEARCH_MEMORY_LIMIT, +# which is the container ceiling the indexer shares with the search process. +#FLUXER_MEILISEARCH_MAX_INDEXING_MEMORY=384mb # Node sizes its own heap from the container memory limit by default, at roughly # 55 percent of it, which always leaves room for the buffers and stacks that live @@ -145,13 +235,16 @@ FLUXER_DISCOVERY_ENABLED=true #FLUXER_POSTGRES_SHARED_BUFFERS=512MB #FLUXER_POSTGRES_EFFECTIVE_CACHE_SIZE=2GB #FLUXER_POSTGRES_WORK_MEM=8MB +#FLUXER_POSTGRES_MAINTENANCE_WORK_MEM=256MB +#FLUXER_POSTGRES_AUTOVACUUM_WORK_MEM=128MB -# The bundled Valkey holds durable state as well as cache: the bulk message -# deletion queue, the account deletion queue and every distributed lock, none of -# which carry an expiry. It therefore runs with an append-only file on a named -# volume and with noeviction, so an over-limit write fails loudly instead of -# silently deleting queued work. Only change the policy if you have moved that -# durable state elsewhere. +# The bundled Valkey holds durable state as well as cache. The bulk message +# deletion queue and the account deletion queue are sorted sets with no expiry, +# and nothing else stores the first of the two. It therefore runs with an +# append-only file on a named volume and with noeviction, so an over-limit write +# fails loudly instead of silently deleting queued work. Distributed locks all +# carry a TTL and are not what the durability is for. Only change the policy if +# you have moved that durable state elsewhere. #FLUXER_VALKEY_MAXMEMORY=192mb #FLUXER_VALKEY_MAXMEMORY_POLICY=noeviction @@ -162,6 +255,12 @@ FLUXER_DISCOVERY_ENABLED=true #FLUXER_ERLANG_SCHEDULERS_MIN=2 #FLUXER_ERLANG_SCHEDULERS_MAX=16 +# In-flight request ceiling for the four services Compose forwards it to: the +# users and messages routers and their shards. The Rust built-in defaults are 192 +# for messages, 320 for snowflakes and 64 elsewhere, and they govern every service +# Compose does not forward this to. +#FLUXER_SVC_MAX_CONCURRENT_REQUESTS=20 + # The api and the Rust services name their fixed Postgres statement shapes so the # server can reuse their plans. Named prepared statements require a session that # outlives the transaction, so set this to false if you put a transaction-pooling diff --git a/deploy/self-hosting/Caddyfile b/deploy/self-hosting/Caddyfile index 15b0b2746..f756efbcc 100644 --- a/deploy/self-hosting/Caddyfile +++ b/deploy/self-hosting/Caddyfile @@ -1,58 +1,85 @@ { servers { - trusted_proxies static private_ranges + trusted_proxies static {$FLUXER_EDGE_TRUSTED_PROXIES:private_ranges} + trusted_proxies_strict } } -{$FLUXER_CADDY_SITE_ADDRESS} { +{$FLUXER_EDGE_SITE_ADDRESS} { encode zstd gzip + handle /_health { + respond "OK" 200 + } + handle_path /api/* { - reverse_proxy api:8080 + reverse_proxy api:8080 { + header_up X-Forwarded-For {client_ip} + } } handle /gateway { rewrite * / - reverse_proxy gateway:8080 + reverse_proxy gateway:8080 { + header_up X-Forwarded-For {client_ip} + } } handle_path /gateway/* { - reverse_proxy gateway:8080 + reverse_proxy gateway:8080 { + header_up X-Forwarded-For {client_ip} + } } handle_path /media/* { - reverse_proxy media-proxy:8080 + reverse_proxy media-proxy:8080 { + header_up X-Forwarded-For {client_ip} + } } handle_path /livekit/* { - reverse_proxy livekit:7880 + reverse_proxy livekit:7880 { + header_up X-Forwarded-For {client_ip} + } } handle /admin { rewrite * / - reverse_proxy admin:8080 + reverse_proxy admin:8080 { + header_up X-Forwarded-For {client_ip} + } } handle_path /admin/* { - reverse_proxy admin:8080 + reverse_proxy admin:8080 { + header_up X-Forwarded-For {client_ip} + } } @staticAssets path /web/* /emoji/* /libs/* /avatars/* /badges/* /desktop/* /embeds/* handle @staticAssets { - reverse_proxy static-proxy:8080 + reverse_proxy static-proxy:8080 { + header_up X-Forwarded-For {client_ip} + } } handle /.well-known/fluxer { - reverse_proxy api:8080 + reverse_proxy api:8080 { + header_up X-Forwarded-For {client_ip} + } } handle { - reverse_proxy app-proxy:8080 + reverse_proxy app-proxy:8080 { + header_up X-Forwarded-For {client_ip} + } } } :8088 { - handle_path /api/* { - reverse_proxy api:8080 + handle /.well-known/fluxer { + reverse_proxy api:8080 { + header_up X-Forwarded-For {client_ip} + } } } diff --git a/deploy/self-hosting/docker-compose.proxy.yml b/deploy/self-hosting/docker-compose.proxy.yml new file mode 100644 index 000000000..4ee5005af --- /dev/null +++ b/deploy/self-hosting/docker-compose.proxy.yml @@ -0,0 +1,17 @@ +# Overlay for running Fluxer behind your own reverse proxy. +# +# docker compose -f docker-compose.yml -f docker-compose.proxy.yml up -d +# +# Or set this once in .env and keep using plain `docker compose up -d`: +# +# COMPOSE_FILE=docker-compose.yml:docker-compose.proxy.yml +# +# Fluxer stops binding 80 and 443 and serves plain HTTP on one port instead. +# That port already does all internal routing, so the proxy in front needs a +# single rule: send everything to it. Terminate TLS there. +services: + edge: + ports: !override + - "${FLUXER_EDGE_BIND:-127.0.0.1:8080}:8080" + environment: + FLUXER_EDGE_SITE_ADDRESS: ":8080" diff --git a/deploy/self-hosting/docker-compose.yml b/deploy/self-hosting/docker-compose.yml index ffdf6777d..b0ab9cd7e 100644 --- a/deploy/self-hosting/docker-compose.yml +++ b/deploy/self-hosting/docker-compose.yml @@ -44,8 +44,8 @@ x-fluxer-env: &fluxer-env FLUXER_S3_BUCKET_DOWNLOADS: fluxer-downloads FLUXER_S3_BUCKET_REPORTS: fluxer-reports FLUXER_S3_BUCKET_HARVESTS: fluxer-harvests - AWS_ACCESS_KEY_ID: ${FLUXER_S3_ACCESS_KEY:?} - AWS_SECRET_ACCESS_KEY: ${FLUXER_S3_SECRET_KEY:?} + AWS_ACCESS_KEY_ID: ${FLUXER_S3_ACCESS_KEY:?set FLUXER_S3_ACCESS_KEY in .env} + AWS_SECRET_ACCESS_KEY: ${FLUXER_S3_SECRET_KEY:?set FLUXER_S3_SECRET_KEY in .env} AWS_DEFAULT_REGION: us-east-1 AWS_EC2_METADATA_DISABLED: "true" @@ -73,6 +73,10 @@ x-fluxer-env: &fluxer-env FLUXER_SMS_ENABLED: "false" FLUXER_CAPTCHA_ENABLED: ${FLUXER_CAPTCHA_ENABLED:-false} FLUXER_CAPTCHA_PROVIDER: ${FLUXER_CAPTCHA_PROVIDER:-none} + FLUXER_CAPTCHA_HCAPTCHA_SITE_KEY: ${FLUXER_CAPTCHA_HCAPTCHA_SITE_KEY:-} + FLUXER_CAPTCHA_HCAPTCHA_SECRET_KEY: ${FLUXER_CAPTCHA_HCAPTCHA_SECRET_KEY:-} + FLUXER_CAPTCHA_TURNSTILE_SITE_KEY: ${FLUXER_CAPTCHA_TURNSTILE_SITE_KEY:-} + FLUXER_CAPTCHA_TURNSTILE_SECRET_KEY: ${FLUXER_CAPTCHA_TURNSTILE_SECRET_KEY:-} FLUXER_STRIPE_ENABLED: "false" FLUXER_NCMEC_ENABLED: "false" FLUXER_CLAMAV_ENABLED: "false" @@ -114,7 +118,7 @@ x-fluxer-svc-healthcheck: &fluxer-svc-healthcheck start_interval: 1s services: - caddy: + edge: image: caddy:2.10-alpine deploy: resources: @@ -127,11 +131,12 @@ services: - "${FLUXER_HTTPS_PORT:-443}:443" - "${FLUXER_HTTPS_PORT:-443}:443/udp" environment: - FLUXER_CADDY_SITE_ADDRESS: ${FLUXER_CADDY_SITE_ADDRESS:?set FLUXER_CADDY_SITE_ADDRESS in .env} + FLUXER_EDGE_SITE_ADDRESS: ${FLUXER_EDGE_SITE_ADDRESS:-${FLUXER_CADDY_SITE_ADDRESS:-${FLUXER_DOMAIN:?set FLUXER_DOMAIN in .env}}} + FLUXER_EDGE_TRUSTED_PROXIES: ${FLUXER_EDGE_TRUSTED_PROXIES:-private_ranges} volumes: - ./Caddyfile:/etc/caddy/Caddyfile:ro - - caddy-data:/data - - caddy-config:/config + - edge-data:/data + - edge-config:/config healthcheck: test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:2019/config/"] interval: 10s @@ -160,8 +165,8 @@ services: -c shared_buffers=${FLUXER_POSTGRES_SHARED_BUFFERS:-512MB} -c effective_cache_size=${FLUXER_POSTGRES_EFFECTIVE_CACHE_SIZE:-2GB} -c work_mem=${FLUXER_POSTGRES_WORK_MEM:-8MB} - -c maintenance_work_mem=256MB - -c autovacuum_work_mem=128MB + -c maintenance_work_mem=${FLUXER_POSTGRES_MAINTENANCE_WORK_MEM:-256MB} + -c autovacuum_work_mem=${FLUXER_POSTGRES_AUTOVACUUM_WORK_MEM:-128MB} -c random_page_cost=1.1 -c effective_io_concurrency=200 -c default_statistics_target=200 @@ -238,7 +243,7 @@ services: environment: MEILI_ENV: production MEILI_NO_ANALYTICS: "true" - MEILI_MAX_INDEXING_MEMORY: 384mb + MEILI_MAX_INDEXING_MEMORY: ${FLUXER_MEILISEARCH_MAX_INDEXING_MEMORY:-384mb} MEILI_MASTER_KEY: ${MEILI_MASTER_KEY:?set MEILI_MASTER_KEY in .env} volumes: - meilisearch-data:/meili_data @@ -276,6 +281,9 @@ services: depends_on: seaweedfs: {condition: service_healthy} restart: "no" + environment: + FLUXER_S3_ACCESS_KEY: ${FLUXER_S3_ACCESS_KEY:?set FLUXER_S3_ACCESS_KEY in .env} + FLUXER_S3_SECRET_KEY: ${FLUXER_S3_SECRET_KEY:?set FLUXER_S3_SECRET_KEY in .env} entrypoint: - /bin/sh - -c @@ -293,6 +301,10 @@ services: echo "$$listed" | grep -q "^[[:space:]]*$$b[[:space:]]" || missing="$${missing:+$$missing }$$b"; done; if [ -z "$$missing" ]; then + if ! echo "s3.configure -user=fluxer -access_key=$$FLUXER_S3_ACCESS_KEY -secret_key=$$FLUXER_S3_SECRET_KEY -actions=Admin,Read,Write,List,Tagging -apply" | timeout 10 weed shell -master=seaweedfs:9333 >/dev/null 2>&1; then + echo "seaweedfs-init could not configure the S3 identity" >&2; + exit 1; + fi; echo "buckets ready"; exit 0; fi; @@ -312,14 +324,25 @@ services: memory: ${FLUXER_LIVEKIT_MEMORY_LIMIT:-512mb} restart: unless-stopped networks: [fluxer] - command: ["--config", "/etc/livekit.yaml"] environment: LIVEKIT_KEYS: "${LIVEKIT_API_KEY:?set LIVEKIT_API_KEY in .env}: ${LIVEKIT_API_SECRET:?set LIVEKIT_API_SECRET in .env}" - volumes: - - ./livekit.yaml:/etc/livekit.yaml:ro + LIVEKIT_CONFIG: | + port: 7880 + log_level: info + rtc: + tcp_port: ${FLUXER_LIVEKIT_TCP_PORT:-7881} + udp_port: ${FLUXER_LIVEKIT_UDP_PORT:-7882} + use_external_ip: true + stun_servers: + - stun.l.google.com:19302 + - stun1.l.google.com:19302 + webhook: + api_key: ${LIVEKIT_API_KEY:?set LIVEKIT_API_KEY in .env} + urls: + - http://api:8080/webhooks/livekit ports: - - "${FLUXER_LIVEKIT_TCP_PORT:-7881}:7881" - - "${FLUXER_LIVEKIT_UDP_PORT:-7882}:7882/udp" + - "${FLUXER_LIVEKIT_TCP_PORT:-7881}:${FLUXER_LIVEKIT_TCP_PORT:-7881}" + - "${FLUXER_LIVEKIT_UDP_PORT:-7882}:${FLUXER_LIVEKIT_UDP_PORT:-7882}/udp" healthcheck: test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:7880/"] interval: 10s @@ -373,7 +396,7 @@ services: reservations: memory: ${FLUXER_WORKER_MEMORY_RESERVATION:-1gb} working_dir: /usr/src/app/fluxer_api - command: ["node", "dist/WorkerEntrypoint.js"] + command: ["sh", "-c", "if [ -f dist/WorkerEntrypoint.js ]; then exec node dist/WorkerEntrypoint.js; else exec ./node_modules/.bin/tsx src/WorkerEntrypoint.ts; fi"] environment: <<: *fluxer-env NODE_OPTIONS: --enable-source-maps${FLUXER_WORKER_NODE_HEAP_MB:+ --max-old-space-size=$FLUXER_WORKER_NODE_HEAP_MB} @@ -412,6 +435,9 @@ services: FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media FLUXER_GATEWAY_STATIC_CDN_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}} FLUXER_GATEWAY_LOGGER_LEVEL: info + FLUXER_ERLANG_COOKIE: ${FLUXER_ERLANG_COOKIE:?set FLUXER_ERLANG_COOKIE in .env} + FLUXER_ERLANG_SCHEDULERS_MIN: "${FLUXER_ERLANG_SCHEDULERS_MIN:-2}" + FLUXER_ERLANG_SCHEDULERS_MAX: "${FLUXER_ERLANG_SCHEDULERS_MAX:-16}" healthcheck: test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:8080/_health/ready"] interval: 10s @@ -436,6 +462,7 @@ services: FLUXER_MEDIA_PROXY_MODE: upload FLUXER_MEDIA_PROXY_STORAGE_BACKEND: s3 FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media + FLUXER_S3_READ_SIGNED: "true" depends_on: seaweedfs-init: {condition: service_completed_successfully} nats: {condition: service_healthy} @@ -463,7 +490,7 @@ services: environment: FLUXER_APP_PROXY_HOST: 0.0.0.0 FLUXER_APP_PROXY_PORT: "8080" - DISCOVERY_UPSTREAM_URL: http://caddy:8088/api/.well-known/fluxer + DISCOVERY_UPSTREAM_URL: http://edge:8088/.well-known/fluxer PUBLIC_BOOTSTRAP_API_ENDPOINT: /api PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/api FLUXER_CSP_EXTRA_DEFAULT_SRC: ${FLUXER_CSP_EXTRA_DEFAULT_SRC:-} @@ -479,7 +506,7 @@ services: FLUXER_CSP_REPORT_URI: ${FLUXER_CSP_REPORT_URI:-} depends_on: api: {condition: service_healthy} - caddy: {condition: service_healthy} + edge: {condition: service_healthy} snowflakes: <<: *fluxer-service @@ -521,6 +548,7 @@ services: memory: ${FLUXER_USERS_MEMORY_LIMIT:-128mb} environment: <<: *fluxer-env + FLUXER_SVC_NAME: users FLUXER_SVC_MODE: router FLUXER_SVC_MAX_CONCURRENT_REQUESTS: "${FLUXER_SVC_MAX_CONCURRENT_REQUESTS:-20}" healthcheck: *fluxer-svc-healthcheck @@ -536,10 +564,11 @@ services: memory: ${FLUXER_USERS_SHARD_MEMORY_LIMIT:-256mb} environment: <<: *fluxer-env + FLUXER_SVC_NAME: users FLUXER_SVC_MODE: shard FLUXER_SVC_SHARD_ID: "0" FLUXER_POSTGRES_MAX_CONNECTIONS: "20" - FLUXER_SVC_MAX_CONCURRENT_REQUESTS: "20" + FLUXER_SVC_MAX_CONCURRENT_REQUESTS: "${FLUXER_SVC_MAX_CONCURRENT_REQUESTS:-20}" healthcheck: *fluxer-svc-healthcheck depends_on: nats: {condition: service_healthy} @@ -607,7 +636,7 @@ services: FLUXER_SVC_MODE: shard FLUXER_SVC_SHARD_ID: "0" FLUXER_POSTGRES_MAX_CONNECTIONS: "20" - FLUXER_SVC_MAX_CONCURRENT_REQUESTS: "20" + FLUXER_SVC_MAX_CONCURRENT_REQUESTS: "${FLUXER_SVC_MAX_CONCURRENT_REQUESTS:-20}" healthcheck: *fluxer-svc-healthcheck depends_on: nats: {condition: service_healthy} @@ -622,6 +651,7 @@ services: memory: ${FLUXER_UNFURL_MEMORY_LIMIT:-128mb} environment: <<: *fluxer-env + FLUXER_SVC_NAME: unfurl FLUXER_SVC_MODE: router FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}} @@ -638,6 +668,7 @@ services: memory: ${FLUXER_UNFURL_SHARD_MEMORY_LIMIT:-256mb} environment: <<: *fluxer-env + FLUXER_SVC_NAME: unfurl FLUXER_SVC_MODE: shard FLUXER_SVC_SHARD_ID: "0" FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media @@ -679,8 +710,8 @@ networks: driver: bridge volumes: - caddy-data: - caddy-config: + edge-data: + edge-config: postgres-data: valkey-data: nats-data: diff --git a/deploy/self-hosting/livekit.yaml b/deploy/self-hosting/livekit.yaml deleted file mode 100644 index 7755bdce2..000000000 --- a/deploy/self-hosting/livekit.yaml +++ /dev/null @@ -1,15 +0,0 @@ -port: 7880 -log_level: info - -rtc: - tcp_port: 7881 - udp_port: 7882 - use_external_ip: true - stun_servers: - - stun.l.google.com:19302 - - stun1.l.google.com:19302 - -webhook: - api_key: fluxer - urls: - - http://api:8080/webhooks/livekit diff --git a/deploy/self-hosting/tunnel.compose.yml b/deploy/self-hosting/tunnel.compose.yml index 66a6f7882..5872eea89 100644 --- a/deploy/self-hosting/tunnel.compose.yml +++ b/deploy/self-hosting/tunnel.compose.yml @@ -1,4 +1,4 @@ services: - caddy: + edge: ports: !override - "${FLUXER_HTTP_PORT:-127.0.0.1:80}:80" diff --git a/fluxer_gateway/scripts/docker_entrypoint.sh b/fluxer_gateway/scripts/docker_entrypoint.sh index bc9de4c1f..482f651a1 100755 --- a/fluxer_gateway/scripts/docker_entrypoint.sh +++ b/fluxer_gateway/scripts/docker_entrypoint.sh @@ -53,9 +53,13 @@ clamp_int() { : "${FLUXER_ERLANG_SCHEDULERS_MIN:=2}" : "${FLUXER_ERLANG_SCHEDULERS_MAX:=16}" : "${FLUXER_ERLANG_NODE_NAME:=fluxer_gateway@127.0.0.1}" -: "${FLUXER_ERLANG_COOKIE:=fluxer_gateway_dev_cookie}" : "${FLUXER_ERLANG_DIST_PORT:=8081}" +if [ -z "${FLUXER_ERLANG_COOKIE:-}" ]; then + echo 'FLUXER_ERLANG_COOKIE is required.' >&2 + exit 1 +fi + if ! is_positive_int "${FLUXER_ERLANG_SCHEDULERS:-}"; then FLUXER_ERLANG_SCHEDULERS="$(clamp_int "$(available_cpu_count)" "$FLUXER_ERLANG_SCHEDULERS_MIN" "$FLUXER_ERLANG_SCHEDULERS_MAX")" fi diff --git a/packages/config/src/__tests__/DeployEnvCoverage.test.ts b/packages/config/src/__tests__/DeployEnvCoverage.test.ts new file mode 100644 index 000000000..8f8c1e7c6 --- /dev/null +++ b/packages/config/src/__tests__/DeployEnvCoverage.test.ts @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {readFileSync} from 'node:fs'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; +import {describe, expect, test} from 'vitest'; + +const SELF_HOSTING = path.join(fileURLToPath(new URL('../../../../', import.meta.url)), 'deploy/self-hosting'); + +const INTERPOLATION_PATTERN = /\$\{([A-Z][A-Z0-9_]*)[:?}-]/g; +const CADDY_PLACEHOLDER_PATTERN = /\{\$([A-Z][A-Z0-9_]*)[:}]/g; +const DECLARATION_PATTERN = /^#?([A-Z][A-Z0-9_]*)=/gm; +const ACTIVE_DECLARATION_PATTERN = /^([A-Z][A-Z0-9_]*)=/gm; + +const read = (name: string) => readFileSync(path.join(SELF_HOSTING, name), 'utf8'); + +const namesMatching = (source: string, pattern: RegExp) => + new Set([...source.matchAll(pattern)].map(([, name]) => name)); + +const interpolatedNames = (source: string) => namesMatching(source, INTERPOLATION_PATTERN); + +const example = read('.env.example'); +const declared = new Set([...example.matchAll(DECLARATION_PATTERN)].map(([, name]) => name)); + +describe('.env.example covers every name the compose files interpolate', () => { + for (const file of ['docker-compose.yml', 'docker-compose.proxy.yml']) { + test(`every \${NAME} in ${file} has a line in .env.example`, () => { + const missing = [...interpolatedNames(read(file))].filter((name) => !declared.has(name)).sort(); + expect(missing).toEqual([]); + }); + } + + test('every {$NAME} in the Caddyfile has a line in .env.example', () => { + const missing = [...namesMatching(read('Caddyfile'), CADDY_PLACEHOLDER_PATTERN)] + .filter((name) => !declared.has(name)) + .sort(); + expect(missing).toEqual([]); + }); + + test('no name is assigned twice', () => { + const seen = new Set(); + const repeated = new Set(); + for (const [, name] of example.matchAll(ACTIVE_DECLARATION_PATTERN)) { + if (seen.has(name)) { + repeated.add(name); + } + seen.add(name); + } + expect([...repeated].sort()).toEqual([]); + }); +}); diff --git a/packages/config/src/__tests__/SelfHostingComposeEnvironment.test.ts b/packages/config/src/__tests__/SelfHostingComposeEnvironment.test.ts new file mode 100644 index 000000000..d0c6a9cb8 --- /dev/null +++ b/packages/config/src/__tests__/SelfHostingComposeEnvironment.test.ts @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {readFileSync} from 'node:fs'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; +import {describe, expect, test} from 'vitest'; + +const SELF_HOSTING = path.join(fileURLToPath(new URL('../../../../', import.meta.url)), 'deploy/self-hosting'); + +const compose = readFileSync(path.join(SELF_HOSTING, 'docker-compose.yml'), 'utf8'); + +const serviceBlock = (name: string): string => { + const start = compose.indexOf(`\n ${name}:\n`); + if (start === -1) { + throw new Error(`docker-compose.yml has no ${name} service`); + } + const rest = compose.slice(start + 1); + const next = rest.slice(1).search(/\n {2}[a-z][a-z0-9_-]*:\n/u); + return next === -1 ? rest : rest.slice(0, next + 1); +}; + +const sharedEnv = compose.slice(compose.indexOf('x-fluxer-env: &fluxer-env'), compose.indexOf('\nx-fluxer-service:')); + +describe('the shipped compose stack wires every service it starts', () => { + test('no service sizes a pool for a connection it cannot make', () => { + for (const [, name] of compose.matchAll(/\n {2}([a-z][a-z0-9_-]*):\n/gu)) { + const block = serviceBlock(name); + if (block.includes('FLUXER_POSTGRES_MAX_CONNECTIONS')) { + expect(block).toMatch(/<<: \*fluxer-(postgres-)?env/u); + } + } + }); + + test('the shared block sets the client-IP trust the merged services read', () => { + expect(sharedEnv).toContain('FLUXER_TRUST_CLIENT_IP_HEADER: "true"'); + expect(sharedEnv).toContain('FLUXER_CLIENT_IP_HEADER_NAME: x-forwarded-for'); + }); +}); diff --git a/packages/config/src/__tests__/SelfHostingObjectStoreAuth.test.ts b/packages/config/src/__tests__/SelfHostingObjectStoreAuth.test.ts new file mode 100644 index 000000000..021c6205c --- /dev/null +++ b/packages/config/src/__tests__/SelfHostingObjectStoreAuth.test.ts @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {readFileSync} from 'node:fs'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; +import {describe, expect, test} from 'vitest'; + +const SELF_HOSTING = path.join(fileURLToPath(new URL('../../../../', import.meta.url)), 'deploy/self-hosting'); + +const compose = readFileSync(path.join(SELF_HOSTING, 'docker-compose.yml'), 'utf8'); + +const serviceBlock = (name: string): string => { + const start = compose.indexOf(`\n ${name}:\n`); + if (start === -1) { + throw new Error(`docker-compose.yml has no ${name} service`); + } + const rest = compose.slice(start + 1); + const next = rest.slice(1).search(/\n {2}[a-z][a-z0-9_-]*:\n/u); + return next === -1 ? rest : rest.slice(0, next + 1); +}; + +describe('the shipped object store checks the credentials the stack sends', () => { + const init = serviceBlock('seaweedfs-init'); + + test('seaweedfs-init applies an S3 identity built from the .env credentials', () => { + expect(init).toContain( + 's3.configure -user=fluxer -access_key=$$FLUXER_S3_ACCESS_KEY -secret_key=$$FLUXER_S3_SECRET_KEY', + ); + expect(init).toContain('-apply'); + }); + + test('seaweedfs-init is handed the same credentials the api requires', () => { + for (const name of ['FLUXER_S3_ACCESS_KEY', 'FLUXER_S3_SECRET_KEY']) { + expect(init).toMatch(new RegExp(`${name}: \\$\\{${name}:\\?set ${name} in \\.env\\}`, 'u')); + } + }); + + test('a failed identity write fails the init instead of leaving the store open', () => { + const configure = init.indexOf('s3.configure'); + const ready = init.indexOf('echo "buckets ready"'); + expect(configure).toBeGreaterThan(-1); + expect(ready).toBeGreaterThan(configure); + expect(init).toContain('echo "seaweedfs-init could not configure the S3 identity" >&2'); + expect(init).toContain('exit 1'); + }); + + test('media-proxy signs its reads, which the store now refuses to serve unsigned', () => { + expect(serviceBlock('media-proxy')).toContain('FLUXER_S3_READ_SIGNED: "true"'); + }); + + test('every service that reaches the store waits for the identity to exist', () => { + for (const name of ['api', 'worker', 'media-proxy']) { + expect(serviceBlock(name)).toContain('seaweedfs-init: {condition: service_completed_successfully}'); + } + }); +});