pepepper/fluxer
1
0
Fork 0
mirror of https://github.com/fluxerapp/fluxer synced 2026-10-07 19:22:14 +09:00
Code Issues Packages Projects Releases Wiki Activity

refactor(self-hosting): forward every setting, drop dead config (#3047)

Browse Source
This commit is contained in:
Hampus
2026-09-30 00:58:43 +02:00
committed by GitHub
parent 39f9beda5a
commit 2b8a743dc5
96 changed files with 1743 additions and 2391 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.devcontainer/docker-compose.yml
-1
View File
@@ -23,7 +23,6 @@ services:
FLUXER_S3_PUBLIC_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"
FLUXER_LIVEKIT_URL: "ws://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/livekit"
FLUXER_LIVEKIT_INTERNAL_URL: "http://livekit:7880"
FLUXER_LIVEKIT_WEBHOOK_URL: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/api/webhooks/livekit"
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media"
FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}/media"
FLUXER_GATEWAY_STATIC_CDN_ENDPOINT: "http://localhost:${FLUXER_DEV_PROXY_PORT:-8088}"
Cargo.lock
Generated
+1 -1
View File
@@ -1904,7 +1904,6 @@ dependencies = [
"thiserror",
"tokio",
"tracing",
"tracing-subscriber",
"url",
]
@@ -2067,6 +2066,7 @@ dependencies = [
"thiserror",
"time",
"tracing",
"tracing-subscriber",
"url",
"urlencoding",
]
config/env/development.env
Vendored
-6
View File
@@ -34,7 +34,6 @@ FLUXER_KV_URL=redis://valkey:6379/0
FLUXER_NATS_URL=nats://nats:4222
FLUXER_NATS_JETSTREAM_URL=nats://nats:4222
FLUXER_INTERNAL_API_ENDPOINT=http://127.0.0.1:8080
FLUXER_INTERNAL_GATEWAY_ENDPOINT=http://127.0.0.1:8771
FLUXER_INTERNAL_MEDIA_PROXY_ENDPOINT=http://127.0.0.1:8082
FLUXER_MEDIA_PROXY_ENDPOINT=http://127.0.0.1:8082
FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT=http://localhost:8088/media
@@ -42,7 +41,6 @@ FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT=http://localhost:8088/media
FLUXER_SVC_NATS_URL=nats://nats:4222
FLUXER_SVC_SHARD_COUNT=1
FLUXER_SVC_CACHE_TTL_MS=30000
FLUXER_SVC_CACHE_HARD_TTL_MS=600000
FLUXER_S3_ENDPOINT=http://127.0.0.1:8333
FLUXER_S3_PUBLIC_ENDPOINT=http://localhost:8088
@@ -61,7 +59,6 @@ FLUXER_LIVEKIT_URL=ws://localhost:8088/livekit
FLUXER_LIVEKIT_INTERNAL_URL=http://localhost:7880
FLUXER_LIVEKIT_API_KEY=devkey
FLUXER_LIVEKIT_API_SECRET=fluxer-livekit-development-secret
FLUXER_LIVEKIT_WEBHOOK_URL=http://localhost:8088/api/webhooks/livekit
FLUXER_LIVEKIT_DEFAULT_REGION={"id":"local","name":"Local","emoji":"LC","latitude":59.3293,"longitude":18.0686}
FLUXER_API_PORT=8080
@@ -128,6 +125,3 @@ PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT=http://localhost:8088/api
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64=Zmx1eGVyLWRldi11cGxvYWQtcmVsYXktc2VjcmV0LTAwMDA=
FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64=Zmx1eGVyLWRldi1hdHRhY2htZW50LXVybC1zZWNyZXQ=
AWS_EC2_METADATA_DISABLED=true
AWS_ACCESS_KEY_ID=fluxer
AWS_SECRET_ACCESS_KEY=fluxer-secret
AWS_DEFAULT_REGION=us-east-1
deploy/self-hosting/.env.example
+318 -23
View File
@@ -1,6 +1,8 @@
# Every variable docker-compose.yml reads, uncommented when it has no default and
# commented with its default when it has one. Compose expands top to bottom, so a
# line using ${...} must sit below every name it reads.
# Every variable docker-compose.yml reads. A value an install must set is
# uncommented. A commented line shows the default, or an example where its comment
# says so, and nothing after = means the service decides. An empty value keeps the
# default too. Compose expands top to bottom, so a line using ${...} must sit below
# every name it reads.
FLUXER_DOMAIN=chat.example.com
FLUXER_PUBLIC_SCHEME=https
@@ -68,7 +70,9 @@ FLUXER_IMAGE_TAG=v1
POSTGRES_PASSWORD=CHANGE_ME
MEILI_MASTER_KEY=CHANGE_ME
# Set these to run Postgres or the object store outside the stack. Backing up a
# store you moved out is yours to arrange, and an upgrade skips it.
# store you moved out is yours to arrange. An upgrade dumps the bundled postgres
# service and skips the dump only when the stack defines none. The values below
# are examples.
#FLUXER_POSTGRES_HOST=db.example.com
#FLUXER_POSTGRES_PORT=5432
#FLUXER_POSTGRES_DATABASE=fluxer
@@ -78,6 +82,7 @@ MEILI_MASTER_KEY=CHANGE_ME
#FLUXER_S3_PUBLIC_ENDPOINT=https://cdn.example.com
#FLUXER_S3_REGION=eu-central-1
#FLUXER_S3_FORCE_PATH_STYLE=false
# Bucket names. The bundled store creates these. An outside store needs them to
# exist already.
#FLUXER_S3_BUCKET_CDN=fluxer
@@ -90,34 +95,89 @@ MEILI_MASTER_KEY=CHANGE_ME
# Needs Compose 2.24.4 or newer.
#COMPOSE_FILE=docker-compose.yml:external-object-store.compose.yml
# A full connection URL wins over the host, port and database above. The URL is an
# example. The CA is the PEM text of the certificate, with \n for line breaks.
#FLUXER_POSTGRES_URL=postgres://fluxer:[email protected]:5432/fluxer
#FLUXER_POSTGRES_SSL_CA=
# The Postgres table that holds the key-value store.
#FLUXER_POSTGRES_KV_TABLE=fluxer_kv
# media-proxy reads through these when the store serves reads from another
# address or bucket.
#FLUXER_S3_READ_ENDPOINT=
#FLUXER_S3_READ_BUCKET=
#FLUXER_S3_READ_BUCKET_STYLE=
# A temporary S3 session token, read by media-proxy only.
#FLUXER_S3_SESSION_TOKEN=
# The bundled store refuses unsigned reads. Set false only for a public-read bucket.
#FLUXER_S3_READ_SIGNED=true
# The other bundled services, pointed elsewhere. Removing a service from the
# stack belongs in an override file, since an upgrade replaces docker-compose.yml.
# The URLs below are examples.
#FLUXER_KV_URL=redis://cache.example.com:6379/0
#FLUXER_NATS_URL=nats://mq.example.com:4222
#FLUXER_NATS_JETSTREAM_URL=nats://mq.example.com:4222
#FLUXER_SVC_NATS_URL=nats://mq.example.com:4222
#FLUXER_SEARCH_URL=https://search.example.com
#FLUXER_LIVEKIT_INTERNAL_URL=http://livekit.example.com:7880
# How the stack talks to those services.
#FLUXER_KV_MODE=standalone
#FLUXER_SEARCH_ENGINE=meilisearch
#FLUXER_SEARCH_USERNAME=
#FLUXER_SEARCH_PASSWORD=
#FLUXER_SEARCH_TLS_REJECT_UNAUTHORIZED=true
# Voice off. The livekit service still runs until an override removes it.
#FLUXER_LIVEKIT_ENABLED=false
# Optional systems, each off unless configured.
#FLUXER_STRIPE_ENABLED=false
#FLUXER_NCMEC_ENABLED=false
#FLUXER_CLAMAV_ENABLED=false
#FLUXER_STRIPE_SECRET_KEY=
#FLUXER_STRIPE_WEBHOOK_SECRET=
# Stripe prices as one JSON object. The admin dashboard can set them instead.
#FLUXER_STRIPE_PRICES={}
#FLUXER_STRIPE_LEGACY_PRICES={}
#FLUXER_API_DONATION_PROXY_KEY=
#FLUXER_VISIONARIES_GUILD_ID=
#FLUXER_VISIONARIES_GUILD_VISIONARY_ROLE_ID=
# NCMEC CyberTipline reporting, off by default. All four values are required
# once it is on. The values below are examples.
#FLUXER_NCMEC_ENABLED=true
#FLUXER_NCMEC_BASE_URL=https://report.cybertip.org/ispws
#FLUXER_NCMEC_USERNAME=
#FLUXER_NCMEC_PASSWORD=
#[email protected]
# Upload virus scanning, off by default. No ClamAV container ships, so point
# this at your own. The values below are examples.
#FLUXER_CLAMAV_ENABLED=true
#FLUXER_CLAMAV_HOST=clamav
#FLUXER_CLAMAV_PORT=3310
#FLUXER_CLAMAV_FAIL_OPEN=false
# Outside lookups, off unless turned on. The breached password check asks
# api.pwnedpasswords.com.
#FLUXER_BREACHED_PASSWORD_CHECK_ENABLED=true
#FLUXER_BREACHED_PASSWORD_CHECK_ENABLED=false
#FLUXER_BLOCKLIST_FEEDS_ENABLED=false
#FLUXER_IPINFO_API_KEY=
# A local path, or an s3:// URL read with the S3 credentials of this file.
#FLUXER_GEOIP_DB_PATH=
# The client address. Name the header your proxy actually writes, and turn the
# trust off when nothing sits in front.
#FLUXER_CLIENT_IP_HEADER_NAME=cf-connecting-ip
# The client address. The edge sets X-Forwarded-For on every hop, so keep the
# trust on and the default header. Turning the trust off makes the api refuse
# every request outside its exempt routes with a 403.
#FLUXER_CLIENT_IP_HEADER_NAME=x-forwarded-for
#FLUXER_TRUST_CLIENT_IP_HEADER=true
# How much the services write. trace, debug, info, warn, error or fatal.
#LOG_LEVEL=debug
# How much the services write. LOG_LEVEL covers the api and worker and takes trace,
# debug, info, warn, error or fatal. RUST_LOG covers the Rust services and takes
# an EnvFilter such as debug. The gateway takes an Erlang level such as notice,
# and LOGGER_LEVEL beats FLUXER_GATEWAY_LOGGER_LEVEL.
#LOG_LEVEL=info
#RUST_LOG=info
#FLUXER_GATEWAY_LOGGER_LEVEL=info
#LOGGER_LEVEL=
FLUXER_S3_ACCESS_KEY=fluxer
FLUXER_S3_SECRET_KEY=CHANGE_ME
@@ -142,7 +202,7 @@ FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME
# exist.
#[email protected]
# Passkeys follow FLUXER_DOMAIN. Set these only if browsers use another host.
# The passkey RP ID defaults to FLUXER_DOMAIN, whatever FLUXER_PUBLIC_ORIGIN says.
# Changing the RP ID invalidates every passkey registered against the old value.
#FLUXER_PASSKEY_RP_ID=chat.example.com
#FLUXER_PASSKEY_RP_NAME=Fluxer
@@ -153,6 +213,29 @@ FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME
#FLUXER_PUSH_SERVICE_QUEUE_CAPACITY=10000
# Provider requests the push container sends at once, 1 to 65536.
#FLUXER_PUSH_SERVICE_SEND_CONCURRENCY=256
# The push container's provider addresses and relay hosts.
#FLUXER_PUSH_SERVICE_APNS_BASE_URL=
#FLUXER_PUSH_SERVICE_FCM_BASE_URL=https://fcm.googleapis.com
#FLUXER_PUSH_SERVICE_MANAGED_RELAY_HOSTS=push.fluxer.com
#FLUXER_PUSH_SERVICE_OWN_RELAY_HOSTS=
#FLUXER_PUSH_SERVICE_RELAY_CONSENT_ACCEPTED=false
# Direct mobile push through your own APNs and FCM credentials, off by default.
#FLUXER_PUSH_APNS_ENABLED=false
#FLUXER_PUSH_APNS_TEAM_ID=
#FLUXER_PUSH_APNS_KEY_ID=
#FLUXER_PUSH_APNS_PRIVATE_KEY=
#FLUXER_PUSH_APNS_PRIVATE_KEY_PATH=
#FLUXER_PUSH_APNS_APPS=
#FLUXER_PUSH_APNS_DEFAULT_ENVIRONMENT=production
#FLUXER_PUSH_FCM_ENABLED=false
#FLUXER_PUSH_FCM_PROJECT_ID=
#FLUXER_PUSH_FCM_CLIENT_EMAIL=
#FLUXER_PUSH_FCM_PRIVATE_KEY=
#FLUXER_PUSH_FCM_PRIVATE_KEY_PATH=
#FLUXER_PUSH_FCM_SERVICE_ACCOUNT_JSON_PATH=
#FLUXER_PUSH_FCM_TOKEN_URI=https://oauth2.googleapis.com/token
#FLUXER_PUSH_FCM_APPS=
# Optional media policies, both off by default. See the operator docs.
@@ -164,8 +247,9 @@ FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME
# working. Needs a secret from openssl rand -base64 32, first entry signs and
# every entry verifies.
#
# Each mode is off, report or enforce. Start at report. media-proxy reads these
# at start, so apply with docker compose up -d media-proxy.
# Each mode is off, report or enforce, and off is the default. Start at report.
# media-proxy reads these at start, so apply with docker compose up -d
# media-proxy. The values below are examples.
#FLUXER_MEDIA_PROXY_CORS_MODE=enforce
#FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS=https://chat.example.com,https://web.fluxer.app
#FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64=
@@ -190,7 +274,7 @@ FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME
# Let the SSO provider resolve to a private address. Off by default, so a
# misconfigured provider URL cannot reach internal services. Turn it on only for
# a provider on your own network.
# a provider on your own network. The value below is an example.
#FLUXER_SSO_ALLOW_PRIVATE_ADDRESSES=true
# These reach both LiveKit and the api. Change them together.
@@ -207,13 +291,20 @@ LIVEKIT_API_SECRET=CHANGE_ME
# LiveKit finds its public address over STUN. A host that cannot reach one stops
# with "could not resolve external IP", so set the address by hand instead, or
# point STUN elsewhere.
# point STUN elsewhere. The values below are examples.
#FLUXER_LIVEKIT_USE_EXTERNAL_IP=false
#FLUXER_LIVEKIT_NODE_IP=203.0.113.10
#FLUXER_LIVEKIT_STUN_PRIMARY=stun.l.google.com:19302
#FLUXER_LIVEKIT_STUN_SECONDARY=stun1.l.google.com:19302
# The voice region users see, and how much LiveKit logs.
#FLUXER_LIVEKIT_DEFAULT_REGION={"id":"default","name":"Default","emoji":"🌍","latitude":0,"longitude":0}
#FLUXER_LIVEKIT_LOG_LEVEL=info
FLUXER_KLIPY_API_KEY=
#FLUXER_YOUTUBE_API_KEY=
# Hosts the api never unfurls, comma separated.
#FLUXER_API_UNFURL_IGNORED_HOSTS=
FLUXER_EMAIL_ENABLED=false
FLUXER_EMAIL_PROVIDER=none
@@ -225,8 +316,93 @@ FLUXER_EMAIL_SMTP_PORT=587
FLUXER_EMAIL_SMTP_USERNAME=
FLUXER_EMAIL_SMTP_PASSWORD=
FLUXER_EMAIL_SMTP_SECURE=true
#FLUXER_EMAIL_WEBHOOK_SECRET=
FLUXER_DISCOVERY_ENABLED=true
#FLUXER_DISCOVERY_MIN_MEMBER_COUNT=1
# Instance identity and account policy.
#FLUXER_APP_PRODUCT_NAME=Fluxer
#FLUXER_APP_ICON_URL=
#FLUXER_APP_SYMBOL_URL=
#FLUXER_APP_LOGO_URL=
#FLUXER_APP_WORDMARK_URL=
#FLUXER_APP_FAVICON_URL=
#FLUXER_APP_THEME_COLOR=
#FLUXER_APP_STATUS_PAGE_URL=
#FLUXER_APP_STATUS_PAGE_INCIDENT_HISTORY_URL=
#FLUXER_INSTANCE_SETUP_CONFIGURED=false
#FLUXER_AUTO_JOIN_INVITE_CODE=
#FLUXER_DELETION_GRACE_PERIOD_HOURS=336
# Sign in with Bluesky, off unless turned on.
#FLUXER_AUTH_BLUESKY_ENABLED=false
#FLUXER_AUTH_BLUESKY_CLIENT_NAME=Fluxer
#FLUXER_AUTH_BLUESKY_CLIENT_URI=
#FLUXER_AUTH_BLUESKY_LOGO_URI=
#FLUXER_AUTH_BLUESKY_TOS_URI=
#FLUXER_AUTH_BLUESKY_POLICY_URI=
#FLUXER_AUTH_BLUESKY_KEYS=
# Public addresses. Each follows the public origin unless set here.
#FLUXER_API_ENDPOINT=
#FLUXER_API_CLIENT_ENDPOINT=
#FLUXER_APP_ENDPOINT=
#FLUXER_GATEWAY_ENDPOINT=
#FLUXER_MEDIA_ENDPOINT=
#FLUXER_STATIC_CDN_ENDPOINT=
#FLUXER_ADMIN_ENDPOINT=
#FLUXER_MARKETING_ENDPOINT=
#FLUXER_INVITE_ENDPOINT=
#FLUXER_GIFT_ENDPOINT=
#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT=
#PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT=
# These follow FLUXER_STATIC_CDN_ENDPOINT first, then the public origin.
#FLUXER_GATEWAY_STATIC_CDN_ENDPOINT=
#FLUXER_UNFURL_STATIC_CDN_ENDPOINT=
# These follow FLUXER_MEDIA_ENDPOINT first, then the public origin.
#FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT=
#FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT=
# Extra hosts for static assets, invites, gifts and the web app. Empty by default.
#FLUXER_STATIC_CDN_DOMAIN=
#FLUXER_INVITE_DOMAIN=
#FLUXER_GIFT_DOMAIN=
#FLUXER_APP_ORIGIN_ALIASES=
# The path the admin panel is served under. The edge and the admin service read
# it. The api follows it through the default FLUXER_ADMIN_ENDPOINT, and not when
# FLUXER_ADMIN_ENDPOINT is set. Write it with a leading slash and no trailing
# slash.
#FLUXER_ADMIN_BASE_PATH=/admin
# The compression the edge offers, as Caddy encode arguments.
#FLUXER_EDGE_ENCODE=zstd gzip
# Images of the bundled services, for a mirror or another tag. A new Postgres
# major needs a dump and restore, as the upgrade guide describes.
#FLUXER_CADDY_IMAGE=caddy:2.11-alpine
#FLUXER_POSTGRES_IMAGE=postgres:16-alpine
#FLUXER_VALKEY_IMAGE=valkey/valkey:9.1-alpine
#FLUXER_NATS_IMAGE=nats:2.14-alpine
#FLUXER_MEILISEARCH_IMAGE=getmeili/meilisearch:v1.53
#FLUXER_SEAWEEDFS_IMAGE=chrislusf/seaweedfs:4.47
#FLUXER_LIVEKIT_IMAGE=livekit/livekit-server:v1.12.0
# Restart policy for every long-running service.
#FLUXER_RESTART_POLICY=unless-stopped
# Health checks. Raise the retries or start periods on a slow host.
#FLUXER_HEALTHCHECK_INTERVAL=10s
#FLUXER_HEALTHCHECK_TIMEOUT=5s
#FLUXER_HEALTHCHECK_RETRIES=10
#FLUXER_APP_HEALTHCHECK_RETRIES=30
#FLUXER_APP_HEALTHCHECK_START_PERIOD=90s
#FLUXER_SVC_HEALTHCHECK_START_PERIOD=60s
#FLUXER_WORKER_HEALTHCHECK_RETRIES=3
#FLUXER_SEAWEEDFS_HEALTHCHECK_RETRIES=20
#FLUXER_SEAWEEDFS_HEALTHCHECK_START_PERIOD=60s
#FLUXER_SEAWEEDFS_INIT_ATTEMPTS=60
# Container memory. These are ceilings, not allocations, and the defaults suit a
# 16 GB host. The reservations bias the kernel away from reclaiming from services
@@ -264,18 +440,31 @@ FLUXER_DISCOVERY_ENABLED=true
# Meilisearch indexing memory. Keep it well under the container limit above.
#FLUXER_MEILISEARCH_MAX_INDEXING_MEMORY=384mb
#FLUXER_MEILISEARCH_ENV=production
#FLUXER_MEILISEARCH_NO_ANALYTICS=true
# SeaweedFS heap ceiling. Go cannot see the container limit, so without this an
# upload burst gets the container OOM-killed. Keep it near three quarters of
# FLUXER_SEAWEEDFS_MEMORY_LIMIT and raise both together.
#FLUXER_SEAWEEDFS_GOMEMLIMIT=1536MiB
#FLUXER_SEAWEEDFS_TELEMETRY=false
# Node sizes its heap from the container limit by default. Leave these unset
# unless you need to pin it. A heap ceiling above the container limit gets the
# container OOM-killed instead of reporting a heap error.
# container OOM-killed instead of reporting a heap error. The values below are
# examples.
#FLUXER_API_NODE_HEAP_MB=1792
#FLUXER_WORKER_NODE_HEAP_MB=1792
# Extra Node flags for api and worker, appended to NODE_OPTIONS. Empty by
# default. The value below is an example.
#FLUXER_API_NODE_OPTIONS=--heapsnapshot-near-heap-limit=1
#FLUXER_WORKER_NODE_OPTIONS=--heapsnapshot-near-heap-limit=1
# Extra CA certificates api and worker trust, as a PEM bundle path inside the
# container. The default is the image's system bundle.
#FLUXER_NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt
# Bundled Postgres tuning. Keep it consistent with the memory limit above. This
# is the server setting, not the per-service pool sizes.
#FLUXER_POSTGRES_SERVER_MAX_CONNECTIONS=150
@@ -285,23 +474,81 @@ FLUXER_DISCOVERY_ENABLED=true
#FLUXER_POSTGRES_MAINTENANCE_WORK_MEM=256MB
#FLUXER_POSTGRES_AUTOVACUUM_WORK_MEM=128MB
#FLUXER_POSTGRES_SHM_SIZE=1gb
#FLUXER_POSTGRES_RANDOM_PAGE_COST=1.1
#FLUXER_POSTGRES_EFFECTIVE_IO_CONCURRENCY=200
#FLUXER_POSTGRES_DEFAULT_STATISTICS_TARGET=200
#FLUXER_POSTGRES_JIT=off
#FLUXER_POSTGRES_MIN_WAL_SIZE=512MB
#FLUXER_POSTGRES_MAX_WAL_SIZE=2GB
#FLUXER_POSTGRES_CHECKPOINT_COMPLETION_TARGET=0.9
#FLUXER_POSTGRES_WAL_BUFFERS=16MB
#FLUXER_POSTGRES_WAL_COMPRESSION=zstd
#FLUXER_POSTGRES_BGWRITER_DELAY=50ms
#FLUXER_POSTGRES_BGWRITER_LRU_MAXPAGES=1000
#FLUXER_POSTGRES_AUTOVACUUM_VACUUM_SCALE_FACTOR=0.05
#FLUXER_POSTGRES_AUTOVACUUM_ANALYZE_SCALE_FACTOR=0.02
#FLUXER_POSTGRES_AUTOVACUUM_VACUUM_COST_LIMIT=2000
#FLUXER_POSTGRES_TRACK_IO_TIMING=on
#FLUXER_POSTGRES_SHARED_PRELOAD_LIBRARIES=pg_stat_statements
# Postgres pool size of each service that opens a pool.
#FLUXER_API_POSTGRES_MAX_CONNECTIONS=25
#FLUXER_WORKER_POSTGRES_MAX_CONNECTIONS=25
#FLUXER_USERS_SHARD_POSTGRES_MAX_CONNECTIONS=20
#FLUXER_MESSAGES_SHARD_POSTGRES_MAX_CONNECTIONS=20
# The bundled Valkey holds durable state as well as cache, so it runs with an
# append-only file and with noeviction, which fails an over-limit write instead
# of dropping queued work. Change the policy only if that state lives elsewhere.
#FLUXER_VALKEY_MAXMEMORY=192mb
#FLUXER_VALKEY_MAXMEMORY_POLICY=noeviction
#FLUXER_VALKEY_APPENDFSYNC=everysec
# The gateway derives its scheduler count from the CPU quota, clamped here. One
# scheduler lets a single blocking operation stall every websocket on the node.
#FLUXER_ERLANG_SCHEDULERS_MIN=2
#FLUXER_ERLANG_SCHEDULERS_MAX=16
# A fixed scheduler count skips the clamp. Dirty CPU schedulers default to two
# thirds of it.
#FLUXER_ERLANG_SCHEDULERS=
#FLUXER_ERLANG_DIRTY_CPU_SCHEDULERS=
# In-flight request ceiling for the users and messages routers and their shards.
# One value replaces the built-in default on all of them, so size it for the
# busiest. Too low a value rejects requests rather than slowing them, and the api
# turns that into a 503.
#FLUXER_SVC_MAX_CONCURRENT_REQUESTS=192
# Gateway push and RPC tuning.
#FLUXER_GATEWAY_PUSH_ENABLED=true
#FLUXER_GATEWAY_PUSH_ENROLLED_CLEAR_NOTIFICATIONS_ENABLED=true
#FLUXER_GATEWAY_PUSH_OUTBOX_REQUEST_TIMEOUT_MS=100000
#FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_ENTRIES=128
#FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_BYTES=1048576
#FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY=512
#FLUXER_GATEWAY_NATS_RPC_MAX_HANDLERS=512
#FLUXER_GATEWAY_HTTP_FAILURE_THRESHOLD=6
#FLUXER_GATEWAY_HTTP_RECOVERY_TIMEOUT_MS=15000
# In-flight request ceiling for every svc router and shard. Unset, each keeps its
# own default: 192 for messages, 320 for snowflakes and 64 for the rest. One value
# replaces all of them, so size it for the busiest. Too low a value rejects
# requests rather than slowing them, and the api turns that into a 503. The value
# below is an example.
#FLUXER_SVC_MAX_CONCURRENT_REQUESTS=320
# svc caches, and how the api calls the svc services over NATS.
#FLUXER_SVC_CACHE_MAX_ENTRIES=100000
#FLUXER_SVC_CACHE_TTL_MS=30000
#FLUXER_GIFS_SHARD_CACHE_MAX_BYTES=536870912
#FLUXER_GIF_SERVICE_NATS_CLIENT_NAME=fluxer-api-gifs
#FLUXER_GIF_SERVICE_TIMEOUT_MS=12000
#FLUXER_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS=3000
#FLUXER_USERS_SERVICE_NATS_CLIENT_NAME=fluxer-api-users
#FLUXER_USERS_SERVICE_TIMEOUT_MS=6000
#FLUXER_USERS_SERVICE_INFLIGHT_MAX_ENTRIES=10000
#FLUXER_SNOWFLAKE_SERVICE_NATS_CLIENT_NAME=fluxer-api-snowflakes
#FLUXER_SNOWFLAKE_SERVICE_BATCH_SIZE=128
#FLUXER_SNOWFLAKE_SERVICE_LOW_WATERMARK=
#FLUXER_SNOWFLAKE_SERVICE_MAX_BUFFER_AGE_MS=5000
#FLUXER_SNOWFLAKE_SERVICE_REQUEST_TIMEOUT_MS=6000
# Worker concurrency per lane, as a JSON object keyed by lane.
#FLUXER_API_WORKER_LANE_CONCURRENCY_OVERRIDES=
# Named prepared statements need a session that outlives the transaction, so set
# this to false behind a transaction-pooling connection pooler. The bundled
@@ -313,3 +560,51 @@ FLUXER_DISCOVERY_ENABLED=true
# is clamped down to the second. Milliseconds, 1000 to 3600000.
#FLUXER_API_HEADERS_TIMEOUT_MS=30000
#FLUXER_API_REQUEST_TIMEOUT_MS=120000
# api request limits and IP bans. A refresh interval of 0 stops the periodic
# ban reload.
#FLUXER_API_MAX_INFLIGHT_REQUESTS=512
#FLUXER_API_IP_BAN_EXEMPT_IPS=
#FLUXER_IP_BAN_REFRESH_INTERVAL_MS=300000
# Uploads and data exports. Presigned exports link to FLUXER_S3_PUBLIC_ENDPOINT,
# so turn them on only once browsers can reach it.
#FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED=true
#FLUXER_API_PRESIGNED_HARVEST_DOWNLOADS_ENABLED=false
#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_MAX_BODY_BYTES=524288000
#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_TOKEN_TTL_SECS=900
#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_KEEP_DIRECT_COUNTRIES=
#FLUXER_API_STORAGE_CHANGE_FEED_ENABLED=false
#FLUXER_API_STORAGE_CHANGE_FEED_STREAM=STORAGE_CHANGES
#FLUXER_API_STORAGE_CHANGE_FEED_SKIP_BUCKETS=
#FLUXER_CACHE_PURGE_ADAPTER=none
#FLUXER_CACHE_PURGE_HTTP_ENDPOINT=
#FLUXER_CACHE_PURGE_HTTP_TOKEN=
#FLUXER_CACHE_PURGE_HTTP_TIMEOUT_MS=10000
# media-proxy limits and timeouts.
#FLUXER_MEDIA_PROXY_READ_ONLY=false
#FLUXER_MEDIA_PROXY_NSFW_THRESHOLD=0.85
#FLUXER_NSFW_SERVICE_ENDPOINT=
#FLUXER_MEDIA_PROXY_MAX_NATIVE_TRANSFORMS=
#FLUXER_MEDIA_PROXY_WORKER_QUEUE_CAPACITY=
#FLUXER_MEDIA_PROXY_MAX_ENCODE_DURATION_MS=30000
#FLUXER_MEDIA_PROXY_MAX_ENCODE_FRAMES=20000
#FLUXER_MEDIA_PROXY_TRANSFORM_TIMEOUT_MS=15000
#FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_BYTES=268435456
#FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_MAX_ENTRY_BYTES=67108864
#FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_TTL_MS=120000
#FLUXER_MEDIA_PROXY_SOCKET_IO_TIMEOUT_MS=30000
#FLUXER_MEDIA_PROXY_SHUTDOWN_GRACE_MS=30000
#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_S3_TIMEOUT_MS=900000
#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_BYTES=33554432
#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_TOTAL_BYTES=536870912
#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_DIR=
#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_CHUNK_BYTES=1048576
#FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_MAX_TOTAL_BYTES=8589934592
# app-proxy discovery refresh, index upstream and manifest scope.
#DISCOVERY_REFRESH_INTERVAL_MS=60000
#FLUXER_APP_PROXY_INDEX_UPSTREAM_URL=
#FLUXER_APP_PROXY_SAME_ORIGIN_HOSTS=
#FLUXER_APP_PROXY_MANIFEST_SCOPE_EXTENSIONS=
deploy/self-hosting/Caddyfile
+3 -3
View File
@@ -6,7 +6,7 @@
}
{$FLUXER_EDGE_SITE_ADDRESS} {
encode zstd gzip
encode {$FLUXER_EDGE_ENCODE:zstd gzip}
handle /_health {
respond "OK" 200
@@ -33,12 +33,12 @@
reverse_proxy livekit:7880
}
handle /admin {
handle {$FLUXER_ADMIN_BASE_PATH:/admin} {
rewrite * /
reverse_proxy admin:8080
}
handle_path /admin/* {
handle_path {$FLUXER_ADMIN_BASE_PATH:/admin}/* {
reverse_proxy admin:8080
}
deploy/self-hosting/docker-compose.yml
+308 -179
View File
@@ -3,39 +3,82 @@ name: fluxer
x-fluxer-postgres-env: &fluxer-postgres-env
FLUXER_DATABASE_BACKEND: postgres
FLUXER_POSTGRES_HOST: ${FLUXER_POSTGRES_HOST:-postgres}
FLUXER_POSTGRES_PORT: "${FLUXER_POSTGRES_PORT:-5432}"
FLUXER_POSTGRES_DATABASE: ${FLUXER_POSTGRES_DATABASE:-fluxer}
FLUXER_POSTGRES_USERNAME: ${FLUXER_POSTGRES_USERNAME:-fluxer}
FLUXER_POSTGRES_PORT: ${FLUXER_POSTGRES_PORT:-}
FLUXER_POSTGRES_DATABASE: ${FLUXER_POSTGRES_DATABASE:-}
FLUXER_POSTGRES_USERNAME: ${FLUXER_POSTGRES_USERNAME:-}
FLUXER_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
FLUXER_POSTGRES_SSL: "${FLUXER_POSTGRES_SSL:-false}"
FLUXER_POSTGRES_PREPARED_STATEMENTS: ${FLUXER_POSTGRES_PREPARED_STATEMENTS:-true}
FLUXER_POSTGRES_URL: ${FLUXER_POSTGRES_URL:-}
FLUXER_POSTGRES_SSL: ${FLUXER_POSTGRES_SSL:-}
FLUXER_POSTGRES_SSL_CA: ${FLUXER_POSTGRES_SSL_CA:-}
FLUXER_POSTGRES_PREPARED_STATEMENTS: ${FLUXER_POSTGRES_PREPARED_STATEMENTS:-}
FLUXER_POSTGRES_KV_TABLE: ${FLUXER_POSTGRES_KV_TABLE:-}
x-fluxer-env: &fluxer-env
<<: *fluxer-postgres-env
FLUXER_ENV: production
NODE_ENV: production
LOG_LEVEL: ${LOG_LEVEL:-info}
LOG_LEVEL: ${LOG_LEVEL:-}
RUST_LOG: ${RUST_LOG:-}
FLUXER_SELF_HOSTED: "true"
FLUXER_BASE_DOMAIN: ${FLUXER_DOMAIN:?set FLUXER_DOMAIN in .env}
FLUXER_PUBLIC_SCHEME: ${FLUXER_PUBLIC_SCHEME:-https}
FLUXER_PUBLIC_PORT: ${FLUXER_PUBLIC_PORT:-443}
FLUXER_PUBLIC_ORIGIN: ${FLUXER_PUBLIC_ORIGIN:-}
FLUXER_TRUST_CLIENT_IP_HEADER: "${FLUXER_TRUST_CLIENT_IP_HEADER:-true}"
FLUXER_CLIENT_IP_HEADER_NAME: ${FLUXER_CLIENT_IP_HEADER_NAME:-x-forwarded-for}
FLUXER_API_HEADERS_TIMEOUT_MS: ${FLUXER_API_HEADERS_TIMEOUT_MS:-30000}
FLUXER_API_REQUEST_TIMEOUT_MS: ${FLUXER_API_REQUEST_TIMEOUT_MS:-120000}
FLUXER_BREACHED_PASSWORD_CHECK_ENABLED: "${FLUXER_BREACHED_PASSWORD_CHECK_ENABLED:-false}"
FLUXER_CLIENT_IP_HEADER_NAME: ${FLUXER_CLIENT_IP_HEADER_NAME:-}
FLUXER_API_HEADERS_TIMEOUT_MS: ${FLUXER_API_HEADERS_TIMEOUT_MS:-}
FLUXER_API_REQUEST_TIMEOUT_MS: ${FLUXER_API_REQUEST_TIMEOUT_MS:-}
FLUXER_API_MAX_INFLIGHT_REQUESTS: ${FLUXER_API_MAX_INFLIGHT_REQUESTS:-}
FLUXER_API_IP_BAN_EXEMPT_IPS: ${FLUXER_API_IP_BAN_EXEMPT_IPS:-}
FLUXER_IP_BAN_REFRESH_INTERVAL_MS: ${FLUXER_IP_BAN_REFRESH_INTERVAL_MS:-}
FLUXER_APP_ORIGIN_ALIASES: ${FLUXER_APP_ORIGIN_ALIASES:-}
FLUXER_BREACHED_PASSWORD_CHECK_ENABLED: ${FLUXER_BREACHED_PASSWORD_CHECK_ENABLED:-}
FLUXER_BLOCKLIST_FEEDS_ENABLED: ${FLUXER_BLOCKLIST_FEEDS_ENABLED:-}
FLUXER_IPINFO_API_KEY: ${FLUXER_IPINFO_API_KEY:-}
FLUXER_GEOIP_DB_PATH: ${FLUXER_GEOIP_DB_PATH:-}
FLUXER_API_ENDPOINT: ${FLUXER_API_ENDPOINT:-}
FLUXER_API_CLIENT_ENDPOINT: ${FLUXER_API_CLIENT_ENDPOINT:-}
FLUXER_APP_ENDPOINT: ${FLUXER_APP_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}}
FLUXER_GATEWAY_ENDPOINT: ${FLUXER_GATEWAY_ENDPOINT:-}
FLUXER_MEDIA_ENDPOINT: ${FLUXER_MEDIA_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media}
FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT:-${FLUXER_MEDIA_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media}}
FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_STATIC_CDN_ENDPOINT:-}
FLUXER_ADMIN_ENDPOINT: ${FLUXER_ADMIN_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}${FLUXER_ADMIN_BASE_PATH:-/admin}}
FLUXER_MARKETING_ENDPOINT: ${FLUXER_MARKETING_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}}
FLUXER_INVITE_ENDPOINT: ${FLUXER_INVITE_ENDPOINT:-}
FLUXER_GIFT_ENDPOINT: ${FLUXER_GIFT_ENDPOINT:-}
FLUXER_STATIC_CDN_DOMAIN: ${FLUXER_STATIC_CDN_DOMAIN:-}
FLUXER_INVITE_DOMAIN: ${FLUXER_INVITE_DOMAIN:-}
FLUXER_GIFT_DOMAIN: ${FLUXER_GIFT_DOMAIN:-}
FLUXER_KV_URL: ${FLUXER_KV_URL:-redis://valkey:6379/0}
FLUXER_KV_MODE: ${FLUXER_KV_MODE:-}
FLUXER_NATS_URL: ${FLUXER_NATS_URL:-nats://nats:4222}
FLUXER_NATS_JETSTREAM_URL: ${FLUXER_NATS_JETSTREAM_URL:-${FLUXER_NATS_URL:-nats://nats:4222}}
FLUXER_NATS_AUTH_TOKEN: ${FLUXER_NATS_AUTH_TOKEN:-}
FLUXER_SVC_NATS_URL: ${FLUXER_SVC_NATS_URL:-${FLUXER_NATS_URL:-nats://nats:4222}}
FLUXER_SVC_SHARD_COUNT: "1"
FLUXER_SVC_MAX_CONCURRENT_REQUESTS: ${FLUXER_SVC_MAX_CONCURRENT_REQUESTS:-}
FLUXER_SVC_CACHE_MAX_ENTRIES: ${FLUXER_SVC_CACHE_MAX_ENTRIES:-}
FLUXER_SVC_CACHE_TTL_MS: ${FLUXER_SVC_CACHE_TTL_MS:-}
FLUXER_GIF_SERVICE_NATS_CLIENT_NAME: ${FLUXER_GIF_SERVICE_NATS_CLIENT_NAME:-}
FLUXER_GIF_SERVICE_TIMEOUT_MS: ${FLUXER_GIF_SERVICE_TIMEOUT_MS:-}
FLUXER_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS: ${FLUXER_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS:-}
FLUXER_USERS_SERVICE_NATS_CLIENT_NAME: ${FLUXER_USERS_SERVICE_NATS_CLIENT_NAME:-}
FLUXER_USERS_SERVICE_TIMEOUT_MS: ${FLUXER_USERS_SERVICE_TIMEOUT_MS:-}
FLUXER_USERS_SERVICE_INFLIGHT_MAX_ENTRIES: ${FLUXER_USERS_SERVICE_INFLIGHT_MAX_ENTRIES:-}
FLUXER_SNOWFLAKE_SERVICE_NATS_CLIENT_NAME: ${FLUXER_SNOWFLAKE_SERVICE_NATS_CLIENT_NAME:-}
FLUXER_SNOWFLAKE_SERVICE_BATCH_SIZE: ${FLUXER_SNOWFLAKE_SERVICE_BATCH_SIZE:-}
FLUXER_SNOWFLAKE_SERVICE_LOW_WATERMARK: ${FLUXER_SNOWFLAKE_SERVICE_LOW_WATERMARK:-}
FLUXER_SNOWFLAKE_SERVICE_MAX_BUFFER_AGE_MS: ${FLUXER_SNOWFLAKE_SERVICE_MAX_BUFFER_AGE_MS:-}
FLUXER_SNOWFLAKE_SERVICE_REQUEST_TIMEOUT_MS: ${FLUXER_SNOWFLAKE_SERVICE_REQUEST_TIMEOUT_MS:-}
FLUXER_SEARCH_ENGINE: meilisearch
FLUXER_SEARCH_ENGINE: ${FLUXER_SEARCH_ENGINE:-meilisearch}
FLUXER_SEARCH_URL: ${FLUXER_SEARCH_URL:-http://meilisearch:7700}
FLUXER_SEARCH_API_KEY: ${MEILI_MASTER_KEY:?set MEILI_MASTER_KEY in .env}
FLUXER_SEARCH_USERNAME: ${FLUXER_SEARCH_USERNAME:-}
FLUXER_SEARCH_PASSWORD: ${FLUXER_SEARCH_PASSWORD:-}
FLUXER_SEARCH_TLS_REJECT_UNAUTHORIZED: ${FLUXER_SEARCH_TLS_REJECT_UNAUTHORIZED:-}
FLUXER_S3_ENDPOINT: ${FLUXER_S3_ENDPOINT:-http://seaweedfs:8333}
FLUXER_S3_PUBLIC_ENDPOINT: ${FLUXER_S3_PUBLIC_ENDPOINT:-${FLUXER_S3_ENDPOINT:-http://seaweedfs:8333}}
@@ -47,45 +90,96 @@ x-fluxer-env: &fluxer-env
FLUXER_S3_BUCKET_UPLOADS: ${FLUXER_S3_BUCKET_UPLOADS:-fluxer-uploads}
FLUXER_S3_BUCKET_REPORTS: ${FLUXER_S3_BUCKET_REPORTS:-fluxer-reports}
FLUXER_S3_BUCKET_HARVESTS: ${FLUXER_S3_BUCKET_HARVESTS:-fluxer-harvests}
AWS_ACCESS_KEY_ID: ${FLUXER_S3_ACCESS_KEY:?set FLUXER_S3_ACCESS_KEY in .env}
AWS_SECRET_ACCESS_KEY: ${FLUXER_S3_SECRET_KEY:?set FLUXER_S3_SECRET_KEY in .env}
AWS_DEFAULT_REGION: ${FLUXER_S3_REGION:-us-east-1}
AWS_EC2_METADATA_DISABLED: "true"
FLUXER_API_PRESIGNED_HARVEST_DOWNLOADS_ENABLED: "${FLUXER_API_PRESIGNED_HARVEST_DOWNLOADS_ENABLED:-false}"
FLUXER_API_STORAGE_CHANGE_FEED_ENABLED: ${FLUXER_API_STORAGE_CHANGE_FEED_ENABLED:-}
FLUXER_API_STORAGE_CHANGE_FEED_STREAM: ${FLUXER_API_STORAGE_CHANGE_FEED_STREAM:-}
FLUXER_API_STORAGE_CHANGE_FEED_SKIP_BUCKETS: ${FLUXER_API_STORAGE_CHANGE_FEED_SKIP_BUCKETS:-}
FLUXER_CACHE_PURGE_ADAPTER: ${FLUXER_CACHE_PURGE_ADAPTER:-}
FLUXER_CACHE_PURGE_HTTP_ENDPOINT: ${FLUXER_CACHE_PURGE_HTTP_ENDPOINT:-}
FLUXER_CACHE_PURGE_HTTP_TOKEN: ${FLUXER_CACHE_PURGE_HTTP_TOKEN:-}
FLUXER_CACHE_PURGE_HTTP_TIMEOUT_MS: ${FLUXER_CACHE_PURGE_HTTP_TIMEOUT_MS:-}
FLUXER_LIVEKIT_ENABLED: "${FLUXER_LIVEKIT_ENABLED:-true}"
FLUXER_LIVEKIT_API_KEY: ${LIVEKIT_API_KEY:?set LIVEKIT_API_KEY in .env}
FLUXER_LIVEKIT_API_SECRET: ${LIVEKIT_API_SECRET:?set LIVEKIT_API_SECRET in .env}
FLUXER_LIVEKIT_INTERNAL_URL: ${FLUXER_LIVEKIT_INTERNAL_URL:-http://livekit:7880}
FLUXER_LIVEKIT_WEBHOOK_URL: http://api:8080/webhooks/livekit
FLUXER_LIVEKIT_DEFAULT_REGION: '{"id":"default","name":"Default","emoji":"🌍","latitude":0,"longitude":0}'
FLUXER_LIVEKIT_DEFAULT_REGION: '${FLUXER_LIVEKIT_DEFAULT_REGION:-{"id":"default","name":"Default","emoji":"🌍","latitude":0,"longitude":0}}'
FLUXER_LIVEKIT_URL: ${FLUXER_LIVEKIT_URL:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}:${FLUXER_PUBLIC_PORT:-443}}/livekit}
FLUXER_KLIPY_API_KEY: ${FLUXER_KLIPY_API_KEY:-}
FLUXER_YOUTUBE_API_KEY: ${FLUXER_YOUTUBE_API_KEY:-}
FLUXER_API_UNFURL_IGNORED_HOSTS: ${FLUXER_API_UNFURL_IGNORED_HOSTS:-}
FLUXER_EMAIL_ENABLED: ${FLUXER_EMAIL_ENABLED:-false}
FLUXER_EMAIL_PROVIDER: ${FLUXER_EMAIL_PROVIDER:-none}
FLUXER_EMAIL_ENABLED: ${FLUXER_EMAIL_ENABLED:-}
FLUXER_EMAIL_PROVIDER: ${FLUXER_EMAIL_PROVIDER:-}
FLUXER_EMAIL_FROM_EMAIL: ${FLUXER_EMAIL_FROM_EMAIL:-noreply@localhost}
FLUXER_EMAIL_FROM_NAME: ${FLUXER_EMAIL_FROM_NAME:-Fluxer}
FLUXER_EMAIL_FROM_NAME: ${FLUXER_EMAIL_FROM_NAME:-}
FLUXER_EMAIL_APP_BASE_URL: ${FLUXER_EMAIL_APP_BASE_URL:-}
FLUXER_EMAIL_WEBHOOK_SECRET: ${FLUXER_EMAIL_WEBHOOK_SECRET:-}
FLUXER_EMAIL_SMTP_HOST: ${FLUXER_EMAIL_SMTP_HOST:-}
FLUXER_EMAIL_SMTP_PORT: ${FLUXER_EMAIL_SMTP_PORT:-587}
FLUXER_EMAIL_SMTP_PORT: ${FLUXER_EMAIL_SMTP_PORT:-}
FLUXER_EMAIL_SMTP_USERNAME: ${FLUXER_EMAIL_SMTP_USERNAME:-}
FLUXER_EMAIL_SMTP_PASSWORD: ${FLUXER_EMAIL_SMTP_PASSWORD:-}
FLUXER_EMAIL_SMTP_SECURE: ${FLUXER_EMAIL_SMTP_SECURE:-true}
FLUXER_EMAIL_SMTP_SECURE: ${FLUXER_EMAIL_SMTP_SECURE:-}
FLUXER_STRIPE_ENABLED: "${FLUXER_STRIPE_ENABLED:-false}"
FLUXER_NCMEC_ENABLED: "${FLUXER_NCMEC_ENABLED:-false}"
FLUXER_CLAMAV_ENABLED: "${FLUXER_CLAMAV_ENABLED:-false}"
FLUXER_DISCOVERY_ENABLED: ${FLUXER_DISCOVERY_ENABLED:-true}
FLUXER_STRIPE_ENABLED: ${FLUXER_STRIPE_ENABLED:-}
FLUXER_STRIPE_SECRET_KEY: ${FLUXER_STRIPE_SECRET_KEY:-}
FLUXER_STRIPE_WEBHOOK_SECRET: ${FLUXER_STRIPE_WEBHOOK_SECRET:-}
FLUXER_STRIPE_PRICES: ${FLUXER_STRIPE_PRICES:-}
FLUXER_STRIPE_LEGACY_PRICES: ${FLUXER_STRIPE_LEGACY_PRICES:-}
FLUXER_API_DONATION_PROXY_KEY: ${FLUXER_API_DONATION_PROXY_KEY:-}
FLUXER_VISIONARIES_GUILD_ID: ${FLUXER_VISIONARIES_GUILD_ID:-}
FLUXER_VISIONARIES_GUILD_VISIONARY_ROLE_ID: ${FLUXER_VISIONARIES_GUILD_VISIONARY_ROLE_ID:-}
FLUXER_NCMEC_ENABLED: ${FLUXER_NCMEC_ENABLED:-}
FLUXER_NCMEC_BASE_URL: ${FLUXER_NCMEC_BASE_URL:-}
FLUXER_NCMEC_USERNAME: ${FLUXER_NCMEC_USERNAME:-}
FLUXER_NCMEC_PASSWORD: ${FLUXER_NCMEC_PASSWORD:-}
FLUXER_NCMEC_REPORTER_EMAIL: ${FLUXER_NCMEC_REPORTER_EMAIL:-}
FLUXER_CLAMAV_ENABLED: ${FLUXER_CLAMAV_ENABLED:-}
FLUXER_CLAMAV_HOST: ${FLUXER_CLAMAV_HOST:-}
FLUXER_CLAMAV_PORT: ${FLUXER_CLAMAV_PORT:-}
FLUXER_CLAMAV_FAIL_OPEN: ${FLUXER_CLAMAV_FAIL_OPEN:-}
FLUXER_APP_PRODUCT_NAME: ${FLUXER_APP_PRODUCT_NAME:-}
FLUXER_APP_ICON_URL: ${FLUXER_APP_ICON_URL:-}
FLUXER_APP_SYMBOL_URL: ${FLUXER_APP_SYMBOL_URL:-}
FLUXER_APP_LOGO_URL: ${FLUXER_APP_LOGO_URL:-}
FLUXER_APP_WORDMARK_URL: ${FLUXER_APP_WORDMARK_URL:-}
FLUXER_APP_FAVICON_URL: ${FLUXER_APP_FAVICON_URL:-}
FLUXER_APP_THEME_COLOR: ${FLUXER_APP_THEME_COLOR:-}
FLUXER_APP_STATUS_PAGE_URL: ${FLUXER_APP_STATUS_PAGE_URL:-}
FLUXER_APP_STATUS_PAGE_INCIDENT_HISTORY_URL: ${FLUXER_APP_STATUS_PAGE_INCIDENT_HISTORY_URL:-}
FLUXER_INSTANCE_SETUP_CONFIGURED: ${FLUXER_INSTANCE_SETUP_CONFIGURED:-}
FLUXER_AUTO_JOIN_INVITE_CODE: ${FLUXER_AUTO_JOIN_INVITE_CODE:-}
FLUXER_DISCOVERY_ENABLED: ${FLUXER_DISCOVERY_ENABLED:-}
FLUXER_DISCOVERY_MIN_MEMBER_COUNT: ${FLUXER_DISCOVERY_MIN_MEMBER_COUNT:-}
FLUXER_DELETION_GRACE_PERIOD_HOURS: ${FLUXER_DELETION_GRACE_PERIOD_HOURS:-}
FLUXER_API_WORKER_LANE_CONCURRENCY_OVERRIDES: ${FLUXER_API_WORKER_LANE_CONCURRENCY_OVERRIDES:-}
FLUXER_AUTH_BLUESKY_ENABLED: ${FLUXER_AUTH_BLUESKY_ENABLED:-}
FLUXER_AUTH_BLUESKY_CLIENT_NAME: ${FLUXER_AUTH_BLUESKY_CLIENT_NAME:-}
FLUXER_AUTH_BLUESKY_CLIENT_URI: ${FLUXER_AUTH_BLUESKY_CLIENT_URI:-}
FLUXER_AUTH_BLUESKY_LOGO_URI: ${FLUXER_AUTH_BLUESKY_LOGO_URI:-}
FLUXER_AUTH_BLUESKY_TOS_URI: ${FLUXER_AUTH_BLUESKY_TOS_URI:-}
FLUXER_AUTH_BLUESKY_POLICY_URI: ${FLUXER_AUTH_BLUESKY_POLICY_URI:-}
FLUXER_AUTH_BLUESKY_KEYS: ${FLUXER_AUTH_BLUESKY_KEYS:-}
FLUXER_PUSH_APNS_ENABLED: ${FLUXER_PUSH_APNS_ENABLED:-}
FLUXER_PUSH_APNS_TEAM_ID: ${FLUXER_PUSH_APNS_TEAM_ID:-}
FLUXER_PUSH_APNS_KEY_ID: ${FLUXER_PUSH_APNS_KEY_ID:-}
FLUXER_PUSH_APNS_PRIVATE_KEY: ${FLUXER_PUSH_APNS_PRIVATE_KEY:-}
FLUXER_PUSH_APNS_PRIVATE_KEY_PATH: ${FLUXER_PUSH_APNS_PRIVATE_KEY_PATH:-}
FLUXER_PUSH_APNS_APPS: ${FLUXER_PUSH_APNS_APPS:-}
FLUXER_SUDO_MODE_SECRET: ${FLUXER_SUDO_MODE_SECRET:?set FLUXER_SUDO_MODE_SECRET in .env}
FLUXER_CONNECTION_INITIATION_SECRET: ${FLUXER_CONNECTION_INITIATION_SECRET:?set FLUXER_CONNECTION_INITIATION_SECRET in .env}
FLUXER_SSO_ALLOW_PRIVATE_ADDRESSES: ${FLUXER_SSO_ALLOW_PRIVATE_ADDRESSES:-false}
FLUXER_SSO_ALLOW_PRIVATE_ADDRESSES: ${FLUXER_SSO_ALLOW_PRIVATE_ADDRESSES:-}
FLUXER_VAPID_PUBLIC_KEY: ${FLUXER_VAPID_PUBLIC_KEY:?set FLUXER_VAPID_PUBLIC_KEY in .env}
FLUXER_VAPID_PRIVATE_KEY: ${FLUXER_VAPID_PRIVATE_KEY:?set FLUXER_VAPID_PRIVATE_KEY in .env}
FLUXER_VAPID_EMAIL: ${FLUXER_VAPID_EMAIL:-admin@${FLUXER_DOMAIN}}
FLUXER_PASSKEY_RP_ID: ${FLUXER_PASSKEY_RP_ID:-${FLUXER_DOMAIN}}
FLUXER_PASSKEY_RP_NAME: ${FLUXER_PASSKEY_RP_NAME:-Fluxer}
FLUXER_PASSKEY_RP_NAME: ${FLUXER_PASSKEY_RP_NAME:-}
FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS: ${FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}}
FLUXER_GATEWAY_RPC_AUTH_TOKEN: ${FLUXER_GATEWAY_RPC_AUTH_TOKEN:?set FLUXER_GATEWAY_RPC_AUTH_TOKEN in .env}
FLUXER_MEDIA_PROXY_SECRET_KEY: ${FLUXER_MEDIA_PROXY_SECRET_KEY:?set FLUXER_MEDIA_PROXY_SECRET_KEY in .env}
@@ -95,34 +189,36 @@ x-fluxer-env: &fluxer-env
FLUXER_ADMIN_OAUTH_CLIENT_SECRET: ${FLUXER_ADMIN_OAUTH_CLIENT_SECRET:?set FLUXER_ADMIN_OAUTH_CLIENT_SECRET in .env}
FLUXER_INTERNAL_API_ENDPOINT: http://api:8080
FLUXER_INTERNAL_GATEWAY_ENDPOINT: http://gateway:8080
FLUXER_INTERNAL_MEDIA_PROXY_ENDPOINT: http://media-proxy:8080
FLUXER_MARKETING_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}
FLUXER_MEDIA_PROXY_ENDPOINT: http://media-proxy:8080
FLUXER_MEDIA_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media}
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_MAX_BODY_BYTES: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_MAX_BODY_BYTES:-}
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_TOKEN_TTL_SECS: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_TOKEN_TTL_SECS:-}
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_KEEP_DIRECT_COUNTRIES: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_KEEP_DIRECT_COUNTRIES:-}
x-fluxer-service: &fluxer-service
restart: unless-stopped
restart: ${FLUXER_RESTART_POLICY:-unless-stopped}
networks: [fluxer]
x-fluxer-svc-healthcheck: &fluxer-svc-healthcheck
test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/8090 && printf 'GET /_health HTTP/1.0\\r\\n\\r\\n' >&3 && head -n 1 <&3 | grep -q ' 200 '"]
interval: 10s
timeout: 5s
retries: 30
start_period: 60s
x-fluxer-app-healthcheck: &fluxer-app-healthcheck
interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s}
timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s}
retries: ${FLUXER_APP_HEALTHCHECK_RETRIES:-30}
start_period: ${FLUXER_APP_HEALTHCHECK_START_PERIOD:-90s}
start_interval: 1s
x-fluxer-svc-healthcheck: &fluxer-svc-healthcheck
<<: *fluxer-app-healthcheck
start_period: ${FLUXER_SVC_HEALTHCHECK_START_PERIOD:-60s}
test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/8090 && printf 'GET /_health HTTP/1.0\\r\\n\\r\\n' >&3 && head -n 1 <&3 | grep -q ' 200 '"]
services:
edge:
image: caddy:2.11-alpine
<<: *fluxer-service
image: ${FLUXER_CADDY_IMAGE:-caddy:2.11-alpine}
deploy:
resources:
limits:
memory: ${FLUXER_CADDY_MEMORY_LIMIT:-256mb}
restart: unless-stopped
networks: [fluxer]
ports:
- "${FLUXER_HTTP_PORT:-80}:80"
- "${FLUXER_HTTPS_PORT:-443}:443"
@@ -130,15 +226,17 @@ services:
environment:
FLUXER_EDGE_SITE_ADDRESS: ${FLUXER_EDGE_SITE_ADDRESS:-${FLUXER_CADDY_SITE_ADDRESS:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN:?set FLUXER_DOMAIN in .env}}}
FLUXER_EDGE_TRUSTED_PROXIES: ${FLUXER_EDGE_TRUSTED_PROXIES:-private_ranges}
FLUXER_EDGE_ENCODE: ${FLUXER_EDGE_ENCODE:-zstd gzip}
FLUXER_ADMIN_BASE_PATH: ${FLUXER_ADMIN_BASE_PATH:-/admin}
volumes:
- ./Caddyfile:/etc/caddy/Caddyfile:ro
- edge-data:/data
- edge-config:/config
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:2019/config/"]
interval: 10s
timeout: 5s
retries: 10
interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s}
timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s}
retries: ${FLUXER_HEALTHCHECK_RETRIES:-10}
depends_on:
api: {condition: service_started}
gateway: {condition: service_healthy}
@@ -147,15 +245,14 @@ services:
admin: {condition: service_started}
postgres:
image: postgres:16-alpine
<<: *fluxer-service
image: ${FLUXER_POSTGRES_IMAGE:-postgres:16-alpine}
deploy:
resources:
limits:
memory: ${FLUXER_POSTGRES_MEMORY_LIMIT:-5gb}
reservations:
memory: ${FLUXER_POSTGRES_MEMORY_RESERVATION:-3gb}
restart: unless-stopped
networks: [fluxer]
command: >
postgres
-c max_connections=${FLUXER_POSTGRES_SERVER_MAX_CONNECTIONS:-150}
@@ -164,82 +261,79 @@ services:
-c work_mem=${FLUXER_POSTGRES_WORK_MEM:-8MB}
-c maintenance_work_mem=${FLUXER_POSTGRES_MAINTENANCE_WORK_MEM:-256MB}
-c autovacuum_work_mem=${FLUXER_POSTGRES_AUTOVACUUM_WORK_MEM:-128MB}
-c random_page_cost=1.1
-c effective_io_concurrency=200
-c default_statistics_target=200
-c jit=off
-c min_wal_size=512MB
-c max_wal_size=2GB
-c checkpoint_completion_target=0.9
-c wal_buffers=16MB
-c wal_compression=zstd
-c bgwriter_delay=50ms
-c bgwriter_lru_maxpages=1000
-c autovacuum_vacuum_scale_factor=0.05
-c autovacuum_analyze_scale_factor=0.02
-c autovacuum_vacuum_cost_limit=2000
-c track_io_timing=on
-c shared_preload_libraries=pg_stat_statements
-c random_page_cost=${FLUXER_POSTGRES_RANDOM_PAGE_COST:-1.1}
-c effective_io_concurrency=${FLUXER_POSTGRES_EFFECTIVE_IO_CONCURRENCY:-200}
-c default_statistics_target=${FLUXER_POSTGRES_DEFAULT_STATISTICS_TARGET:-200}
-c jit=${FLUXER_POSTGRES_JIT:-off}
-c min_wal_size=${FLUXER_POSTGRES_MIN_WAL_SIZE:-512MB}
-c max_wal_size=${FLUXER_POSTGRES_MAX_WAL_SIZE:-2GB}
-c checkpoint_completion_target=${FLUXER_POSTGRES_CHECKPOINT_COMPLETION_TARGET:-0.9}
-c wal_buffers=${FLUXER_POSTGRES_WAL_BUFFERS:-16MB}
-c wal_compression=${FLUXER_POSTGRES_WAL_COMPRESSION:-zstd}
-c bgwriter_delay=${FLUXER_POSTGRES_BGWRITER_DELAY:-50ms}
-c bgwriter_lru_maxpages=${FLUXER_POSTGRES_BGWRITER_LRU_MAXPAGES:-1000}
-c autovacuum_vacuum_scale_factor=${FLUXER_POSTGRES_AUTOVACUUM_VACUUM_SCALE_FACTOR:-0.05}
-c autovacuum_analyze_scale_factor=${FLUXER_POSTGRES_AUTOVACUUM_ANALYZE_SCALE_FACTOR:-0.02}
-c autovacuum_vacuum_cost_limit=${FLUXER_POSTGRES_AUTOVACUUM_VACUUM_COST_LIMIT:-2000}
-c track_io_timing=${FLUXER_POSTGRES_TRACK_IO_TIMING:-on}
-c shared_preload_libraries=${FLUXER_POSTGRES_SHARED_PRELOAD_LIBRARIES:-pg_stat_statements}
shm_size: ${FLUXER_POSTGRES_SHM_SIZE:-1gb}
environment:
POSTGRES_DB: fluxer
POSTGRES_USER: fluxer
POSTGRES_DB: ${FLUXER_POSTGRES_DATABASE:-fluxer}
POSTGRES_USER: ${FLUXER_POSTGRES_USERNAME:-fluxer}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
volumes:
- postgres-data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U fluxer -d fluxer"]
interval: 10s
timeout: 5s
retries: 10
test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""]
interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s}
timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s}
retries: ${FLUXER_HEALTHCHECK_RETRIES:-10}
valkey:
image: valkey/valkey:9.1-alpine
<<: *fluxer-service
image: ${FLUXER_VALKEY_IMAGE:-valkey/valkey:9.1-alpine}
deploy:
resources:
limits:
memory: ${FLUXER_VALKEY_MEMORY_LIMIT:-256mb}
restart: unless-stopped
networks: [fluxer]
command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec", "--dir", "/data",
command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "${FLUXER_VALKEY_APPENDFSYNC:-everysec}", "--dir", "/data",
"--maxmemory", "${FLUXER_VALKEY_MAXMEMORY:-192mb}",
"--maxmemory-policy", "${FLUXER_VALKEY_MAXMEMORY_POLICY:-noeviction}"]
volumes:
- valkey-data:/data
healthcheck:
test: ["CMD", "valkey-cli", "ping"]
interval: 10s
timeout: 5s
retries: 10
interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s}
timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s}
retries: ${FLUXER_HEALTHCHECK_RETRIES:-10}
nats:
image: nats:2.14-alpine
<<: *fluxer-service
image: ${FLUXER_NATS_IMAGE:-nats:2.14-alpine}
deploy:
resources:
limits:
memory: ${FLUXER_NATS_MEMORY_LIMIT:-256mb}
restart: unless-stopped
networks: [fluxer]
command: ["-js", "-sd", "/data", "-m", "8222"]
volumes:
- nats-data:/data
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8222/healthz"]
interval: 10s
timeout: 5s
retries: 10
interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s}
timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s}
retries: ${FLUXER_HEALTHCHECK_RETRIES:-10}
meilisearch:
image: getmeili/meilisearch:v1.53
<<: *fluxer-service
image: ${FLUXER_MEILISEARCH_IMAGE:-getmeili/meilisearch:v1.53}
deploy:
resources:
limits:
memory: ${FLUXER_MEILISEARCH_MEMORY_LIMIT:-768mb}
restart: unless-stopped
networks: [fluxer]
environment:
MEILI_ENV: production
MEILI_NO_ANALYTICS: "true"
MEILI_ENV: ${FLUXER_MEILISEARCH_ENV:-production}
MEILI_NO_ANALYTICS: "${FLUXER_MEILISEARCH_NO_ANALYTICS:-true}"
MEILI_UPGRADE_DB: "true"
MEILI_MAX_INDEXING_MEMORY: ${FLUXER_MEILISEARCH_MAX_INDEXING_MEMORY:-384mb}
MEILI_MASTER_KEY: ${MEILI_MASTER_KEY:?set MEILI_MASTER_KEY in .env}
@@ -247,32 +341,31 @@ services:
- meilisearch-data:/meili_data
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:7700/health"]
interval: 10s
timeout: 5s
retries: 10
interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s}
timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s}
retries: ${FLUXER_HEALTHCHECK_RETRIES:-10}
seaweedfs:
image: chrislusf/seaweedfs:4.47
<<: *fluxer-service
image: ${FLUXER_SEAWEEDFS_IMAGE:-chrislusf/seaweedfs:4.47}
deploy:
resources:
limits:
memory: ${FLUXER_SEAWEEDFS_MEMORY_LIMIT:-2gb}
restart: unless-stopped
networks: [fluxer]
environment:
GOMEMLIMIT: ${FLUXER_SEAWEEDFS_GOMEMLIMIT:-1536MiB}
command: ["server", "-s3", "-dir=/data", "-master.telemetry=false"]
command: ["server", "-s3", "-dir=/data", "-master.telemetry=${FLUXER_SEAWEEDFS_TELEMETRY:-false}"]
volumes:
- seaweedfs-data:/data
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8333/healthz"]
interval: 10s
timeout: 5s
retries: 20
start_period: 60s
interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s}
timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s}
retries: ${FLUXER_SEAWEEDFS_HEALTHCHECK_RETRIES:-20}
start_period: ${FLUXER_SEAWEEDFS_HEALTHCHECK_START_PERIOD:-60s}
seaweedfs-init:
image: chrislusf/seaweedfs:4.47
image: ${FLUXER_SEAWEEDFS_IMAGE:-chrislusf/seaweedfs:4.47}
deploy:
resources:
limits:
@@ -288,13 +381,14 @@ services:
FLUXER_S3_BUCKET_UPLOADS: ${FLUXER_S3_BUCKET_UPLOADS:-fluxer-uploads}
FLUXER_S3_BUCKET_REPORTS: ${FLUXER_S3_BUCKET_REPORTS:-fluxer-reports}
FLUXER_S3_BUCKET_HARVESTS: ${FLUXER_S3_BUCKET_HARVESTS:-fluxer-harvests}
FLUXER_SEAWEEDFS_INIT_ATTEMPTS: ${FLUXER_SEAWEEDFS_INIT_ATTEMPTS:-60}
entrypoint:
- /bin/sh
- -c
- >
buckets="$$FLUXER_S3_BUCKET_CDN $$FLUXER_S3_BUCKET_UPLOADS $$FLUXER_S3_BUCKET_REPORTS $$FLUXER_S3_BUCKET_HARVESTS";
missing="$$buckets";
for attempt in $$(seq 1 60); do
for attempt in $$(seq 1 $$FLUXER_SEAWEEDFS_INIT_ATTEMPTS); do
if ! nc -z seaweedfs 9333 2>/dev/null; then
sleep 2;
continue;
@@ -321,18 +415,17 @@ services:
exit 1;
livekit:
image: livekit/livekit-server:v1.12.0
<<: *fluxer-service
image: ${FLUXER_LIVEKIT_IMAGE:-livekit/livekit-server:v1.12.0}
deploy:
resources:
limits:
memory: ${FLUXER_LIVEKIT_MEMORY_LIMIT:-512mb}
restart: unless-stopped
networks: [fluxer]
environment:
LIVEKIT_KEYS: "${LIVEKIT_API_KEY:?set LIVEKIT_API_KEY in .env}: ${LIVEKIT_API_SECRET:?set LIVEKIT_API_SECRET in .env}"
LIVEKIT_CONFIG: |
port: 7880
log_level: info
log_level: ${FLUXER_LIVEKIT_LOG_LEVEL:-info}
rtc:
tcp_port: ${FLUXER_LIVEKIT_TCP_PORT:-7881}
udp_port: ${FLUXER_LIVEKIT_UDP_PORT:-7882}
@@ -350,9 +443,9 @@ services:
- "${FLUXER_LIVEKIT_UDP_PORT:-7882}:${FLUXER_LIVEKIT_UDP_PORT:-7882}/udp"
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:7880/"]
interval: 10s
timeout: 5s
retries: 10
interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s}
timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s}
retries: ${FLUXER_HEALTHCHECK_RETRIES:-10}
api:
<<: *fluxer-service
@@ -366,16 +459,13 @@ services:
environment:
<<: *fluxer-env
FLUXER_API_PORT: "8080"
NODE_OPTIONS: --enable-source-maps${FLUXER_API_NODE_HEAP_MB:+ --max-old-space-size=$FLUXER_API_NODE_HEAP_MB}
FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED: "true"
FLUXER_POSTGRES_MAX_CONNECTIONS: "25"
NODE_OPTIONS: --enable-source-maps${FLUXER_API_NODE_HEAP_MB:+ --max-old-space-size=$FLUXER_API_NODE_HEAP_MB}${FLUXER_API_NODE_OPTIONS:+ $FLUXER_API_NODE_OPTIONS}
NODE_EXTRA_CA_CERTS: ${FLUXER_NODE_EXTRA_CA_CERTS:-/etc/ssl/certs/ca-certificates.crt}
FLUXER_POSTGRES_MAX_CONNECTIONS: "${FLUXER_API_POSTGRES_MAX_CONNECTIONS:-25}"
FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED: "${FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED:-true}"
healthcheck:
test: ["CMD-SHELL", "node -e \"fetch('http://127.0.0.1:8080/_health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))\""]
interval: 10s
timeout: 5s
retries: 30
start_period: 90s
start_interval: 1s
<<: *fluxer-app-healthcheck
depends_on:
postgres: {condition: service_healthy}
valkey: {condition: service_healthy}
@@ -400,21 +490,18 @@ services:
memory: ${FLUXER_WORKER_MEMORY_LIMIT:-2560mb}
reservations:
memory: ${FLUXER_WORKER_MEMORY_RESERVATION:-1gb}
working_dir: /usr/src/app/fluxer_api
command: ["sh", "-c", "if [ -f dist/WorkerEntrypoint.js ]; then exec node dist/WorkerEntrypoint.js; else exec ./node_modules/.bin/tsx src/WorkerEntrypoint.ts; fi"]
command: ["node", "dist/WorkerEntrypoint.js"]
environment:
<<: *fluxer-env
NODE_OPTIONS: --enable-source-maps${FLUXER_WORKER_NODE_HEAP_MB:+ --max-old-space-size=$FLUXER_WORKER_NODE_HEAP_MB}
NODE_OPTIONS: --enable-source-maps${FLUXER_WORKER_NODE_HEAP_MB:+ --max-old-space-size=$FLUXER_WORKER_NODE_HEAP_MB}${FLUXER_WORKER_NODE_OPTIONS:+ $FLUXER_WORKER_NODE_OPTIONS}
NODE_EXTRA_CA_CERTS: ${FLUXER_NODE_EXTRA_CA_CERTS:-/etc/ssl/certs/ca-certificates.crt}
FLUXER_API_WORKER_MODE: all_lanes
FLUXER_API_WORKER_ENABLE_CRON_SCHEDULER: "true"
FLUXER_POSTGRES_MAX_CONNECTIONS: "25"
FLUXER_POSTGRES_MAX_CONNECTIONS: "${FLUXER_WORKER_POSTGRES_MAX_CONNECTIONS:-25}"
healthcheck:
test: ["CMD", "node", "-e", "const age=Date.now()-require('node:fs').statSync('/tmp/fluxer-worker-heartbeat').mtimeMs;if(age>30000){console.error('worker heartbeat is '+Math.round(age)+'ms old');process.exit(1)}"]
interval: 10s
timeout: 5s
retries: 3
start_period: 90s
start_interval: 1s
<<: *fluxer-app-healthcheck
retries: ${FLUXER_WORKER_HEALTHCHECK_RETRIES:-3}
depends_on:
postgres: {condition: service_healthy}
valkey: {condition: service_healthy}
@@ -436,18 +523,30 @@ services:
environment:
<<: *fluxer-env
FLUXER_GATEWAY_PORT: "8080"
FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media
FLUXER_GATEWAY_STATIC_CDN_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}
FLUXER_GATEWAY_LOGGER_LEVEL: info
FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: ${FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT:-${FLUXER_MEDIA_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media}}
FLUXER_GATEWAY_STATIC_CDN_ENDPOINT: ${FLUXER_GATEWAY_STATIC_CDN_ENDPOINT:-${FLUXER_STATIC_CDN_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}}}
FLUXER_GATEWAY_LOGGER_LEVEL: ${FLUXER_GATEWAY_LOGGER_LEVEL:-}
LOGGER_LEVEL: ${LOGGER_LEVEL:-}
FLUXER_GATEWAY_PUSH_ENABLED: ${FLUXER_GATEWAY_PUSH_ENABLED:-}
FLUXER_GATEWAY_PUSH_ENROLLED_CLEAR_NOTIFICATIONS_ENABLED: ${FLUXER_GATEWAY_PUSH_ENROLLED_CLEAR_NOTIFICATIONS_ENABLED:-}
FLUXER_GATEWAY_PUSH_OUTBOX_REQUEST_TIMEOUT_MS: ${FLUXER_GATEWAY_PUSH_OUTBOX_REQUEST_TIMEOUT_MS:-}
FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_ENTRIES: ${FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_ENTRIES:-}
FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_BYTES: ${FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_BYTES:-}
FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY: ${FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY:-}
FLUXER_GATEWAY_NATS_RPC_MAX_HANDLERS: ${FLUXER_GATEWAY_NATS_RPC_MAX_HANDLERS:-}
FLUXER_GATEWAY_HTTP_FAILURE_THRESHOLD: ${FLUXER_GATEWAY_HTTP_FAILURE_THRESHOLD:-}
FLUXER_GATEWAY_HTTP_RECOVERY_TIMEOUT_MS: ${FLUXER_GATEWAY_HTTP_RECOVERY_TIMEOUT_MS:-}
FLUXER_ERLANG_COOKIE: ${FLUXER_ERLANG_COOKIE:?set FLUXER_ERLANG_COOKIE in .env}
FLUXER_ERLANG_SCHEDULERS_MIN: "${FLUXER_ERLANG_SCHEDULERS_MIN:-2}"
FLUXER_ERLANG_SCHEDULERS_MAX: "${FLUXER_ERLANG_SCHEDULERS_MAX:-16}"
FLUXER_ERLANG_SCHEDULERS: ${FLUXER_ERLANG_SCHEDULERS:-}
FLUXER_ERLANG_SCHEDULERS_MIN: ${FLUXER_ERLANG_SCHEDULERS_MIN:-}
FLUXER_ERLANG_SCHEDULERS_MAX: ${FLUXER_ERLANG_SCHEDULERS_MAX:-}
FLUXER_ERLANG_DIRTY_CPU_SCHEDULERS: ${FLUXER_ERLANG_DIRTY_CPU_SCHEDULERS:-}
healthcheck:
test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:8080/_health/ready"]
interval: 10s
timeout: 5s
retries: 30
start_period: 90s
interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s}
timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s}
retries: ${FLUXER_APP_HEALTHCHECK_RETRIES:-30}
start_period: ${FLUXER_APP_HEALTHCHECK_START_PERIOD:-90s}
depends_on:
nats: {condition: service_healthy}
valkey: {condition: service_healthy}
@@ -461,15 +560,36 @@ services:
memory: ${FLUXER_MEDIA_PROXY_MEMORY_LIMIT:-512mb}
environment:
<<: *fluxer-env
FLUXER_MEDIA_PROXY_HOST: 0.0.0.0
FLUXER_MEDIA_PROXY_PORT: "8080"
FLUXER_MEDIA_PROXY_MODE: upload
FLUXER_MEDIA_PROXY_STORAGE_BACKEND: s3
FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media
FLUXER_MEDIA_PROXY_CORS_MODE: ${FLUXER_MEDIA_PROXY_CORS_MODE:-off}
FLUXER_MEDIA_PROXY_CORS_MODE: ${FLUXER_MEDIA_PROXY_CORS_MODE:-}
FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS: ${FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}:${FLUXER_PUBLIC_PORT:-443}}}
FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE: ${FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE:-off}
FLUXER_S3_READ_SIGNED: "true"
FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE: ${FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE:-}
FLUXER_MEDIA_PROXY_READ_ONLY: ${FLUXER_MEDIA_PROXY_READ_ONLY:-}
FLUXER_MEDIA_PROXY_NSFW_THRESHOLD: ${FLUXER_MEDIA_PROXY_NSFW_THRESHOLD:-}
FLUXER_NSFW_SERVICE_ENDPOINT: ${FLUXER_NSFW_SERVICE_ENDPOINT:-}
FLUXER_MEDIA_PROXY_MAX_NATIVE_TRANSFORMS: ${FLUXER_MEDIA_PROXY_MAX_NATIVE_TRANSFORMS:-}
FLUXER_MEDIA_PROXY_WORKER_QUEUE_CAPACITY: ${FLUXER_MEDIA_PROXY_WORKER_QUEUE_CAPACITY:-}
FLUXER_MEDIA_PROXY_MAX_ENCODE_DURATION_MS: ${FLUXER_MEDIA_PROXY_MAX_ENCODE_DURATION_MS:-}
FLUXER_MEDIA_PROXY_MAX_ENCODE_FRAMES: ${FLUXER_MEDIA_PROXY_MAX_ENCODE_FRAMES:-}
FLUXER_MEDIA_PROXY_TRANSFORM_TIMEOUT_MS: ${FLUXER_MEDIA_PROXY_TRANSFORM_TIMEOUT_MS:-}
FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_BYTES: ${FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_BYTES:-}
FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_MAX_ENTRY_BYTES: ${FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_MAX_ENTRY_BYTES:-}
FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_TTL_MS: ${FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_TTL_MS:-}
FLUXER_MEDIA_PROXY_SOCKET_IO_TIMEOUT_MS: ${FLUXER_MEDIA_PROXY_SOCKET_IO_TIMEOUT_MS:-}
FLUXER_MEDIA_PROXY_SHUTDOWN_GRACE_MS: ${FLUXER_MEDIA_PROXY_SHUTDOWN_GRACE_MS:-}
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_S3_TIMEOUT_MS: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_S3_TIMEOUT_MS:-}
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_BYTES: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_BYTES:-}
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_TOTAL_BYTES: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_TOTAL_BYTES:-}
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_DIR: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_DIR:-}
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_CHUNK_BYTES: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_CHUNK_BYTES:-}
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_MAX_TOTAL_BYTES: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_MAX_TOTAL_BYTES:-}
FLUXER_S3_SESSION_TOKEN: ${FLUXER_S3_SESSION_TOKEN:-}
FLUXER_S3_READ_ENDPOINT: ${FLUXER_S3_READ_ENDPOINT:-}
FLUXER_S3_READ_BUCKET: ${FLUXER_S3_READ_BUCKET:-}
FLUXER_S3_READ_BUCKET_STYLE: ${FLUXER_S3_READ_BUCKET_STYLE:-}
FLUXER_S3_READ_SIGNED: "${FLUXER_S3_READ_SIGNED:-true}"
depends_on:
seaweedfs-init: {condition: service_completed_successfully}
nats: {condition: service_healthy}
@@ -483,17 +603,26 @@ services:
memory: ${FLUXER_PUSH_MEMORY_LIMIT:-256mb}
environment:
<<: *fluxer-env
FLUXER_PUSH_SERVICE_HOST: 0.0.0.0
FLUXER_PUSH_SERVICE_PORT: "8126"
FLUXER_PUSH_SERVICE_QUEUE_CAPACITY: "${FLUXER_PUSH_SERVICE_QUEUE_CAPACITY:-}"
FLUXER_PUSH_SERVICE_SEND_CONCURRENCY: "${FLUXER_PUSH_SERVICE_SEND_CONCURRENCY:-}"
FLUXER_PUSH_SERVICE_QUEUE_CAPACITY: ${FLUXER_PUSH_SERVICE_QUEUE_CAPACITY:-}
FLUXER_PUSH_SERVICE_SEND_CONCURRENCY: ${FLUXER_PUSH_SERVICE_SEND_CONCURRENCY:-}
FLUXER_PUSH_SERVICE_APNS_BASE_URL: ${FLUXER_PUSH_SERVICE_APNS_BASE_URL:-}
FLUXER_PUSH_SERVICE_FCM_BASE_URL: ${FLUXER_PUSH_SERVICE_FCM_BASE_URL:-}
FLUXER_PUSH_SERVICE_MANAGED_RELAY_HOSTS: ${FLUXER_PUSH_SERVICE_MANAGED_RELAY_HOSTS:-}
FLUXER_PUSH_SERVICE_OWN_RELAY_HOSTS: ${FLUXER_PUSH_SERVICE_OWN_RELAY_HOSTS:-}
FLUXER_PUSH_SERVICE_RELAY_CONSENT_ACCEPTED: ${FLUXER_PUSH_SERVICE_RELAY_CONSENT_ACCEPTED:-}
FLUXER_PUSH_APNS_DEFAULT_ENVIRONMENT: ${FLUXER_PUSH_APNS_DEFAULT_ENVIRONMENT:-}
FLUXER_PUSH_FCM_ENABLED: ${FLUXER_PUSH_FCM_ENABLED:-}
FLUXER_PUSH_FCM_PROJECT_ID: ${FLUXER_PUSH_FCM_PROJECT_ID:-}
FLUXER_PUSH_FCM_CLIENT_EMAIL: ${FLUXER_PUSH_FCM_CLIENT_EMAIL:-}
FLUXER_PUSH_FCM_PRIVATE_KEY: ${FLUXER_PUSH_FCM_PRIVATE_KEY:-}
FLUXER_PUSH_FCM_PRIVATE_KEY_PATH: ${FLUXER_PUSH_FCM_PRIVATE_KEY_PATH:-}
FLUXER_PUSH_FCM_SERVICE_ACCOUNT_JSON_PATH: ${FLUXER_PUSH_FCM_SERVICE_ACCOUNT_JSON_PATH:-}
FLUXER_PUSH_FCM_TOKEN_URI: ${FLUXER_PUSH_FCM_TOKEN_URI:-}
FLUXER_PUSH_FCM_APPS: ${FLUXER_PUSH_FCM_APPS:-}
healthcheck:
test: ["CMD", "/usr/local/bin/fluxer-push", "healthcheck"]
interval: 10s
timeout: 5s
retries: 30
start_period: 60s
start_interval: 1s
<<: *fluxer-app-healthcheck
start_period: ${FLUXER_SVC_HEALTHCHECK_START_PERIOD:-60s}
depends_on:
nats: {condition: service_healthy}
api: {condition: service_healthy}
@@ -507,9 +636,9 @@ services:
memory: ${FLUXER_STATIC_PROXY_MEMORY_LIMIT:-256mb}
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/avatars/0.png"]
interval: 10s
timeout: 5s
retries: 10
interval: ${FLUXER_HEALTHCHECK_INTERVAL:-10s}
timeout: ${FLUXER_HEALTHCHECK_TIMEOUT:-5s}
retries: ${FLUXER_HEALTHCHECK_RETRIES:-10}
app-proxy:
<<: *fluxer-service
@@ -519,15 +648,29 @@ services:
limits:
memory: ${FLUXER_APP_PROXY_MEMORY_LIMIT:-256mb}
environment:
FLUXER_APP_PROXY_HOST: 0.0.0.0
RUST_LOG: ${RUST_LOG:-}
FLUXER_APP_PROXY_PORT: "8080"
FLUXER_BASE_DOMAIN: ${FLUXER_DOMAIN:?set FLUXER_DOMAIN in .env}
FLUXER_PUBLIC_SCHEME: ${FLUXER_PUBLIC_SCHEME:-https}
FLUXER_PUBLIC_PORT: ${FLUXER_PUBLIC_PORT:-443}
FLUXER_PUBLIC_ORIGIN: ${FLUXER_PUBLIC_ORIGIN:-}
FLUXER_TRUST_CLIENT_IP_HEADER: "${FLUXER_TRUST_CLIENT_IP_HEADER:-true}"
FLUXER_CLIENT_IP_HEADER_NAME: ${FLUXER_CLIENT_IP_HEADER_NAME:-}
FLUXER_GEOIP_DB_PATH: ${FLUXER_GEOIP_DB_PATH:-}
FLUXER_S3_ENDPOINT: ${FLUXER_S3_ENDPOINT:-http://seaweedfs:8333}
FLUXER_S3_PUBLIC_ENDPOINT: ${FLUXER_S3_PUBLIC_ENDPOINT:-}
FLUXER_S3_REGION: ${FLUXER_S3_REGION:-us-east-1}
FLUXER_S3_ACCESS_KEY_ID: ${FLUXER_S3_ACCESS_KEY:?set FLUXER_S3_ACCESS_KEY in .env}
FLUXER_S3_SECRET_ACCESS_KEY: ${FLUXER_S3_SECRET_KEY:?set FLUXER_S3_SECRET_KEY in .env}
FLUXER_S3_BUCKET_UPLOADS: ${FLUXER_S3_BUCKET_UPLOADS:-}
FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_STATIC_CDN_ENDPOINT:-}
DISCOVERY_UPSTREAM_URL: http://edge:8088/.well-known/fluxer
DISCOVERY_REFRESH_INTERVAL_MS: ${DISCOVERY_REFRESH_INTERVAL_MS:-}
PUBLIC_BOOTSTRAP_API_ENDPOINT: /api
PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/api
PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT: ${PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/api}
FLUXER_APP_PROXY_INDEX_UPSTREAM_URL: ${FLUXER_APP_PROXY_INDEX_UPSTREAM_URL:-}
FLUXER_APP_PROXY_SAME_ORIGIN_HOSTS: ${FLUXER_APP_PROXY_SAME_ORIGIN_HOSTS:-}
FLUXER_APP_PROXY_MANIFEST_SCOPE_EXTENSIONS: ${FLUXER_APP_PROXY_MANIFEST_SCOPE_EXTENSIONS:-}
FLUXER_CSP_EXTRA_DEFAULT_SRC: ${FLUXER_CSP_EXTRA_DEFAULT_SRC:-}
FLUXER_CSP_EXTRA_CONNECT_SRC: ${FLUXER_CSP_EXTRA_CONNECT_SRC:-}
FLUXER_CSP_EXTRA_IMG_SRC: ${FLUXER_CSP_EXTRA_IMG_SRC:-}
@@ -585,7 +728,6 @@ services:
<<: *fluxer-env
FLUXER_SVC_NAME: users
FLUXER_SVC_MODE: router
FLUXER_SVC_MAX_CONCURRENT_REQUESTS: "${FLUXER_SVC_MAX_CONCURRENT_REQUESTS:-}"
healthcheck: *fluxer-svc-healthcheck
depends_on:
nats: {condition: service_healthy}
@@ -602,8 +744,7 @@ services:
FLUXER_SVC_NAME: users
FLUXER_SVC_MODE: shard
FLUXER_SVC_SHARD_ID: "0"
FLUXER_POSTGRES_MAX_CONNECTIONS: "20"
FLUXER_SVC_MAX_CONCURRENT_REQUESTS: "${FLUXER_SVC_MAX_CONCURRENT_REQUESTS:-}"
FLUXER_POSTGRES_MAX_CONNECTIONS: "${FLUXER_USERS_SHARD_POSTGRES_MAX_CONNECTIONS:-20}"
healthcheck: *fluxer-svc-healthcheck
depends_on:
nats: {condition: service_healthy}
@@ -620,7 +761,6 @@ services:
<<: *fluxer-env
FLUXER_SVC_NAME: gifs
FLUXER_SVC_MODE: router
FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media
healthcheck: *fluxer-svc-healthcheck
depends_on:
nats: {condition: service_healthy}
@@ -637,7 +777,7 @@ services:
FLUXER_SVC_NAME: gifs
FLUXER_SVC_MODE: shard
FLUXER_SVC_SHARD_ID: "0"
FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media
FLUXER_GIFS_SHARD_CACHE_MAX_BYTES: ${FLUXER_GIFS_SHARD_CACHE_MAX_BYTES:-}
healthcheck: *fluxer-svc-healthcheck
depends_on:
nats: {condition: service_healthy}
@@ -653,7 +793,6 @@ services:
<<: *fluxer-env
FLUXER_SVC_NAME: messages
FLUXER_SVC_MODE: router
FLUXER_SVC_MAX_CONCURRENT_REQUESTS: "${FLUXER_SVC_MAX_CONCURRENT_REQUESTS:-}"
healthcheck: *fluxer-svc-healthcheck
depends_on:
nats: {condition: service_healthy}
@@ -670,8 +809,7 @@ services:
FLUXER_SVC_NAME: messages
FLUXER_SVC_MODE: shard
FLUXER_SVC_SHARD_ID: "0"
FLUXER_POSTGRES_MAX_CONNECTIONS: "20"
FLUXER_SVC_MAX_CONCURRENT_REQUESTS: "${FLUXER_SVC_MAX_CONCURRENT_REQUESTS:-}"
FLUXER_POSTGRES_MAX_CONNECTIONS: "${FLUXER_MESSAGES_SHARD_POSTGRES_MAX_CONNECTIONS:-20}"
healthcheck: *fluxer-svc-healthcheck
depends_on:
nats: {condition: service_healthy}
@@ -688,8 +826,6 @@ services:
<<: *fluxer-env
FLUXER_SVC_NAME: unfurl
FLUXER_SVC_MODE: router
FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media
FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}
healthcheck: *fluxer-svc-healthcheck
depends_on:
nats: {condition: service_healthy}
@@ -706,8 +842,9 @@ services:
FLUXER_SVC_NAME: unfurl
FLUXER_SVC_MODE: shard
FLUXER_SVC_SHARD_ID: "0"
FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media
FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}
FLUXER_MEDIA_PROXY_ENDPOINT: http://media-proxy:8080
FLUXER_UNFURL_STATIC_CDN_ENDPOINT: ${FLUXER_UNFURL_STATIC_CDN_ENDPOINT:-}
FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_STATIC_CDN_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}}
healthcheck: *fluxer-svc-healthcheck
depends_on:
nats: {condition: service_healthy}
@@ -721,22 +858,14 @@ services:
memory: ${FLUXER_ADMIN_MEMORY_LIMIT:-256mb}
environment:
<<: *fluxer-env
FLUXER_ADMIN_HOST: 0.0.0.0
FLUXER_ADMIN_PORT: "8080"
FLUXER_ADMIN_BASE_PATH: /admin
FLUXER_ADMIN_BASE_PATH: ${FLUXER_ADMIN_BASE_PATH:-/admin}
FLUXER_API_ENDPOINT: http://api:8080
FLUXER_ADMIN_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/admin
FLUXER_APP_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}
FLUXER_MEDIA_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media
FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}
FLUXER_ADMIN_OAUTH_REDIRECT_URI: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/admin/oauth2_callback
FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_STATIC_CDN_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}}
healthcheck:
test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/8080 && printf 'GET /_health HTTP/1.0\\r\\n\\r\\n' >&3 && head -n 1 <&3 | grep -q ' 200 '"]
interval: 10s
timeout: 5s
retries: 30
start_period: 60s
start_interval: 1s
<<: *fluxer-app-healthcheck
start_period: ${FLUXER_SVC_HEALTHCHECK_START_PERIOD:-60s}
depends_on:
api: {condition: service_healthy}
fluxer_admin/Cargo.toml
+1 -1
View File
@@ -26,7 +26,7 @@ tokio = { version = "1.53.1", features = ["macros", "net", "rt-multi-thread", "s
tower = { version = "0.5.3", features = ["util"] }
tower-http = { version = "0.7.1", features = ["compression-gzip", "trace"] }
tracing = "0.1.44"
tracing-subscriber = { version = "0.3.23", features = ["env-filter"] }
tracing-subscriber = "0.3.23"
url = "2.5"
urlencoding = "2.1.3"
progenitor-client = { version = "0.15.0", default-features = false }
fluxer_admin/Dockerfile
-3
View File
@@ -2,7 +2,6 @@
FROM rust:1-trixie AS builder
ARG BUILD_VERSION=""
ARG TARGETARCH
WORKDIR /usr/src/app
@@ -45,8 +44,6 @@ RUN printf '%s\n' \
'strip = "symbols"' \
> Cargo.toml
ENV FLUXER_BUILD_VERSION="${BUILD_VERSION}"
RUN cargo build --release -p fluxer_admin \
&& cp target/release/fluxer_admin /usr/local/bin/fluxer-admin
fluxer_admin/src/config/mod.rs
+14 -68
View File
@@ -1,7 +1,9 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use fluxer_common::config::normalize_public_endpoint_from_env;
use std::env;
use fluxer_common::config::{
normalize_base_path, normalize_public_endpoint_from_env, read_bool_env, read_env,
read_first_env, trim_trailing_slash,
};
const DEFAULT_ADMIN_OAUTH_CLIENT_ID: &str = "1234567890123456789";
@@ -17,12 +19,10 @@ pub struct AdminConfig {
pub static_cdn_endpoint: String,
pub admin_endpoint: String,
pub web_app_endpoint: String,
pub kv_url: String,
pub oauth_client_id: String,
pub oauth_client_secret: String,
pub oauth_redirect_uri: String,
pub build_version: String,
pub release_channel: String,
pub self_hosted: bool,
pub proxy: ProxyConfig,
}
@@ -47,8 +47,8 @@ impl AdminConfig {
"FLUXER_ADMIN_ENDPOINT",
"https://admin.fluxer.app",
)));
let oauth_redirect_uri = normalize_public_endpoint_from_env(&read_env_preferred(
&["FLUXER_ADMIN_OAUTH_REDIRECT_URI"],
let oauth_redirect_uri = normalize_public_endpoint_from_env(&read_env(
"FLUXER_ADMIN_OAUTH_REDIRECT_URI",
&format!("{admin_endpoint}/oauth2_callback"),
));
let secret_key_base = read_env("FLUXER_ADMIN_SECRET_KEY_BASE", "");
@@ -82,38 +82,22 @@ impl AdminConfig {
"FLUXER_APP_ENDPOINT",
"https://app.fluxer.app",
))),
kv_url: read_env("FLUXER_KV_URL", ""),
oauth_client_id: read_env(
"FLUXER_ADMIN_OAUTH_CLIENT_ID",
DEFAULT_ADMIN_OAUTH_CLIENT_ID,
),
oauth_client_secret: read_env("FLUXER_ADMIN_OAUTH_CLIENT_SECRET", ""),
oauth_redirect_uri,
build_version: read_env_preferred(
build_version: read_first_env(
&["BUILD_VERSION", "FLUXER_BUILD_VERSION"],
env!("CARGO_PKG_VERSION"),
),
release_channel: read_env_preferred(
&["RELEASE_CHANNEL", "FLUXER_RELEASE_CHANNEL"],
"stable",
),
self_hosted: read_bool_env(&["FLUXER_SELF_HOSTED"], false),
self_hosted: read_bool_env("FLUXER_SELF_HOSTED", false),
proxy: ProxyConfig {
trust_client_ip_header: read_bool_env(
&["FLUXER_TRUST_CLIENT_IP_HEADER", "TRUST_CLIENT_IP_HEADER"],
false,
),
client_ip_header_name: read_env_preferred(
&[
"FLUXER_CLIENT_IP_HEADER_NAME",
"FLUXER_CLIENT_IP_HEADER",
"CLIENT_IP_HEADER_NAME",
"CLIENT_IP_HEADER",
],
"x-forwarded-for",
)
.trim()
.to_ascii_lowercase(),
trust_client_ip_header: read_bool_env("FLUXER_TRUST_CLIENT_IP_HEADER", false),
client_ip_header_name: read_env("FLUXER_CLIENT_IP_HEADER_NAME", "x-forwarded-for")
.trim()
.to_ascii_lowercase(),
},
})
}
@@ -146,55 +130,21 @@ impl RuntimeEnv {
}
}
pub fn normalize_base_path(value: &str) -> String {
let trimmed = value.trim().trim_matches('/');
if trimmed.is_empty() {
String::new()
} else {
format!("/{trimmed}")
}
}
pub fn trim_trailing_slash(value: &str) -> String {
value.trim_end_matches('/').to_owned()
}
pub(crate) fn read_env(name: &str, fallback: &str) -> String {
env::var(name).unwrap_or_else(|_| fallback.to_owned())
}
pub(crate) fn read_env_preferred(names: &[&str], fallback: &str) -> String {
names
.iter()
.find_map(|name| env::var(name).ok().filter(|value| !value.trim().is_empty()))
.unwrap_or_else(|| fallback.to_owned())
}
pub(crate) fn read_bool_env(names: &[&str], fallback: bool) -> bool {
let Some(value) = names.iter().find_map(|name| env::var(name).ok()) else {
return fallback;
};
matches!(
value.trim().to_ascii_lowercase().as_str(),
"1" | "true" | "yes" | "on"
)
}
#[cfg(test)]
mod tests {
use super::*;
use std::env;
use std::sync::Mutex;
static ENV_LOCK: Mutex<()> = Mutex::new(());
const MANAGED_ENV: [&str; 11] = [
const MANAGED_ENV: [&str; 10] = [
"FLUXER_ENV",
"FLUXER_ADMIN_HOST",
"FLUXER_ADMIN_PORT",
"FLUXER_ADMIN_ENDPOINT",
"FLUXER_ADMIN_OAUTH_CLIENT_ID",
"FLUXER_ADMIN_OAUTH_REDIRECT_URI",
"FLUXER_MASTER_CONFIG",
"FLUXER_APP_ENDPOINT",
"FLUXER_MEDIA_ENDPOINT",
"FLUXER_STATIC_CDN_ENDPOINT",
@@ -291,12 +241,10 @@ mod tests {
admin_endpoint: String::new(),
web_app_endpoint: String::new(),
kv_url: String::new(),
oauth_client_id: String::new(),
oauth_client_secret: String::new(),
oauth_redirect_uri: String::new(),
build_version: String::new(),
release_channel: String::new(),
self_hosted: false,
proxy: ProxyConfig {
trust_client_ip_header: false,
@@ -321,12 +269,10 @@ mod tests {
admin_endpoint: String::new(),
web_app_endpoint: String::new(),
kv_url: String::new(),
oauth_client_id: String::new(),
oauth_client_secret: String::new(),
oauth_redirect_uri: String::new(),
build_version: String::new(),
release_channel: String::new(),
self_hosted: false,
proxy: ProxyConfig {
trust_client_ip_header: false,
fluxer_admin/src/main.rs
+1 -3
View File
@@ -8,9 +8,7 @@ use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
#[tokio::main]
async fn main() -> anyhow::Result<()> {
tracing_subscriber::registry()
.with(
tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()),
)
.with(fluxer_common::config::env_filter("info"))
.with(tracing_subscriber::fmt::layer())
.init();
fluxer_admin/src/middleware/csrf.rs
-2
View File
@@ -215,12 +215,10 @@ mod tests {
static_cdn_endpoint: String::new(),
admin_endpoint: admin_endpoint.to_owned(),
web_app_endpoint: String::new(),
kv_url: String::new(),
oauth_client_id: String::new(),
oauth_client_secret: String::new(),
oauth_redirect_uri: String::new(),
build_version: "test".to_owned(),
release_channel: String::new(),
self_hosted: false,
proxy: ProxyConfig {
trust_client_ip_header: false,
fluxer_admin/tests/csrf_legacy_cookie.rs
-2
View File
@@ -95,12 +95,10 @@ fn production_config(api_endpoint: String) -> AdminConfig {
static_cdn_endpoint: "https://static.example.test".to_owned(),
admin_endpoint: ADMIN_ORIGIN.to_owned(),
web_app_endpoint: "https://app.example.test".to_owned(),
kv_url: String::new(),
oauth_client_id: "admin-client".to_owned(),
oauth_client_secret: "admin-secret".to_owned(),
oauth_redirect_uri: "https://admin.example.test/callback".to_owned(),
build_version: "test".to_owned(),
release_channel: "test".to_owned(),
self_hosted: false,
proxy: ProxyConfig {
trust_client_ip_header: false,
fluxer_admin/tests/htmx_acceptance.rs
-2
View File
@@ -1298,12 +1298,10 @@ fn test_config(api_endpoint: String) -> AdminConfig {
static_cdn_endpoint: "https://static.example.test".to_owned(),
admin_endpoint: "https://admin.example.test".to_owned(),
web_app_endpoint: "https://app.example.test".to_owned(),
kv_url: String::new(),
oauth_client_id: "admin-client".to_owned(),
oauth_client_secret: "admin-secret".to_owned(),
oauth_redirect_uri: "https://admin.example.test/callback".to_owned(),
build_version: "test".to_owned(),
release_channel: "test".to_owned(),
self_hosted: false,
proxy: ProxyConfig {
trust_client_ip_header: false,
fluxer_admin/tests/parity/rust_server.rs
-2
View File
@@ -55,12 +55,10 @@ fn admin_config(port: u16, api_endpoint: &str, admin_endpoint: &str) -> AdminCon
static_cdn_endpoint: "https://static.example.test".to_owned(),
admin_endpoint: admin_endpoint.to_owned(),
web_app_endpoint: "http://127.0.0.1:8088".to_owned(),
kv_url: "redis://127.0.0.1:6379/0".to_owned(),
oauth_client_id: "1234567890123456789".to_owned(),
oauth_client_secret: "test-admin-oauth-secret".to_owned(),
oauth_redirect_uri: format!("{admin_endpoint}/oauth2_callback"),
build_version: "parity".to_owned(),
release_channel: "parity".to_owned(),
self_hosted: false,
proxy: ProxyConfig {
trust_client_ip_header: false,
fluxer_admin/tests/voice_restriction_writes.rs
-2
View File
@@ -326,12 +326,10 @@ fn test_config(api_endpoint: String) -> AdminConfig {
static_cdn_endpoint: "https://static.example.test".to_owned(),
admin_endpoint: "https://admin.example.test".to_owned(),
web_app_endpoint: "https://app.example.test".to_owned(),
kv_url: String::new(),
oauth_client_id: "admin-client".to_owned(),
oauth_client_secret: "admin-secret".to_owned(),
oauth_redirect_uri: "https://admin.example.test/callback".to_owned(),
build_version: "test".to_owned(),
release_channel: "test".to_owned(),
self_hosted: false,
proxy: ProxyConfig {
trust_client_ip_header: false,
fluxer_api/pkgs/kv_client/src/KVClient.ts
+2 -6
View File
@@ -302,10 +302,8 @@ export class KVClient implements IKVProvider {
}
private createClusterClient(clusterConfig: ResolvedKVClientConfig): Cluster {
const {nodes, redisOptions} = resolveKVClusterConnection(clusterConfig.url, clusterConfig.clusterNodes);
const natMap = clusterConfig.clusterNatMap;
const hasNatMap = Object.keys(natMap).length > 0;
return new Cluster(nodes, {
const {node, redisOptions} = resolveKVClusterConnection(clusterConfig.url);
return new Cluster([node], {
clusterRetryStrategy: createRetryStrategy(),
redisOptions: {
...redisOptions,
@@ -315,7 +313,6 @@ export class KVClient implements IKVProvider {
protocol: 2,
},
scaleReads: 'master',
...(hasNatMap ? {natMap} : {}),
});
}
@@ -591,7 +588,6 @@ export class KVClient implements IKVProvider {
return new KVSubscription({
url: this.url,
mode: this.config.mode,
clusterNodes: this.config.clusterNodes,
timeoutMs: this.timeoutMs,
logger: this.logger,
});
fluxer_api/pkgs/kv_client/src/KVClientConfig.ts
-11
View File
@@ -9,16 +9,9 @@ export interface IKVLogger {
export type KVClientMode = 'standalone' | 'cluster';
export interface KVClusterNode {
host: string;
port: number;
}
export interface KVClientConfig {
url: string;
mode?: KVClientMode;
clusterNodes?: Array<KVClusterNode>;
clusterNatMap?: Record<string, KVClusterNode>;
timeoutMs?: number;
logger?: IKVLogger;
}
@@ -26,8 +19,6 @@ export interface KVClientConfig {
export interface ResolvedKVClientConfig {
url: string;
mode: KVClientMode;
clusterNodes: Array<KVClusterNode>;
clusterNatMap: Record<string, KVClusterNode>;
timeoutMs: number;
logger: IKVLogger;
}
@@ -42,8 +33,6 @@ export function resolveKVClientConfig(config: KVClientConfig | string): Resolved
return {
url: normalizeUrl(options.url),
mode: options.mode ?? 'standalone',
clusterNodes: options.clusterNodes ?? [],
clusterNatMap: options.clusterNatMap ?? {},
timeoutMs: options.timeoutMs ?? DEFAULT_KV_TIMEOUT_MS,
logger: options.logger ?? noopLogger,
};
fluxer_api/pkgs/kv_client/src/KVClusterConnection.ts
+3 -13
View File
@@ -1,13 +1,12 @@
import {domainToASCII} from 'node:url';
import type {KVClusterNode} from '@pkgs/kv_client/src/KVClientConfig';
import type {RedisOptions} from 'ioredis';
interface KVClusterConnection {
nodes: Array<KVClusterNode>;
node: {host: string; port: number};
redisOptions: RedisOptions;
}
export function resolveKVClusterConnection(url: string, nodes: ReadonlyArray<KVClusterNode>): KVClusterConnection {
export function resolveKVClusterConnection(url: string): KVClusterConnection {
const normalizedUrl = url.trim();
for (let index = 0; index < normalizedUrl.length; index++) {
const code = normalizedUrl.charCodeAt(index);
@@ -46,17 +45,8 @@ export function resolveKVClusterConnection(url: string, nodes: ReadonlyArray<KVC
redisOptions.tls = {};
}
const host = resolveClusterHost(authority, parsed);
const resolvedNodes = nodes.length > 0 ? [...nodes] : [{host, port: Number(parsed.port || '6379')}];
for (const node of resolvedNodes) {
if (node.host.trim().length === 0) {
throw new Error('KV cluster node must include a host');
}
if (!Number.isInteger(node.port) || node.port < 1 || node.port > 65535) {
throw new Error('KV cluster node port must be an integer between 1 and 65535');
}
}
return {
nodes: resolvedNodes,
node: {host, port: Number(parsed.port || '6379')},
redisOptions,
};
}
fluxer_api/pkgs/kv_client/src/KVSubscription.ts
+3 -6
View File
@@ -1,14 +1,13 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {IKVSubscription} from '@pkgs/kv_client/src/IKVProvider';
import type {IKVLogger, KVClientMode, KVClusterNode} from '@pkgs/kv_client/src/KVClientConfig';
import type {IKVLogger, KVClientMode} from '@pkgs/kv_client/src/KVClientConfig';
import {resolveKVClusterConnection} from '@pkgs/kv_client/src/KVClusterConnection';
import Redis, {type RedisOptions} from 'ioredis';
interface KVSubscriptionConfig {
url: string;
mode?: KVClientMode;
clusterNodes?: Array<KVClusterNode>;
timeoutMs: number;
logger: IKVLogger;
}
@@ -21,7 +20,6 @@ interface KVSubscriptionConnect {
export class KVSubscription implements IKVSubscription {
private readonly url: string;
private readonly mode: KVClientMode;
private readonly clusterNodes: Array<KVClusterNode>;
private readonly timeoutMs: number;
private readonly logger: IKVLogger;
private readonly desiredChannels = new Set<string>();
@@ -34,7 +32,6 @@ export class KVSubscription implements IKVSubscription {
constructor(config: KVSubscriptionConfig) {
this.url = config.url;
this.mode = config.mode ?? 'standalone';
this.clusterNodes = config.clusterNodes ?? [];
this.timeoutMs = config.timeoutMs;
this.logger = config.logger;
}
@@ -75,9 +72,9 @@ export class KVSubscription implements IKVSubscription {
protocol: 2,
retryStrategy: createRetryStrategy(),
};
const connection = this.mode === 'cluster' ? resolveKVClusterConnection(this.url, this.clusterNodes) : null;
const connection = this.mode === 'cluster' ? resolveKVClusterConnection(this.url) : null;
const client = connection
? new Redis({...connection.redisOptions, ...connection.nodes[0], db: 0, ...options})
? new Redis({...connection.redisOptions, ...connection.node, db: 0, ...options})
: new Redis(this.url, options);
client.on('message', (channel: string, message: string) => {
if (this.client !== client || this.closing !== null) {
fluxer_api/pkgs/mime_utils/package.json
-3
View File
@@ -2,10 +2,7 @@
"name": "@pkgs/mime_utils",
"version": "0.0.0",
"type": "module",
"main": "./src/index.ts",
"types": "./src/index.ts",
"exports": {
".": "./src/index.ts",
"./src/*": "./src/*",
"./*": "./*"
},
fluxer_api/pkgs/mime_utils/src/index.ts
-7
View File
@@ -1,7 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
export {
getContentTypeFromFilename,
isSupportedMediaContentType,
normalizeContentType,
} from '@pkgs/mime_utils/src/ContentTypeUtils';
fluxer_api/pkgs/worker/package.json
+1 -31
View File
@@ -4,10 +4,6 @@
"private": true,
"type": "module",
"exports": {
"./WorkerFactory": {
"import": "./src/runtime/WorkerFactory.ts",
"types": "./src/runtime/WorkerFactory.ts"
},
"./WorkerContext": {
"import": "./src/context/WorkerContext.ts",
"types": "./src/context/WorkerContext.ts"
@@ -24,39 +20,13 @@
"import": "./src/contracts/WorkerTypes.ts",
"types": "./src/contracts/WorkerTypes.ts"
},
"./IQueueProvider": {
"import": "./src/providers/IQueueProvider.ts",
"types": "./src/providers/IQueueProvider.ts"
},
"./HttpWorkerQueue": {
"import": "./src/providers/HttpWorkerQueue.ts",
"types": "./src/providers/HttpWorkerQueue.ts"
},
"./QueueProviderFactory": {
"import": "./src/providers/QueueProviderFactory.ts",
"types": "./src/providers/QueueProviderFactory.ts"
},
"./WorkerRunner": {
"import": "./src/runtime/WorkerRunner.ts",
"types": "./src/runtime/WorkerRunner.ts"
},
"./WorkerService": {
"import": "./src/services/WorkerService.ts",
"types": "./src/services/WorkerService.ts"
},
"./WorkerTaskRegistry": {
"import": "./src/runtime/WorkerTaskRegistry.ts",
"types": "./src/runtime/WorkerTaskRegistry.ts"
},
"./*": "./*"
},
"scripts": {
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@fluxer/constants": "workspace:*",
"@fluxer/logger": "workspace:*",
"itty-time": "catalog:"
"@fluxer/logger": "workspace:*"
},
"devDependencies": {
"@types/node": "catalog:",
fluxer_api/pkgs/worker/src/contracts/WorkerTypes.ts
-50
View File
@@ -2,56 +2,6 @@
export type WorkerJobPayload = Record<string, unknown>;
export interface WorkerRuntimeConfig {
workerId?: string | undefined;
concurrency?: number | undefined;
taskTypes?: Array<string> | undefined;
}
export interface WorkerQueueConfig {
queueBaseUrl: string;
requestTimeoutMs?: number | undefined;
}
export interface WorkerConfig extends WorkerRuntimeConfig, WorkerQueueConfig {}
export interface TracingInterface {
withSpan<T>(
options: {
name: string;
attributes?: Record<string, unknown>;
},
fn: () => Promise<T>,
): Promise<T>;
addSpanEvent(name: string, attributes?: Record<string, unknown>): void;
setSpanAttributes(attributes: Record<string, unknown>): void;
}
export interface QueueJob {
id: string;
task_type: string;
payload: WorkerJobPayload;
priority: number;
run_at: string;
created_at: string;
attempts: number;
max_attempts: number;
error?: string | null;
deduplication_id?: string | null;
}
export interface LeasedQueueJob {
receipt: string;
visibility_deadline: string;
job: QueueJob;
}
export interface EnqueueOptions {
runAt?: Date | undefined;
maxAttempts?: number | undefined;
priority?: number | undefined;
}
export interface WorkerJobOptions {
queueName?: string | undefined;
runAt?: Date | undefined;
fluxer_api/pkgs/worker/src/providers/HttpWorkerQueue.ts
-234
View File
@@ -1,234 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {DEFAULT_HTTP_WORKER_TIMEOUT_MS} from '@fluxer/constants/src/Timeouts';
import type {
EnqueueOptions,
LeasedQueueJob,
TracingInterface,
WorkerJobPayload,
} from '@pkgs/worker/src/contracts/WorkerTypes';
import type {IQueueProvider} from '@pkgs/worker/src/providers/IQueueProvider';
export class HttpWorkerQueue implements IQueueProvider {
private readonly baseUrl: string;
private readonly timeoutMs: number;
private readonly tracing: TracingInterface | undefined;
constructor(options: {
baseUrl: string;
timeoutMs?: number | undefined;
tracing?: TracingInterface | undefined;
}) {
this.baseUrl = options.baseUrl;
this.timeoutMs = options.timeoutMs ?? DEFAULT_HTTP_WORKER_TIMEOUT_MS;
this.tracing = options.tracing;
}
private async withResponse<T>(
input: string | URL,
init: RequestInit,
consume: (response: Response) => Promise<T>,
): Promise<T> {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), this.timeoutMs);
try {
const response = await fetch(input, {
...init,
signal: controller.signal,
});
return await consume(response);
} finally {
clearTimeout(timeoutId);
controller.abort();
}
}
private async requireSuccess(response: Response, action: string): Promise<void> {
if (response.ok) {
return;
}
const text = await response.text();
throw new Error(`Failed to ${action}: ${response.status} ${text}`);
}
private async withOptionalSpan<T>(
options: {
name: string;
attributes?: Record<string, unknown>;
},
fn: () => Promise<T>,
): Promise<T> {
if (this.tracing) {
return this.tracing.withSpan(options, fn);
}
return fn();
}
private addSpanEvent(name: string, attributes?: Record<string, unknown>): void {
if (this.tracing) {
this.tracing.addSpanEvent(name, attributes);
}
}
private setSpanAttributes(attributes: Record<string, unknown>): void {
if (this.tracing) {
this.tracing.setSpanAttributes(attributes);
}
}
async enqueue(taskType: string, payload: WorkerJobPayload, options?: EnqueueOptions): Promise<string> {
return await this.withOptionalSpan(
{
name: 'queue.enqueue',
attributes: {
'queue.task_type': taskType,
'queue.priority': options?.priority ?? 0,
'queue.max_attempts': options?.maxAttempts ?? 5,
'queue.scheduled': options?.runAt !== undefined,
'net.peer.name': new URL(this.baseUrl).hostname,
},
},
async () => {
const body = {
task_type: taskType,
payload,
priority: options?.priority ?? 0,
run_at: options?.runAt?.toISOString(),
max_attempts: options?.maxAttempts ?? 5,
};
return this.withResponse(
`${this.baseUrl}/enqueue`,
{
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify(body),
},
async (response) => {
await this.requireSuccess(response, 'enqueue job');
const jobIdResult = (await response.json()) as {
job_id: string;
};
this.setSpanAttributes({'queue.job_id': jobIdResult.job_id});
return jobIdResult.job_id;
},
);
},
);
}
async dequeue(taskTypes: Array<string>, limit = 1): Promise<Array<LeasedQueueJob>> {
return await this.withOptionalSpan(
{
name: 'queue.dequeue',
attributes: {
'queue.task_types': taskTypes.join(','),
'queue.limit': limit,
'queue.service': 'fluxer-queue',
},
},
async () => {
this.addSpanEvent('dequeue.start');
const url = new URL(`${this.baseUrl}/dequeue`);
url.searchParams.set('task_types', taskTypes.join(','));
url.searchParams.set('limit', limit.toString());
url.searchParams.set('wait_time_ms', '0');
return this.withResponse(url, {method: 'GET'}, async (response) => {
await this.requireSuccess(response, 'dequeue job');
this.addSpanEvent('dequeue.parse_response');
const jobs = (await response.json()) as Array<LeasedQueueJob>;
const jobCount = jobs?.length ?? 0;
this.setSpanAttributes({
'queue.jobs_returned': jobCount,
'queue.empty': jobCount === 0,
});
this.addSpanEvent('dequeue.complete');
return jobs ?? [];
});
},
);
}
async upsertCron(id: string, taskType: string, payload: WorkerJobPayload, cronExpression: string): Promise<void> {
await this.withResponse(
`${this.baseUrl}/cron`,
{
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({id, task_type: taskType, payload, cron_expression: cronExpression}),
},
(response) => this.requireSuccess(response, 'upsert cron job'),
);
}
async complete(receipt: string): Promise<void> {
return await this.withOptionalSpan(
{
name: 'queue.complete',
attributes: {
'queue.receipt': receipt,
},
},
async () =>
this.withResponse(
`${this.baseUrl}/ack`,
{
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({receipt}),
},
(response) => this.requireSuccess(response, 'complete job'),
),
);
}
async fail(receipt: string, error: string): Promise<void> {
return await this.withOptionalSpan(
{
name: 'queue.fail',
attributes: {
'queue.receipt': receipt,
'queue.error_message': error,
},
},
async () =>
this.withResponse(
`${this.baseUrl}/nack`,
{
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({receipt, error}),
},
(response) => this.requireSuccess(response, 'fail job'),
),
);
}
async cancelJob(jobId: string): Promise<boolean> {
return this.withResponse(`${this.baseUrl}/job/${jobId}`, {method: 'DELETE'}, async (response) => {
if (!response.ok) {
const text = await response.text();
if (response.status === 404) {
return false;
}
throw new Error(`Failed to cancel job: ${response.status} ${text}`);
}
const result = (await response.json()) as {
success: boolean;
};
return result.success ?? true;
});
}
async retryDeadLetterJob(jobId: string): Promise<boolean> {
return this.withResponse(`${this.baseUrl}/retry/${jobId}`, {method: 'POST'}, async (response) => {
if (!response.ok) {
const text = await response.text();
if (response.status === 404) {
return false;
}
throw new Error(`Failed to retry job: ${response.status} ${text}`);
}
return true;
});
}
}
fluxer_api/pkgs/worker/src/providers/IQueueProvider.ts
-13
View File
@@ -1,13 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {EnqueueOptions, LeasedQueueJob, WorkerJobPayload} from '@pkgs/worker/src/contracts/WorkerTypes';
export interface IQueueProvider {
enqueue(taskType: string, payload: WorkerJobPayload, options?: EnqueueOptions): Promise<string>;
dequeue(taskTypes: Array<string>, limit?: number): Promise<Array<LeasedQueueJob>>;
upsertCron(id: string, taskType: string, payload: WorkerJobPayload, cronExpression: string): Promise<void>;
complete(receipt: string): Promise<void>;
fail(receipt: string, error: string): Promise<void>;
cancelJob(jobId: string): Promise<boolean>;
retryDeadLetterJob(jobId: string): Promise<boolean>;
}
fluxer_api/pkgs/worker/src/providers/QueueProviderFactory.ts
-26
View File
@@ -1,26 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {TracingInterface} from '@pkgs/worker/src/contracts/WorkerTypes';
import {HttpWorkerQueue} from '@pkgs/worker/src/providers/HttpWorkerQueue';
import type {IQueueProvider} from '@pkgs/worker/src/providers/IQueueProvider';
export interface QueueProviderFactoryOptions {
queueProvider?: IQueueProvider | undefined;
queueBaseUrl?: string | undefined;
timeoutMs?: number | undefined;
tracing?: TracingInterface | undefined;
}
export function createQueueProvider(options: QueueProviderFactoryOptions): IQueueProvider {
if (options.queueProvider) {
return options.queueProvider;
}
if (!options.queueBaseUrl) {
throw new Error('Queue provider requires either queueProvider or queueBaseUrl');
}
return new HttpWorkerQueue({
baseUrl: options.queueBaseUrl,
timeoutMs: options.timeoutMs,
tracing: options.tracing,
});
}
fluxer_api/pkgs/worker/src/runtime/WorkerFactory.ts
-175
View File
@@ -1,175 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {LoggerInterface} from '@fluxer/logger/src/LoggerInterface';
import {setWorkerDependencies} from '@pkgs/worker/src/context/WorkerContext';
import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
import type {
LeasedQueueJob,
TracingInterface,
WorkerConfig,
WorkerQueueConfig,
WorkerRuntimeConfig,
} from '@pkgs/worker/src/contracts/WorkerTypes';
import type {IQueueProvider} from '@pkgs/worker/src/providers/IQueueProvider';
import {WorkerRunner} from '@pkgs/worker/src/runtime/WorkerRunner';
import {WorkerTaskRegistry} from '@pkgs/worker/src/runtime/WorkerTaskRegistry';
import {WorkerService} from '@pkgs/worker/src/services/WorkerService';
export interface CreateWorkerOptions {
queue: WorkerQueueOptions;
runtime?: WorkerRuntimeConfig | undefined;
logger: LoggerInterface;
dependencies?: unknown;
taskRegistry?: WorkerTaskRegistry | undefined;
tracing?: TracingInterface | undefined;
}
export interface CreateWorkerLegacyOptions {
config: WorkerConfig;
queueProvider?: IQueueProvider | undefined;
logger: LoggerInterface;
dependencies?: unknown;
taskRegistry?: WorkerTaskRegistry | undefined;
tracing?: TracingInterface | undefined;
}
export interface WorkerQueueOptions {
queueProvider?: IQueueProvider | undefined;
queueBaseUrl?: string | undefined;
requestTimeoutMs?: number | undefined;
}
interface ResolvedWorkerFactoryOptions {
queue: WorkerQueueOptions;
runtime: WorkerRuntimeConfig;
logger: LoggerInterface;
dependencies?: unknown;
taskRegistry?: WorkerTaskRegistry | undefined;
tracing?: TracingInterface | undefined;
}
export interface WorkerResult {
start: () => Promise<void>;
shutdown: () => Promise<void>;
processTask: (job: LeasedQueueJob) => Promise<void>;
getRunner: () => WorkerRunner;
getWorkerService: () => IWorkerService;
registerTask: <TPayload = Record<string, unknown>>(name: string, handler: WorkerTaskHandler<TPayload>) => void;
registerTasks: (tasks: Record<string, WorkerTaskHandler>) => void;
}
type WorkerFactoryOptions = CreateWorkerOptions | CreateWorkerLegacyOptions;
function isLegacyCreateWorkerOptions(options: WorkerFactoryOptions): options is CreateWorkerLegacyOptions {
return 'config' in options;
}
function resolveLegacyQueueOptions(config: WorkerQueueConfig, queueProvider?: IQueueProvider): WorkerQueueOptions {
return {
queueProvider,
queueBaseUrl: config.queueBaseUrl,
requestTimeoutMs: config.requestTimeoutMs,
};
}
function resolveWorkerFactoryOptions(options: WorkerFactoryOptions): ResolvedWorkerFactoryOptions {
if (isLegacyCreateWorkerOptions(options)) {
return {
queue: resolveLegacyQueueOptions(options.config, options.queueProvider),
runtime: {
workerId: options.config.workerId,
taskTypes: options.config.taskTypes,
concurrency: options.config.concurrency,
},
logger: options.logger,
dependencies: options.dependencies,
taskRegistry: options.taskRegistry,
tracing: options.tracing,
};
}
return {
queue: options.queue,
runtime: options.runtime ?? {},
logger: options.logger,
dependencies: options.dependencies,
taskRegistry: options.taskRegistry,
tracing: options.tracing,
};
}
function assertTaskRegistryMutable(runner: WorkerRunner | null): void {
if (runner?.isRunning()) {
throw new Error('Cannot register tasks after worker start. Register tasks before starting the worker.');
}
}
export function createWorker(options: WorkerFactoryOptions): WorkerResult {
const resolvedOptions = resolveWorkerFactoryOptions(options);
const {queue, runtime, logger, dependencies, taskRegistry: providedRegistry, tracing} = resolvedOptions;
if (dependencies !== undefined) {
setWorkerDependencies(dependencies);
}
const taskRegistry = providedRegistry ?? new WorkerTaskRegistry();
let runner: WorkerRunner | null = null;
let workerService: WorkerService | null = null;
function ensureRunner(): WorkerRunner {
if (!runner) {
runner = new WorkerRunner({
tasks: taskRegistry.getTasks(),
queueBaseUrl: queue.queueBaseUrl,
queueProvider: queue.queueProvider,
logger,
workerId: runtime.workerId,
taskTypes: runtime.taskTypes,
concurrency: runtime.concurrency,
tracing,
requestTimeoutMs: queue.requestTimeoutMs,
});
}
return runner;
}
function ensureWorkerService(): WorkerService {
if (!workerService) {
workerService = new WorkerService({
queueBaseUrl: queue.queueBaseUrl,
queueProvider: queue.queueProvider,
logger,
tracing,
timeoutMs: queue.requestTimeoutMs,
});
}
return workerService;
}
return {
async start() {
const r = ensureRunner();
await r.start();
},
async shutdown() {
if (runner) {
await runner.stop();
}
},
async processTask(job: LeasedQueueJob) {
const r = ensureRunner();
await r.processJob(job);
},
getRunner() {
return ensureRunner();
},
getWorkerService() {
return ensureWorkerService();
},
registerTask<TPayload = Record<string, unknown>>(name: string, handler: WorkerTaskHandler<TPayload>) {
assertTaskRegistryMutable(runner);
taskRegistry.register(name, handler);
runner = null;
},
registerTasks(tasks: Record<string, WorkerTaskHandler>) {
assertTaskRegistryMutable(runner);
taskRegistry.registerAll(tasks);
runner = null;
},
};
}
fluxer_api/pkgs/worker/src/runtime/WorkerRunner.ts
-187
View File
@@ -1,187 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {randomUUID} from 'node:crypto';
import type {LoggerInterface} from '@fluxer/logger/src/LoggerInterface';
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
import type {LeasedQueueJob, TracingInterface} from '@pkgs/worker/src/contracts/WorkerTypes';
import type {IQueueProvider} from '@pkgs/worker/src/providers/IQueueProvider';
import {createQueueProvider} from '@pkgs/worker/src/providers/QueueProviderFactory';
import {WorkerService} from '@pkgs/worker/src/services/WorkerService';
import {ms} from 'itty-time';
export interface WorkerRunnerOptions {
tasks: Record<string, WorkerTaskHandler>;
queueBaseUrl?: string | undefined;
queueProvider?: IQueueProvider | undefined;
logger: LoggerInterface;
workerId?: string | undefined;
taskTypes?: Array<string> | undefined;
concurrency?: number | undefined;
tracing?: TracingInterface | undefined;
requestTimeoutMs?: number | undefined;
}
export class WorkerRunner {
private readonly tasks: Record<string, WorkerTaskHandler>;
private readonly workerId: string;
private readonly taskTypes: Array<string>;
private readonly concurrency: number;
private readonly queue: IQueueProvider;
private readonly workerService: WorkerService;
private readonly logger: LoggerInterface;
private readonly tracing: TracingInterface | undefined;
private running = false;
private abortController: AbortController | null = null;
private workerLoopPromises: Array<Promise<void>> = [];
constructor(options: WorkerRunnerOptions) {
this.tasks = options.tasks;
this.workerId = options.workerId ?? `worker-${randomUUID()}`;
this.taskTypes = options.taskTypes ?? Object.keys(options.tasks);
this.concurrency = options.concurrency ?? 1;
this.logger = options.logger;
this.tracing = options.tracing;
this.queue = createQueueProvider({
queueProvider: options.queueProvider,
queueBaseUrl: options.queueBaseUrl,
timeoutMs: options.requestTimeoutMs,
tracing: options.tracing,
});
this.workerService = new WorkerService({
queueProvider: this.queue,
logger: options.logger,
});
}
async start(): Promise<void> {
if (this.running) {
this.logger.warn({workerId: this.workerId}, 'Worker already running');
return;
}
this.running = true;
this.abortController = new AbortController();
this.logger.info(
{workerId: this.workerId, taskTypes: this.taskTypes, concurrency: this.concurrency},
'Worker starting',
);
this.workerLoopPromises = Array.from({length: this.concurrency}, (_, i) =>
this.workerLoop(i, this.abortController!.signal),
);
Promise.all(this.workerLoopPromises).catch((error) => {
this.logger.error({workerId: this.workerId, error}, 'Worker loop failed unexpectedly');
});
}
async stop(): Promise<void> {
if (!this.running) {
return;
}
this.running = false;
this.abortController?.abort();
const stopTimeout = new Promise<void>((resolve) => setTimeout(resolve, ms('2 seconds')));
await Promise.race([Promise.all(this.workerLoopPromises), stopTimeout]);
this.workerLoopPromises = [];
this.logger.info({workerId: this.workerId}, 'Worker stopped');
}
async processJob(leasedJob: LeasedQueueJob): Promise<void> {
await this.executeJob(leasedJob);
}
getWorkerService(): WorkerService {
return this.workerService;
}
getQueue(): IQueueProvider {
return this.queue;
}
isRunning(): boolean {
return this.running;
}
private async workerLoop(workerIndex: number, signal: AbortSignal): Promise<void> {
this.logger.info({workerId: this.workerId, workerIndex}, 'Worker loop started');
while (!signal.aborted) {
try {
const leasedJobs = await this.queue.dequeue(this.taskTypes, 1);
if (!leasedJobs || leasedJobs.length === 0) {
await this.sleep(100);
continue;
}
const leasedJob = leasedJobs[0]!;
const job = leasedJob.job;
this.logger.info(
{
workerId: this.workerId,
workerIndex,
jobId: job.id,
taskType: job.task_type,
attempts: job.attempts,
receipt: leasedJob.receipt,
},
'Processing job',
);
await this.executeJob(leasedJob);
this.logger.info({workerId: this.workerId, workerIndex, jobId: job.id}, 'Job completed successfully');
} catch (error) {
this.logger.error({workerId: this.workerId, workerIndex, error}, 'Worker loop error');
await this.sleep(ms('1 second'));
}
}
this.logger.info({workerId: this.workerId, workerIndex}, 'Worker loop stopped');
}
private async executeJob(leasedJob: LeasedQueueJob): Promise<void> {
const execute = async () => {
const task = this.tasks[leasedJob.job.task_type];
if (!task) {
throw new Error(`Unknown task: ${leasedJob.job.task_type}`);
}
this.tracing?.addSpanEvent('job.execution.start');
try {
await task(leasedJob.job.payload, {
logger: this.logger.child({jobId: leasedJob.job.id}),
jobId: 0n,
addJob: this.workerService.addJob.bind(this.workerService),
reportProgress: async () => {},
shouldCancel: async () => false,
setContextLink: async () => {},
});
this.tracing?.addSpanEvent('job.execution.success');
this.tracing?.setSpanAttributes({'job.status': 'success'});
await this.queue.complete(leasedJob.receipt);
} catch (error) {
this.logger.error({jobId: leasedJob.job.id, error}, 'Job failed');
this.tracing?.setSpanAttributes({
'job.status': 'failed',
'job.error': error instanceof Error ? error.message : String(error),
});
this.tracing?.addSpanEvent('job.execution.failed', {
error: error instanceof Error ? error.message : String(error),
});
await this.queue.fail(leasedJob.receipt, String(error));
}
};
if (this.tracing) {
await this.tracing.withSpan(
{
name: 'worker.process_job',
attributes: {
'worker.id': this.workerId,
'job.id': leasedJob.job.id,
'job.task_type': leasedJob.job.task_type,
'job.attempts': leasedJob.job.attempts,
},
},
execute,
);
} else {
await execute();
}
}
private async sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
}
fluxer_api/pkgs/worker/src/runtime/WorkerTaskRegistry.ts
-43
View File
@@ -1,43 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
export class WorkerTaskRegistry {
private readonly tasks: Map<string, WorkerTaskHandler> = new Map();
register<TPayload = Record<string, unknown>>(name: string, handler: WorkerTaskHandler<TPayload>): this {
this.tasks.set(name, handler as WorkerTaskHandler);
return this;
}
registerAll(tasks: Record<string, WorkerTaskHandler>): this {
for (const [name, handler] of Object.entries(tasks)) {
this.tasks.set(name, handler);
}
return this;
}
get(name: string): WorkerTaskHandler | undefined {
return this.tasks.get(name);
}
has(name: string): boolean {
return this.tasks.has(name);
}
getTaskNames(): Array<string> {
return Array.from(this.tasks.keys());
}
getTasks(): Record<string, WorkerTaskHandler> {
return Object.fromEntries(this.tasks);
}
get size(): number {
return this.tasks.size;
}
}
export function createTaskRegistry(): WorkerTaskRegistry {
return new WorkerTaskRegistry();
}
fluxer_api/pkgs/worker/src/services/WorkerService.ts
-83
View File
@@ -1,83 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {LoggerInterface} from '@fluxer/logger/src/LoggerInterface';
import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';
import type {TracingInterface, WorkerJobOptions, WorkerJobPayload} from '@pkgs/worker/src/contracts/WorkerTypes';
import type {IQueueProvider} from '@pkgs/worker/src/providers/IQueueProvider';
import {createQueueProvider} from '@pkgs/worker/src/providers/QueueProviderFactory';
export interface WorkerServiceOptions {
queueBaseUrl?: string | undefined;
queueProvider?: IQueueProvider | undefined;
logger: LoggerInterface;
tracing?: TracingInterface | undefined;
timeoutMs?: number | undefined;
}
export class WorkerService implements IWorkerService {
private readonly queue: IQueueProvider;
private readonly logger: LoggerInterface;
constructor(options: WorkerServiceOptions) {
this.queue = createQueueProvider({
queueProvider: options.queueProvider,
queueBaseUrl: options.queueBaseUrl,
timeoutMs: options.timeoutMs,
tracing: options.tracing,
});
this.logger = options.logger;
}
async addJob<TPayload extends WorkerJobPayload = WorkerJobPayload>(
taskType: string,
payload: TPayload,
options?: WorkerJobOptions,
): Promise<bigint> {
try {
await this.queue.enqueue(taskType, payload, {
runAt: options?.runAt,
maxAttempts: options?.maxAttempts,
priority: options?.priority,
});
this.logger.debug({taskType, payload}, 'Job queued successfully');
} catch (error) {
this.logger.error({error, taskType, payload}, 'Failed to queue job');
throw error;
}
return 0n;
}
async cancelJob(jobId: bigint): Promise<boolean> {
try {
const cancelled = await this.queue.cancelJob(jobId.toString());
if (cancelled) {
this.logger.info({jobId: jobId.toString()}, 'Job cancelled successfully');
} else {
this.logger.debug({jobId: jobId.toString()}, 'Job not found (may have already been processed)');
}
return cancelled;
} catch (error) {
this.logger.error({error, jobId: jobId.toString()}, 'Failed to cancel job');
throw error;
}
}
async retryDeadLetterJob(jobId: bigint): Promise<boolean> {
try {
const retried = await this.queue.retryDeadLetterJob(jobId.toString());
if (retried) {
this.logger.info({jobId: jobId.toString()}, 'Dead letter job retried successfully');
} else {
this.logger.debug({jobId: jobId.toString()}, 'Job not found in dead letter queue');
}
return retried;
} catch (error) {
this.logger.error({error, jobId: jobId.toString()}, 'Failed to retry dead letter job');
throw error;
}
}
getQueue(): IQueueProvider {
return this.queue;
}
}
fluxer_api/src/api/Config.ts
+4 -84
View File
@@ -39,28 +39,6 @@ function resolveEmailAppBaseUrl(master: MasterConfig): string {
}
}
function resolveGatewayInternalUrl(master: MasterConfig): string {
const configuredInternalGateway = (
master.internal as {
gateway?: string;
}
).gateway;
if (typeof configuredInternalGateway === 'string' && configuredInternalGateway.length > 0) {
return trimTrailingSlash(configuredInternalGateway);
}
try {
const gatewayUrl = new URL(master.endpoints.gateway);
if (gatewayUrl.protocol === 'ws:') {
gatewayUrl.protocol = 'http:';
} else if (gatewayUrl.protocol === 'wss:') {
gatewayUrl.protocol = 'https:';
}
return trimTrailingSlash(gatewayUrl.toString());
} catch {
throw new Error(`Invalid gateway endpoint URL: ${master.endpoints.gateway}`);
}
}
function isBoolean(value: unknown): value is boolean {
return typeof value === 'boolean';
}
@@ -100,26 +78,23 @@ function normalizeIpBanExemptIps(values: Array<string>): Array<string> {
return Array.from(normalized);
}
function mapPushProviderApps(
function mapApnsApps(
apps:
| Array<{
app_id?: string;
topic?: string;
environment?: 'production' | 'development';
project_id?: string;
}>
| undefined,
configName: string,
): APIConfig['push']['apns']['apps'] {
return (apps ?? []).map((app) => {
if (!app.app_id) {
throw new Error(`${configName} contains an entry with no app_id`);
throw new Error('FLUXER_PUSH_APNS_APPS contains an entry with no app_id');
}
return {
appId: app.app_id,
topic: app.topic,
environment: app.environment,
projectId: app.project_id,
};
});
}
@@ -194,34 +169,8 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
backend: master.database.backend,
},
kv: {
provider: 'redis' as const,
url: master.internal.kv,
mode: ((
master.internal as {
kv_mode?: string;
}
).kv_mode ?? 'standalone') as 'standalone' | 'cluster',
clusterNodes:
(
master.internal as {
kv_cluster_nodes?: Array<{
host: string;
port: number;
}>;
}
).kv_cluster_nodes ?? [],
clusterNatMap:
(
master.internal as {
kv_cluster_nat_map?: Record<
string,
{
host: string;
port: number;
}
>;
}
).kv_cluster_nat_map ?? {},
mode: master.internal.kv_mode,
},
nats: {
coreUrl: master.services.nats?.core_url ?? 'nats://127.0.0.1:4222',
@@ -277,7 +226,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
staticCdn: master.endpoints.static_cdn,
},
internal: {
gateway: resolveGatewayInternalUrl(master),
gatewayRpcAuthToken: master.services.gateway.rpc_auth_token ?? '',
donationProxyKey,
},
@@ -285,15 +233,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
marketing: extractHostname(master.endpoints.marketing),
unfurlIgnored: master.services.api.unfurl_ignored_hosts,
},
embeds: {
oEmbedHtmlEnabled: master.services.api.embeds.oembed_html_enabled,
oEmbedHtmlAllowUntrustedOnSelfHosted: master.services.api.embeds.oembed_html_allow_untrusted_on_self_hosted,
oEmbedHtmlAllowedHosts: master.services.api.embeds.oembed_html_allowed_hosts,
cacheDefaultTtlSeconds: master.services.api.embeds.cache_default_ttl_seconds,
cacheMaxTtlSeconds: master.services.api.embeds.cache_max_ttl_seconds,
cacheMinTtlSeconds: master.services.api.embeds.cache_min_ttl_seconds,
cacheRespectRemoteTtl: master.services.api.embeds.cache_respect_remote_ttl,
},
s3: {
endpoint: s3Config.endpoint,
presignedUrlBase: s3Config.presigned_url_base,
@@ -329,14 +268,10 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
breachedPasswordCheck: {
enabled: master.integrations.breached_password_check.enabled ?? !master.instance.self_hosted,
},
contentModeration: {
nsfwThreshold: master.services.api.content_moderation?.nsfw_threshold ?? 0.7,
},
voice: {
enabled: master.integrations.voice.enabled,
apiKey: master.integrations.voice.api_key,
apiSecret: master.integrations.voice.api_secret,
webhookUrl: master.integrations.voice.webhook_url,
url: master.integrations.voice.url,
internalUrl: master.integrations.voice.internal_url,
defaultRegion: master.integrations.voice.default_region,
@@ -409,7 +344,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
reporterEmail: master.integrations.ncmec.reporter_email ?? '',
},
admin: {
basePath: master.services.admin.base_path,
oauthClientSecret: master.services.admin.oauth_client_secret,
},
auth: {
@@ -454,9 +388,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
configured: master.instance.setup.configured,
},
},
domain: {
baseDomain: master.domain.base_domain,
},
discovery: {
enabled: master.discovery.enabled,
minMemberCount: master.discovery.min_member_count,
@@ -481,18 +412,7 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
keyId: master.integrations.push.apns.key_id,
privateKey: master.integrations.push.apns.private_key,
privateKeyPath: master.integrations.push.apns.private_key_path,
defaultEnvironment: master.integrations.push.apns.default_environment ?? 'production',
apps: mapPushProviderApps(master.integrations.push.apns.apps, 'FLUXER_PUSH_APNS_APPS'),
},
fcm: {
enabled: master.integrations.push.fcm.enabled,
projectId: master.integrations.push.fcm.project_id,
clientEmail: master.integrations.push.fcm.client_email,
privateKey: master.integrations.push.fcm.private_key,
privateKeyPath: master.integrations.push.fcm.private_key_path,
serviceAccountJsonPath: master.integrations.push.fcm.service_account_json_path,
tokenUri: master.integrations.push.fcm.token_uri ?? 'https://oauth2.googleapis.com/token',
apps: mapPushProviderApps(master.integrations.push.fcm.apps, 'FLUXER_PUSH_FCM_APPS'),
apps: mapApnsApps(master.integrations.push.apns.apps),
},
},
worker: {
fluxer_api/src/api/config/APIConfig.ts
-42
View File
@@ -11,7 +11,6 @@ export interface PushProviderAppConfig {
appId: string;
topic?: string;
environment?: PushProviderEnvironment;
projectId?: string;
}
export interface APICachePurgeConfig {
@@ -69,20 +68,8 @@ export interface APIConfig {
backend: 'cassandra' | 'postgres';
};
kv: {
provider: 'redis';
url: string;
mode: 'standalone' | 'cluster';
clusterNodes: Array<{
host: string;
port: number;
}>;
clusterNatMap: Record<
string,
{
host: string;
port: number;
}
>;
};
nats: {
coreUrl: string;
@@ -136,7 +123,6 @@ export interface APIConfig {
gift: string;
};
internal: {
gateway: string;
gatewayRpcAuthToken: string;
donationProxyKey: string;
};
@@ -144,15 +130,6 @@ export interface APIConfig {
marketing: string;
unfurlIgnored: Array<string>;
};
embeds: {
oEmbedHtmlEnabled: boolean;
oEmbedHtmlAllowUntrustedOnSelfHosted: boolean;
oEmbedHtmlAllowedHosts: Array<string>;
cacheDefaultTtlSeconds: number;
cacheMaxTtlSeconds: number;
cacheMinTtlSeconds: number;
cacheRespectRemoteTtl: boolean;
};
s3: {
endpoint: string;
presignedUrlBase: string | undefined;
@@ -191,14 +168,10 @@ export interface APIConfig {
breachedPasswordCheck: {
enabled: boolean;
};
contentModeration: {
nsfwThreshold: number;
};
voice: {
enabled: boolean;
apiKey?: string;
apiSecret?: string;
webhookUrl?: string;
url?: string;
internalUrl?: string;
defaultRegion?: {
@@ -261,7 +234,6 @@ export interface APIConfig {
failOpen: boolean;
};
admin: {
basePath: string;
oauthClientSecret?: string;
};
auth: {
@@ -306,9 +278,6 @@ export interface APIConfig {
configured: boolean;
};
};
domain: {
baseDomain: string;
};
discovery: {
enabled: boolean;
minMemberCount: number;
@@ -333,17 +302,6 @@ export interface APIConfig {
keyId?: string;
privateKey?: string;
privateKeyPath?: string;
defaultEnvironment: PushProviderEnvironment;
apps: Array<PushProviderAppConfig>;
};
fcm: {
enabled: boolean;
projectId?: string;
clientEmail?: string;
privateKey?: string;
privateKeyPath?: string;
serviceAccountJsonPath?: string;
tokenUri: string;
apps: Array<PushProviderAppConfig>;
};
};
fluxer_api/src/api/gif/NatsGifProvider.ts
+3 -3
View File
@@ -3,7 +3,7 @@
import {Config} from '@app/api/Config';
import type {GifProviderMeta, IGifProvider} from '@app/api/gif/IGifProvider';
import {Logger} from '@app/api/Logger';
import {readOptionalIntegerEnv, requireIntegerInRange} from '@app/api/utils/IntegerOptions';
import {readOptionalEnv, readOptionalIntegerEnv, requireIntegerInRange} from '@app/api/utils/IntegerOptions';
import {isJsonRecord, parseJsonUnknown} from '@app/api/utils/JsonBoundaryUtils';
import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';
import {ServiceUnavailableError} from '@fluxer/errors/src/domains/core/ServiceUnavailableError';
@@ -19,7 +19,7 @@ import {NatsConnectionManager} from '@pkgs/nats/src/NatsConnectionManager';
const textEncoder = new TextEncoder();
const textDecoder = new TextDecoder();
const GIF_SERVICE_SUBJECT = process.env.FLUXER_GIF_SERVICE_SUBJECT || 'svc.gifs';
const GIF_SERVICE_SUBJECT = readOptionalEnv('FLUXER_GIF_SERVICE_SUBJECT') ?? 'svc.gifs';
const DEFAULT_GIF_SERVICE_TIMEOUT_MS = 12_000;
const DEFAULT_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS = 3_000;
const MAX_REQUEST_TIMEOUT_MS = 2_147_483_647;
@@ -279,7 +279,7 @@ export function createNatsGifProvider(apiKeyResolver: GifApiKeyResolver): NatsGi
const manager = new NatsConnectionManager({
url: Config.nats.coreUrl,
token: Config.nats.authToken || undefined,
name: process.env.FLUXER_GIF_SERVICE_NATS_CLIENT_NAME || 'fluxer-api-gifs',
name: readOptionalEnv('FLUXER_GIF_SERVICE_NATS_CLIENT_NAME') ?? 'fluxer-api-gifs',
});
const provider = new NatsGifProvider(
manager,
fluxer_api/src/api/infrastructure/UsersServiceClient.ts
+3 -3
View File
@@ -5,7 +5,7 @@ import {Config} from '@app/api/Config';
import {throwForSvcErrorReply} from '@app/api/infrastructure/SvcErrorReply';
import {Logger} from '@app/api/Logger';
import {awaitAll} from '@app/api/utils/ConcurrencyUtils';
import {readOptionalIntegerEnv, requireIntegerInRange} from '@app/api/utils/IntegerOptions';
import {readOptionalEnv, readOptionalIntegerEnv, requireIntegerInRange} from '@app/api/utils/IntegerOptions';
import {isJsonRecord, parseJsonRecord, parseJsonWithGuard} from '@app/api/utils/JsonBoundaryUtils';
import type {UserPartialResponse} from '@fluxer/schema/src/domains/user/UserResponseSchemas';
import type {INatsConnectionManager} from '@pkgs/nats/src/INatsConnectionManager';
@@ -14,7 +14,7 @@ import {NatsConnectionManager} from '@pkgs/nats/src/NatsConnectionManager';
const textEncoder = new TextEncoder();
const textDecoder = new TextDecoder();
const USERS_SERVICE_SUBJECT = process.env.FLUXER_USERS_SERVICE_SUBJECT || 'svc.users';
const USERS_SERVICE_SUBJECT = readOptionalEnv('FLUXER_USERS_SERVICE_SUBJECT') ?? 'svc.users';
const DEFAULT_USERS_SERVICE_TIMEOUT_MS = 6000;
const DEFAULT_USERS_SERVICE_INFLIGHT_MAX_ENTRIES = 10000;
const MAX_REQUEST_TIMEOUT_MS = 2_147_483_647;
@@ -196,7 +196,7 @@ export function createUsersServiceClient(): IUsersServiceClient {
const manager = new NatsConnectionManager({
url: Config.nats.coreUrl,
token: Config.nats.authToken || undefined,
name: process.env.FLUXER_USERS_SERVICE_NATS_CLIENT_NAME || 'fluxer-api-users',
name: readOptionalEnv('FLUXER_USERS_SERVICE_NATS_CLIENT_NAME') ?? 'fluxer-api-users',
});
usersServiceClient = new NatsUsersServiceClient(
manager,
fluxer_api/src/api/middleware/IpBanMiddleware.ts
+2 -1
View File
@@ -7,6 +7,7 @@ import {IP_BAN_REFRESH_CHANNEL} from '@app/api/constants/IpBan';
import {sharedListHas} from '@app/api/infrastructure/activity/SharedLists';
import {Logger} from '@app/api/Logger';
import type {HonoEnv} from '@app/api/types/HonoEnv';
import {readOptionalEnv} from '@app/api/utils/IntegerOptions';
import {parseIpBanEntry, tryParseSingleIp} from '@app/api/utils/IpRangeUtils';
import {RefreshSubscription} from '@app/api/utils/RefreshSubscription';
import {getRequestClientIp} from '@app/api/utils/RequestClientIp';
@@ -70,7 +71,7 @@ class IpBanCache {
channels: [IP_BAN_REFRESH_CHANNEL],
refresh: () => this.refresh(),
periodicIntervalMs: () => {
const intervalMs = Number(process.env.FLUXER_IP_BAN_REFRESH_INTERVAL_MS ?? '300000');
const intervalMs = Number(readOptionalEnv('FLUXER_IP_BAN_REFRESH_INTERVAL_MS') ?? '300000');
return Number.isFinite(intervalMs) && intervalMs > 0 ? intervalMs : null;
},
onRefreshError: (err, trigger) => {
fluxer_api/src/api/middleware/ServiceRegistry.ts
+3 -5
View File
@@ -22,7 +22,7 @@ import type {InstanceConfigRepository} from '@app/api/instance/InstanceConfigRep
import {Logger} from '@app/api/Logger';
import {setInjectedSearchProvider} from '@app/api/SearchFactory';
import type {ISearchProvider} from '@app/api/search/ISearchProvider';
import {readOptionalIntegerEnv} from '@app/api/utils/IntegerOptions';
import {readOptionalEnv, readOptionalIntegerEnv} from '@app/api/utils/IntegerOptions';
import {VoiceAvailabilityService} from '@app/api/voice/VoiceAvailabilityService';
import {VoiceRepository} from '@app/api/voice/VoiceRepository';
import {VoiceServerLoadTracker} from '@app/api/voice/VoiceServerLoad';
@@ -37,11 +37,11 @@ export function createSnowflakeService(): SnowflakeService {
const connectionManager = new NatsConnectionManager({
url: Config.nats.coreUrl,
token: Config.nats.authToken || undefined,
name: process.env.FLUXER_SNOWFLAKE_SERVICE_NATS_CLIENT_NAME || 'fluxer-api-snowflakes',
name: readOptionalEnv('FLUXER_SNOWFLAKE_SERVICE_NATS_CLIENT_NAME') ?? 'fluxer-api-snowflakes',
});
return new SnowflakeService({
connectionManager,
subject: process.env.FLUXER_SNOWFLAKE_SERVICE_SUBJECT || undefined,
subject: readOptionalEnv('FLUXER_SNOWFLAKE_SERVICE_SUBJECT'),
batchSize: readOptionalIntegerEnv('FLUXER_SNOWFLAKE_SERVICE_BATCH_SIZE'),
lowWatermark: readOptionalIntegerEnv('FLUXER_SNOWFLAKE_SERVICE_LOW_WATERMARK'),
maxBufferAgeMs: readOptionalIntegerEnv('FLUXER_SNOWFLAKE_SERVICE_MAX_BUFFER_AGE_MS'),
@@ -64,8 +64,6 @@ export function getKVClient(): IKVProvider {
_kvClient = new KVClient({
url: Config.kv.url,
mode: Config.kv.mode,
clusterNodes: Config.kv.clusterNodes,
clusterNatMap: Config.kv.clusterNatMap,
});
}
return _kvClient;
fluxer_api/src/api/rpc/RpcTimings.ts
+2 -11
View File
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {hostname} from 'node:os';
import {readOptionalEnv} from '@app/api/utils/IntegerOptions';
import type {RpcSessionTimings, RpcTimingNode} from '@fluxer/schema/src/domains/rpc/RpcSchemas';
export type RpcTimingSteps = Record<string, RpcTimingNode>;
@@ -20,18 +21,8 @@ export function createRpcTimingNode(startedAtNs: bigint, steps?: RpcTimingSteps)
};
}
function firstRuntimeName(names: Array<string>, fallback: string): string {
for (const name of names) {
const value = process.env[name];
if (value && value.length > 0) {
return value;
}
}
return fallback;
}
function apiPodName(): string {
return firstRuntimeName(['POD_NAME', 'HOSTNAME'], hostname());
return readOptionalEnv('POD_NAME') ?? readOptionalEnv('HOSTNAME') ?? hostname();
}
export async function timeRpcStep<T>(steps: RpcTimingSteps, name: string, operation: () => Promise<T>): Promise<T> {
fluxer_api/src/api/test/Setup.ts
-4
View File
@@ -53,8 +53,6 @@ function setDefaultTestEnv(): void {
FLUXER_NATS_URL: natsUrl,
FLUXER_NATS_CORE_URL: natsUrl,
FLUXER_NATS_JETSTREAM_URL: natsUrl,
FLUXER_INTERNAL_API_ENDPOINT: 'http://127.0.0.1:8088/api',
FLUXER_INTERNAL_GATEWAY_ENDPOINT: 'http://127.0.0.1:8088/gateway',
FLUXER_INTERNAL_MEDIA_PROXY_ENDPOINT: 'http://127.0.0.1:8088/media',
FLUXER_S3_ENDPOINT: 'http://127.0.0.1:3900',
FLUXER_S3_REGION: 'local',
@@ -65,8 +63,6 @@ function setDefaultTestEnv(): void {
FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64: 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=',
FLUXER_ADMIN_SECRET_KEY_BASE: 'test-admin-secret',
FLUXER_ADMIN_OAUTH_CLIENT_SECRET: 'test-admin-oauth-secret',
FLUXER_APP_PROXY_PORT: '8773',
FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: 'http://127.0.0.1:8088/media',
FLUXER_GATEWAY_RPC_AUTH_TOKEN: 'test-gateway-rpc-token',
FLUXER_SUDO_MODE_SECRET: 'test-sudo-secret',
FLUXER_CONNECTION_INITIATION_SECRET: 'test-connection-secret',
fluxer_api/src/api/utils/IntegerOptions.ts
+8 -2
View File
@@ -1,6 +1,12 @@
import {readEnvValue} from '@fluxer/config/src/config_loader/EnvironmentOverrides';
export function readOptionalEnv(name: string): string | undefined {
return readEnvValue(process.env, name);
}
export function readOptionalIntegerEnv(name: string): number | undefined {
const raw = process.env[name]?.trim();
if (raw === undefined || raw === '') return undefined;
const raw = readOptionalEnv(name)?.trim();
if (raw === undefined) return undefined;
const value = Number(raw);
if (!/^-?\d+$/.test(raw) || !Number.isSafeInteger(value)) {
throw new Error(`${name} must be a safe integer`);
fluxer_app_proxy/Cargo.toml
+1 -1
View File
@@ -21,5 +21,5 @@ tokio-util = { version = "0.7.19", features = ["io"] }
tower = { version = "0.5.3", features = ["util"] }
tower-http = { version = "0.7.1", features = ["compression-gzip", "trace"] }
tracing = "0.1.44"
tracing-subscriber = { version = "0.3.23", features = ["env-filter"] }
tracing-subscriber = "0.3.23"
fluxer_app_proxy/src/config.rs
+16 -32
View File
@@ -434,27 +434,23 @@ fn read_csp_sources(name: &'static str) -> Vec<CspSource> {
}
fn read_csp_report_uri(name: &'static str) -> Option<CspReportUri> {
let value = cfg::non_empty_env(name)?;
CspReportUri::parse(name, &value)
let value = cfg::env_value(name)?;
CspReportUri::parse(name, value.trim())
.inspect_err(warn_invalid)
.ok()
}
impl AppProxyConfig {
pub fn from_env() -> Self {
let release_channel = ReleaseChannel::from_env_value(&cfg::read_env_preferred(
&["RELEASE_CHANNEL"],
"stable",
));
let geoip_source = cfg::parse_geoip_source_config(
&cfg::read_first_env(&["FLUXER_GEOIP_DB_PATH", "MAXMIND_DB_PATH"], ""),
"app_proxy",
);
let release_channel =
ReleaseChannel::from_env_value(&cfg::read_env("RELEASE_CHANNEL", "stable"));
let geoip_source =
cfg::parse_geoip_source_config(&cfg::read_env("FLUXER_GEOIP_DB_PATH", ""), "app_proxy");
let geoip_s3_config = cfg::read_geoip_s3_config_from_env(&geoip_source);
let s3_public_endpoint = parse_optional_http_endpoint(
"FLUXER_S3_PUBLIC_ENDPOINT",
cfg::non_empty_env("FLUXER_S3_PUBLIC_ENDPOINT"),
cfg::env_value("FLUXER_S3_PUBLIC_ENDPOINT"),
);
let s3_uploads_bucket = cfg::read_env("FLUXER_S3_BUCKET_UPLOADS", "fluxer-uploads");
let s3_uploads_endpoint = s3_public_endpoint.as_ref().and_then(|endpoint| {
@@ -474,11 +470,11 @@ impl AppProxyConfig {
static_dir: cfg::read_env("FLUXER_STATIC_DIR", "./static"),
index_upstream_url: parse_optional_http_url(
"FLUXER_APP_PROXY_INDEX_UPSTREAM_URL",
cfg::non_empty_env("FLUXER_APP_PROXY_INDEX_UPSTREAM_URL"),
cfg::env_value("FLUXER_APP_PROXY_INDEX_UPSTREAM_URL"),
),
static_cdn_endpoint: parse_optional_http_endpoint(
"FLUXER_STATIC_CDN_ENDPOINT",
cfg::non_empty_env("FLUXER_STATIC_CDN_ENDPOINT"),
cfg::env_value("FLUXER_STATIC_CDN_ENDPOINT"),
),
s3_public_endpoint,
s3_uploads_endpoint,
@@ -489,7 +485,7 @@ impl AppProxyConfig {
60_000u64,
),
release_channel,
build_version: cfg::read_env_preferred(
build_version: cfg::read_first_env(
&["BUILD_VERSION", "FLUXER_BUILD_VERSION"],
env!("CARGO_PKG_VERSION"),
),
@@ -498,21 +494,10 @@ impl AppProxyConfig {
csp: CspConfig::from_env(),
geoip_source,
geoip_s3_config,
trust_client_ip_header: cfg::read_bool_env(
&["FLUXER_TRUST_CLIENT_IP_HEADER", "TRUST_CLIENT_IP_HEADER"],
false,
),
client_ip_header_name: cfg::read_first_env(
&[
"FLUXER_CLIENT_IP_HEADER_NAME",
"FLUXER_CLIENT_IP_HEADER",
"CLIENT_IP_HEADER_NAME",
"CLIENT_IP_HEADER",
],
"x-forwarded-for",
)
.trim()
.to_ascii_lowercase(),
trust_client_ip_header: cfg::read_bool_env("FLUXER_TRUST_CLIENT_IP_HEADER", false),
client_ip_header_name: cfg::read_env("FLUXER_CLIENT_IP_HEADER_NAME", "x-forwarded-for")
.trim()
.to_ascii_lowercase(),
same_origin_hosts: parse_same_origin_hosts(
"FLUXER_APP_PROXY_SAME_ORIGIN_HOSTS",
&cfg::read_env("FLUXER_APP_PROXY_SAME_ORIGIN_HOSTS", ""),
@@ -609,12 +594,11 @@ fn parse_same_origin_host(
}
fn resolve_discovery_upstream_url_from_env() -> String {
resolve_discovery_upstream_url(|name| env::var(name).ok())
resolve_discovery_upstream_url(cfg::env_value)
}
fn resolve_bootstrap_api_public_endpoint_from_env() -> Option<String> {
resolve_bootstrap_api_public_endpoint(|name| env::var(name).ok())
.unwrap_or_else(|error| panic!("{error}"))
resolve_bootstrap_api_public_endpoint(cfg::env_value).unwrap_or_else(|error| panic!("{error}"))
}
fn resolve_bootstrap_api_public_endpoint<F>(mut read_var: F) -> anyhow::Result<Option<String>>
fluxer_app_proxy/src/main.rs
+1 -3
View File
@@ -17,9 +17,7 @@ use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
fn main() -> anyhow::Result<()> {
tracing_subscriber::registry()
.with(
tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()),
)
.with(fluxer_common::config::env_filter("info"))
.with(tracing_subscriber::fmt::layer())
.init();
fluxer_common/Cargo.toml
+1
View File
@@ -16,6 +16,7 @@ serde_json = "1.0.151"
tempfile = "3.27.0"
time = { version = "0.3.55", features = ["formatting", "macros", "parsing"] }
tracing = "0.1.44"
tracing-subscriber = { version = "0.3.23", default-features = false, features = ["env-filter"] }
base64 = "0.23"
hex = "0.4.3"
hmac = "0.13.0"
fluxer_common/src/config.rs
+52 -17
View File
@@ -34,7 +34,7 @@ pub struct GeoipS3Config {
}
pub fn read_geoip_s3_config_from_env(source: &GeoipSourceConfig) -> Option<GeoipS3Config> {
read_geoip_s3_config(source, |name| env::var(name).ok())
read_geoip_s3_config(source, env_value)
}
fn read_geoip_s3_config<F>(source: &GeoipSourceConfig, mut read_var: F) -> Option<GeoipS3Config>
@@ -123,26 +123,23 @@ fn percent_decode(value: &str) -> String {
.unwrap_or_else(|_| value.to_owned())
}
pub fn read_env(name: &str, fallback: &str) -> String {
env::var(name).unwrap_or_else(|_| fallback.to_owned())
pub fn env_value(name: &str) -> Option<String> {
env::var(name).ok().filter(|value| !value.trim().is_empty())
}
pub fn read_env_preferred(names: &[&str], fallback: &str) -> String {
names
.iter()
.find_map(|name| env::var(name).ok().filter(|value| !value.trim().is_empty()))
.unwrap_or_else(|| fallback.to_owned())
pub fn read_env(name: &str, fallback: &str) -> String {
env_value(name).unwrap_or_else(|| fallback.to_owned())
}
pub fn read_first_env(names: &[&str], fallback: &str) -> String {
names
.iter()
.find_map(|name| env::var(name).ok())
.find_map(|name| env_value(name))
.unwrap_or_else(|| fallback.to_owned())
}
pub fn read_bool_env(names: &[&str], fallback: bool) -> bool {
let Some(value) = names.iter().find_map(|name| env::var(name).ok()) else {
pub fn read_bool_env(name: &str, fallback: bool) -> bool {
let Some(value) = env_value(name) else {
return fallback;
};
matches!(
@@ -151,11 +148,10 @@ pub fn read_bool_env(names: &[&str], fallback: bool) -> bool {
)
}
pub fn non_empty_env(name: &str) -> Option<String> {
env::var(name)
.ok()
.map(|v| v.trim().to_owned())
.filter(|v| !v.is_empty())
pub fn env_filter(default: &str) -> tracing_subscriber::EnvFilter {
env_value("RUST_LOG")
.and_then(|filter| tracing_subscriber::EnvFilter::try_new(filter).ok())
.unwrap_or_else(|| tracing_subscriber::EnvFilter::new(default))
}
pub fn normalize_base_path(value: &str) -> String {
@@ -295,7 +291,7 @@ pub fn normalize_public_endpoint(url: &str, base_domain: &str, public_port: Opti
}
pub fn try_normalize_public_endpoint_from_env(url: &str) -> anyhow::Result<String> {
let (base_domain, public_port) = resolve_public_domain_and_port(|name| env::var(name).ok())?;
let (base_domain, public_port) = resolve_public_domain_and_port(env_value)?;
Ok(normalize_public_endpoint(url, &base_domain, public_port))
}
@@ -767,6 +763,45 @@ mod tests {
assert_eq!(None, port);
}
#[test]
fn a_blank_value_reads_as_unset() {
unsafe {
env::set_var("FLUXER_COMMON_TEST_BLANK_EMPTY", "");
env::set_var("FLUXER_COMMON_TEST_BLANK_SPACES", " \t");
env::set_var("FLUXER_COMMON_TEST_BLANK_SET", "value");
env::set_var("FLUXER_COMMON_TEST_BLANK_TRUE", "true");
}
assert_eq!(None, env_value("FLUXER_COMMON_TEST_BLANK_EMPTY"));
assert_eq!(None, env_value("FLUXER_COMMON_TEST_BLANK_SPACES"));
assert_eq!(None, env_value("FLUXER_COMMON_TEST_BLANK_MISSING"));
assert_eq!(
"fallback",
read_env("FLUXER_COMMON_TEST_BLANK_EMPTY", "fallback")
);
assert_eq!(
"fallback",
read_env("FLUXER_COMMON_TEST_BLANK_SPACES", "fallback")
);
assert_eq!(
"value",
read_env("FLUXER_COMMON_TEST_BLANK_SET", "fallback")
);
assert_eq!(
"value",
read_first_env(
&[
"FLUXER_COMMON_TEST_BLANK_EMPTY",
"FLUXER_COMMON_TEST_BLANK_SPACES",
"FLUXER_COMMON_TEST_BLANK_SET",
],
"fallback"
)
);
assert!(read_bool_env("FLUXER_COMMON_TEST_BLANK_EMPTY", true));
assert!(read_bool_env("FLUXER_COMMON_TEST_BLANK_SPACES", true));
assert!(read_bool_env("FLUXER_COMMON_TEST_BLANK_TRUE", false));
}
#[test]
fn trim_trailing_slash_works() {
assert_eq!(
fluxer_docs/src/content/docs/operator/configuration.mdx
+463 -367
View File
File diff suppressed because it is too large Load Diff
fluxer_docs/src/content/docs/operator/get-started.mdx
+3 -3
View File
@@ -238,12 +238,12 @@ The desktop client opens the hosted web app for its release channel, so reach yo
Recovery requires `.env`, a database dump and a backup of `seaweedfs-data`, which holds uploads, avatars, reports and harvests. See [Volumes and buckets](/operator/configuration/#volumes-and-buckets) for everything that needs backing up.
The dump costs no downtime, so run it on a schedule while the stack serves:
The dump costs no downtime, so run it on a schedule while the stack serves. It runs inside the `postgres` container and takes the role and database from that container's environment, which follows `FLUXER_POSTGRES_USERNAME` and `FLUXER_POSTGRES_DATABASE`. It is for the bundled database. A database outside the stack is backed up with its own tools:
```bash
cd ~/fluxer
mkdir -p backups
docker compose exec -T postgres pg_dump -U fluxer -d fluxer --format=custom \
docker compose exec -T postgres sh -c 'pg_dump -U $POSTGRES_USER -d $POSTGRES_DB --format=custom' \
> "backups/fluxer-$(date -u +%Y%m%dT%H%M%SZ).dump"
```
@@ -251,7 +251,7 @@ PowerShell writes UTF-16 through `>`, which corrupts a binary dump. On Windows,
```powershell
mkdir -Force backups
docker compose exec -T postgres pg_dump -U fluxer -d fluxer --format=custom -f /tmp/fluxer.dump
docker compose exec -T postgres sh -c 'pg_dump -U $POSTGRES_USER -d $POSTGRES_DB --format=custom -f /tmp/fluxer.dump'
docker compose cp postgres:/tmp/fluxer.dump backups/fluxer.dump
docker compose exec -T postgres rm /tmp/fluxer.dump
```
fluxer_docs/src/content/docs/operator/reverse-proxy.mdx
+1 -1
View File
@@ -396,7 +396,7 @@ Forward every path and query string unchanged. The edge handles routing:
| `/.well-known/assetlinks.json` | Android app association |
| `/version.json` | Client version metadata |
All other paths serve the web app.
All other paths serve the web app. The admin path follows `FLUXER_ADMIN_BASE_PATH`, `/admin` by default.
Pass the query string on `/gateway` through untouched. Clients always send `?v=`, `?encoding=`, `?compress=` and `?stream=`, and `1` is the only version the Gateway accepts.
fluxer_docs/src/content/docs/operator/upgrading.mdx
+20 -12
View File
@@ -25,6 +25,8 @@ The images come from `FLUXER_IMAGE_TAG` in `.env`. The stack files come from a g
The repository holds no ref by the name of a pinned image tag. A release tags each image on its own, as `[email protected]`, so pass `--ref` with that tag or with the commit it points at. Without that override the download fails with exit 4.
The stack files on `main` can run ahead of the `v1` images. The current `docker-compose.yml` passes an optional setting with no Compose default through empty when `.env` leaves it out, and only images built from the same change or later read an empty value as unset. Older images reject some of those empty values and `api`, `worker` and `media-proxy` fail to start. Refresh the stack files only once the images the tag names are at least as new, or pin both with `--ref`.
## What the script does
`--update` runs these steps in this order, and stops on the first one that fails:
@@ -195,6 +197,10 @@ Within minutes of the upgrade, `worker` starts deleting existing data that is pa
Instances on Cassandra already behave this way.
## Check the svc request ceiling
`docker-compose.yml` used to pass `FLUXER_SVC_MAX_CONCURRENT_REQUESTS` only to the `users` and `messages` routers and shards. It now reaches every svc container, so a value set in `.env` also caps `snowflakes`, `gifs` and `unfurl`. Unset, `snowflakes` allows `320` requests in flight, `messages` `192` and the rest `64`. An instance that sets the name either raises it to at least `320` or removes it from `.env`, then runs `docker compose up -d`.
## Add the passkey columns on Cassandra
An instance on the Postgres backend needs nothing here. An instance on Cassandra or Scylla adds two columns to `webauthn_credentials` before it starts the new `api` and `worker` images. Replace `fluxer` with the value of `FLUXER_CASSANDRA_KEYSPACE`.
@@ -229,7 +235,7 @@ Nothing else is copied. The dump covers `postgres-data`. The other volumes eithe
Put a dump on a timer as well. On Linux and macOS, this crontab line writes one a night and keeps two weeks of them:
```cron
15 3 * * * cd /srv/fluxer && docker compose exec -T postgres pg_dump -U fluxer -d fluxer --format=custom > "backups/fluxer-$(date -u +\%Y\%m\%dT\%H\%M\%SZ).dump" && find backups -name 'fluxer-*.dump' -mtime +14 -delete
15 3 * * * cd /srv/fluxer && docker compose exec -T postgres sh -c 'pg_dump -U $POSTGRES_USER -d $POSTGRES_DB --format=custom' > "backups/fluxer-$(date -u +\%Y\%m\%dT\%H\%M\%SZ).dump" && find backups -name 'fluxer-*.dump' -mtime +14 -delete
```
`/srv/fluxer` stands for the directory holding `.env`. Write it as an absolute path, because cron does not expand `~`. An unescaped `%` in a crontab is a newline, which is why every one above has a backslash.
@@ -269,7 +275,7 @@ The `seaweedfs-data` volume stays until you delete it. Copy its objects to the n
The other bundled services have no shipped overlay. Take one out with an override file listed in `COMPOSE_FILE`, which [Keep a local compose change](#keep-a-local-compose-change) describes, rather than by editing `docker-compose.yml`, which the Place step replaces on every upgrade.
The installer backs up only the bundled database and object store. Arrange separate backups for external stores before upgrading, and keep them with the release's backup record.
The installer backs up only the bundled database and object store. It skips the database dump when the stack defines no `postgres` service, and when `FLUXER_POSTGRES_HOST` or the host in `FLUXER_POSTGRES_URL` names anything other than `postgres`. The bundled service is idle in that shape, and its data directory still holds the role and database it was first started with, so the by-hand dump and restore commands on this page do not work against it either. Arrange separate backups for external stores before upgrading, and keep them with the release's backup record.
## Roll back
@@ -295,12 +301,12 @@ The database restores from the custom-format dump:
```bash
docker compose stop
docker compose up -d --wait postgres
docker compose exec -T postgres pg_restore -U fluxer -d fluxer --clean --if-exists \
docker compose exec -T postgres sh -c 'pg_restore -U $POSTGRES_USER -d $POSTGRES_DB --clean --if-exists' \
< backups/record-20260831T120000Z/fluxer.dump
docker compose up -d
```
`--clean` prints notices about objects that do not exist yet, which is expected against a fresh directory.
`--clean` prints notices about objects that do not exist yet, which is expected against a fresh directory. The role and database come from the `postgres` container's environment, which follows `FLUXER_POSTGRES_USERNAME` and `FLUXER_POSTGRES_DATABASE`, the same way the installer's dump reads them. These commands are for the bundled database. A database outside the stack restores with its own tools.
PowerShell has no `<` redirection. On Windows, copy the dump into the container and name it as a file:
@@ -308,7 +314,7 @@ PowerShell has no `<` redirection. On Windows, copy the dump into the container
docker compose stop
docker compose up -d --wait postgres
docker compose cp backups\record-20260831T120000Z\fluxer.dump postgres:/tmp/fluxer.dump
docker compose exec -T postgres pg_restore -U fluxer -d fluxer --clean --if-exists /tmp/fluxer.dump
docker compose exec -T postgres sh -c 'pg_restore -U $POSTGRES_USER -d $POSTGRES_DB --clean --if-exists /tmp/fluxer.dump'
docker compose exec -T postgres rm /tmp/fluxer.dump
docker compose up -d
```
@@ -327,12 +333,12 @@ For a `seaweedfs-data.tar`, write `tar xf` in place of `tar xzf`. `tar` extracts
## Move to a new Postgres major version
`docker-compose.yml` pins `postgres:16-alpine`. A newer major does not read the data directory an older major wrote, so the data moves across through a dump. The upgrade refuses when the refreshed compose file changes that pin, because the move destroys the volume holding the database.
`docker-compose.yml` runs `postgres:16-alpine` unless `FLUXER_POSTGRES_IMAGE` in `.env` names another image. A newer major does not read the data directory an older major wrote, so the data moves across through a dump. The upgrade refuses when the refreshed compose file would run a different major from the current one, because the move destroys the volume holding the database. It reads the current major from the image the old file names, and takes `FLUXER_POSTGRES_IMAGE` into account only for a file that reads it.
Take the dump while the old major is still serving, then stop everything and drop the volume:
```bash
docker compose exec -T postgres pg_dump -U fluxer -d fluxer --format=custom \
docker compose exec -T postgres sh -c 'pg_dump -U $POSTGRES_USER -d $POSTGRES_DB --format=custom' \
> backups/pre-major.dump
docker compose down
docker volume rm fluxer_postgres-data
@@ -342,16 +348,18 @@ On Windows, run `pg_dump` inside the container with `-f /tmp/pre-major.dump` and
Once that volume is removed, the dump is the only copy of the database.
Put the new `postgres` tag in `docker-compose.yml`, start the database on its own, and restore into the empty directory:
`FLUXER_POSTGRES_IMAGE` takes effect only once `docker-compose.yml` reads it. An instance on older stack files runs `sh install.sh --update` first, or edits the tag in its `docker-compose.yml` as before.
Set `FLUXER_POSTGRES_IMAGE` in `.env` to the new tag, such as `postgres:17-alpine`, start the database on its own, and restore into the empty directory:
```bash
docker compose up -d --wait postgres
docker compose exec -T postgres pg_restore -U fluxer -d fluxer --clean --if-exists \
docker compose exec -T postgres sh -c 'pg_restore -U $POSTGRES_USER -d $POSTGRES_DB --clean --if-exists' \
< backups/pre-major.dump
docker compose up -d
```
Run `sh install.sh --update` afterwards to finish the move on the refreshed files.
Run `sh install.sh --update` afterwards to finish the move on the refreshed files. Remove the `FLUXER_POSTGRES_IMAGE` line once the refreshed `docker-compose.yml` runs the same major by default.
## Pin a release
@@ -359,11 +367,11 @@ Run `sh install.sh --update` afterwards to finish the move on the refreshed file
A pinned instance also pins its stack files. [Match the images to the stack files](#match-the-images-to-the-stack-files) has the `--ref` a pinned tag needs.
The tag applies only to the Fluxer images. `caddy`, `postgres`, `valkey`, `nats`, `meilisearch`, `seaweedfs` and `livekit` are pinned in `docker-compose.yml`, and they move only when an upgrade refreshes that file.
The tag applies only to the Fluxer images. `caddy`, `postgres`, `valkey`, `nats`, `meilisearch`, `seaweedfs` and `livekit` take their tags from `docker-compose.yml`, and move when an upgrade refreshes that file. An image name in `.env`, such as `FLUXER_POSTGRES_IMAGE`, holds one of them in place instead. [Images](/operator/configuration/#images) lists the names.
## Change the domain or the passkey relying party
`FLUXER_DOMAIN` supplies the default `FLUXER_PASSKEY_RP_ID`, which is the domain a passkey is tied to. A passkey works only under the identifier it was created with.
`FLUXER_DOMAIN` supplies the default `FLUXER_PASSKEY_RP_ID`, which is the domain a passkey is tied to. `FLUXER_PUBLIC_ORIGIN` does not move it. A passkey works only under the identifier it was created with.
:::danger[Changing either value invalidates every passkey]
Passkeys registered under the old `FLUXER_PASSKEY_RP_ID` stop working and cannot be recovered. Every user has to register a new one, so keep another sign-in method working before you change it.
fluxer_docs/src/installer/install.ps1
+80 -14
View File
@@ -435,8 +435,8 @@ function Get-FluxerRefForTag([string]$Tag) {
}
# The images come from FLUXER_IMAGE_TAG and the stack files come from a git ref. A release tags its
# images and its commit with the same CalVer string, so a pinned tag names the commit that carries
# its compose files. The moving tags v1 and latest track main.
# images and its commit with the same CalVer string, so a pinned tag names the commit that holds its
# compose files. The moving tags v1 and latest map to main, which can run ahead of the v1 images.
function Assert-FluxerDerivedRef([string]$Value, [string]$EnvPath) {
if ($Value.Length -eq 0) {
Stop-Fluxer "$EnvPath declares no FLUXER_IMAGE_TAG, so no ref can be derived. Pass -Ref." $FluxerExitRefused
@@ -631,8 +631,10 @@ function Remove-FluxerStagingDirectory([string]$Path) {
# Invoke-WebRequest -Uri https://raw.githubusercontent.com/fluxerapp/fluxer/main/deploy/self-hosting/docker-compose.yml -OutFile docker-compose.yml
#
# The files come from a git ref and the images come from FLUXER_IMAGE_TAG. The ref is derived from
# the tag unless -Ref names one, which is the pairing rule that stops a compose file from asking for
# a variable the running images do not read.
# the tag unless -Ref names one. A pinned CalVer tag names the commit its images were built from, so
# its compose file asks only for variables those images read. The moving tags v1 and latest map to
# main, and main's compose file can run ahead of the v1 images until the image builds are dispatched
# again.
#
# Everything lands in a staging directory first, so a failed download leaves the working directory
# on the set it already had, and so the upgrade can compare old against new before replacing.
@@ -995,15 +997,34 @@ function Get-FluxerRunningImageId($Running, [string]$Reference) {
return ''
}
function Get-FluxerPostgresMajor([string]$Path) {
function Get-FluxerPostgresMajor([string]$Path, [string]$EnvPath) {
if (-not (Test-Path -LiteralPath $Path)) {
return ''
}
$image = ''
foreach ($line in [System.IO.File]::ReadAllText($Path).Split("`n")) {
if ($line -match '^\s*image:\s*postgres:(\d+)') {
return $Matches[1]
if ($line -match '^\s*image:\s*(postgres:\S*)') {
$image = $Matches[1]
break
}
}
if ($image.Length -eq 0) {
foreach ($line in [System.IO.File]::ReadAllText($Path).Split("`n")) {
if ($line -match '^\s*image:\s*\$\{FLUXER_POSTGRES_IMAGE:-([^}]*)\}') {
$image = $Matches[1]
$configured = Get-FluxerComposeValue $EnvPath 'FLUXER_POSTGRES_IMAGE'
if ($configured.Length -gt 0) {
$image = $configured
}
break
}
}
}
$name = ($image -split '@')[0]
$name = ($name -split '/')[-1]
if ($name -match ':(\d+)[^:]*$') {
return $Matches[1]
}
return ''
}
@@ -1297,9 +1318,12 @@ function Test-FluxerDumpHeader([string]$Path) {
# schema change.
#
# By hand:
# docker compose exec -T postgres pg_dump -U fluxer -d fluxer --format=custom > backups\fluxer.dump
# docker compose exec -T postgres sh -c 'pg_dump -U $POSTGRES_USER -d $POSTGRES_DB --format=custom -f /tmp/fluxer.dump'
# docker compose cp postgres:/tmp/fluxer.dump backups\fluxer.dump
# docker compose exec -T postgres rm /tmp/fluxer.dump
#
# The database and the role are both named fluxer and are fixed in docker-compose.yml. Keep -T.
# The postgres container's POSTGRES_USER and POSTGRES_DB follow FLUXER_POSTGRES_USERNAME and
# FLUXER_POSTGRES_DATABASE, so the command needs no names. Keep -T.
# Without it Docker attaches a terminal to the command and the dump arrives corrupted, which is
# why the first five bytes are checked against the custom-format magic rather than only the size.
#
@@ -1327,11 +1351,44 @@ function Test-FluxerStackDefinesService([string]$Name, [string]$TargetDir) {
return $script:FluxerStackServices -contains $Name
}
# The bundled postgres container takes POSTGRES_USER and POSTGRES_DB from FLUXER_POSTGRES_USERNAME
# and FLUXER_POSTGRES_DATABASE, and the image applies them only to an empty data directory. When
# those names point the apps at a database outside the stack, the bundled directory still holds the
# role and database it was first started with, so a dump that reads the container env asks for a
# role that is not there. The bundled service is idle in that shape and the dump skips it.
function Test-FluxerPostgresExternal([string]$TargetDir) {
$envPath = Join-Path $TargetDir '.env'
$pgHost = Get-FluxerComposeValue $envPath 'FLUXER_POSTGRES_HOST'
if ($pgHost.Length -gt 0 -and $pgHost -ne 'postgres') {
return $true
}
$url = Get-FluxerComposeValue $envPath 'FLUXER_POSTGRES_URL'
if ($url.Length -eq 0) {
return $false
}
$urlHost = $url
$scheme = $urlHost.IndexOf('://')
if ($scheme -ge 0) {
$urlHost = $urlHost.Substring($scheme + 3)
}
$urlHost = ($urlHost -split '[/?]', 2)[0]
$at = $urlHost.LastIndexOf('@')
if ($at -ge 0) {
$urlHost = $urlHost.Substring($at + 1)
}
$urlHost = ($urlHost -split ':', 2)[0]
return $urlHost -ne 'postgres'
}
function Backup-FluxerDatabase([string]$Record, [string]$TargetDir) {
if (-not (Test-FluxerStackDefinesService 'postgres' $TargetDir)) {
Write-FluxerLine 'Skipping the database dump. This stack defines no postgres service, so its database runs outside the stack and only the operator of that database can dump it.'
return
}
if (Test-FluxerPostgresExternal $TargetDir) {
Write-FluxerLine 'Skipping the database dump. FLUXER_POSTGRES_HOST or FLUXER_POSTGRES_URL points the stack at a database outside it, so the bundled postgres service is idle and only the operator of that database can dump it.'
return
}
if (-not (Test-FluxerPostgresRunning)) {
Write-FluxerLine 'Postgres is not running. Starting it for the dump.'
if ((Invoke-FluxerDocker @('compose', 'up', '-d', '--wait', 'postgres')) -ne 0) {
@@ -1340,7 +1397,7 @@ function Backup-FluxerDatabase([string]$Record, [string]$TargetDir) {
}
$dump = Join-Path $Record $FluxerDumpFile
Write-FluxerLine 'Dumping the database.'
$code = Invoke-FluxerDockerToFile @('compose', 'exec', '-T', 'postgres', 'pg_dump', '-U', 'fluxer', '-d', 'fluxer', '--format=custom') $dump $TargetDir
$code = Invoke-FluxerDockerToFile @('compose', 'exec', '-T', 'postgres', 'sh', '-c', '"exec pg_dump -U $POSTGRES_USER -d $POSTGRES_DB --format=custom"') $dump $TargetDir
if ($code -ne 0) {
Remove-FluxerTemporary $dump
Stop-Fluxer 'pg_dump failed. The instance is untouched.' $FluxerExitBackup
@@ -1462,8 +1519,9 @@ function Backup-FluxerInstance([string]$Record, [string]$TargetDir, [string]$Pro
# The refreshed file is still staged when this runs, so a refusal here leaves the instance exactly
# as it was.
function Assert-FluxerPostgresMajor([string]$TargetDir, [string]$StagingDir) {
$old = Get-FluxerPostgresMajor (Join-Path $TargetDir $script:FluxerComposeBase)
$new = Get-FluxerPostgresMajor (Join-Path $StagingDir 'docker-compose.yml')
$envPath = Join-Path $TargetDir '.env'
$old = Get-FluxerPostgresMajor (Join-Path $TargetDir $script:FluxerComposeBase) $envPath
$new = Get-FluxerPostgresMajor (Join-Path $StagingDir 'docker-compose.yml') $envPath
if ($old.Length -eq 0 -or $new.Length -eq 0 -or $old -eq $new) {
return
}
@@ -1569,8 +1627,8 @@ function Show-FluxerUpdatePlan([string]$TargetDir, [string]$EnvPath, [string]$Ba
if ($changed -eq 0) {
Write-FluxerLine " Note: ref $Ref moves no stack file"
}
$old = Get-FluxerPostgresMajor (Join-Path $TargetDir $script:FluxerComposeBase)
$new = Get-FluxerPostgresMajor (Join-Path $staging 'docker-compose.yml')
$old = Get-FluxerPostgresMajor (Join-Path $TargetDir $script:FluxerComposeBase) $EnvPath
$new = Get-FluxerPostgresMajor (Join-Path $staging 'docker-compose.yml') $EnvPath
if ($old.Length -gt 0 -and $new.Length -gt 0 -and $old -ne $new) {
Write-FluxerLine " Refusal: postgres moves from $old to $new, which this script does not do"
Write-FluxerLine ' Outcome: the run stops at that refusal and changes nothing'
@@ -1823,6 +1881,14 @@ function Get-FluxerEnvScalar([string]$EnvPath, [string]$Name) {
return $raw
}
function Get-FluxerComposeValue([string]$EnvPath, [string]$Name) {
$value = [string][Environment]::GetEnvironmentVariable($Name)
if ($value.Length -eq 0 -and (Test-Path -LiteralPath $EnvPath)) {
$value = Get-FluxerEnvScalar $EnvPath $Name
}
return $value
}
function Get-FluxerComposeSetting([string]$EnvPath) {
$value = ''
$source = 'the environment'
fluxer_docs/src/installer/install.sh
+61 -12
View File
@@ -711,8 +711,8 @@ fluxer_ref_for_tag() {
# The images come from FLUXER_IMAGE_TAG and the stack files come from a git ref.
# A release tags its images and its commit with the same CalVer string, so a
# pinned tag names the commit that carries its compose files. The moving tags v1
# and latest track main.
# pinned tag names the commit that holds its compose files. The moving tags v1
# and latest map to main, which can run ahead of the v1 images.
fluxer_resolve_ref() {
[ -z "$opt_ref" ] || return 0
if [ "$opt_update" -eq 1 ] || [ "$opt_rollback" -eq 1 ]; then
@@ -763,9 +763,11 @@ fluxer_open_scratch() {
# curl -fsSL https://raw.githubusercontent.com/fluxerapp/fluxer/main/deploy/self-hosting/docker-compose.yml -o docker-compose.yml
#
# The files come from a git ref and the images come from FLUXER_IMAGE_TAG. The
# ref is derived from the tag unless --ref names one, which is the pairing rule
# that stops a compose file from asking for a variable the running images do not
# read, or from pinning a service image the release never built.
# ref is derived from the tag unless --ref names one. A pinned CalVer tag names
# the commit its images were built from, so its compose file asks only for
# variables those images read and pins only images the release built. The moving
# tags v1 and latest map to main, and main's compose file can run ahead of the v1
# images until the image builds are dispatched again.
fluxer_fetch_stack() {
fluxer_say "Downloading the stack files from ref $opt_ref."
fluxer_stack_files > "$fluxer_scratch/files"
@@ -1237,6 +1239,14 @@ fluxer_env_scalar() {
printf '%s' "$fluxer_scalar"
}
fluxer_compose_value() {
eval "fluxer_cv=\${$1:-}"
if [ -z "$fluxer_cv" ]; then
fluxer_cv=$(fluxer_env_scalar "$1")
fi
printf '%s' "$fluxer_cv"
}
fluxer_read_compose_setting() {
fluxer_compose_file=${COMPOSE_FILE:-}
fluxer_compose_from="the environment"
@@ -1469,12 +1479,13 @@ fluxer_postgres_running() {
# the pull is the only way back across a schema change.
#
# By hand:
# docker compose exec -T postgres pg_dump -U fluxer -d fluxer --format=custom > backups/fluxer.dump
# docker compose exec -T postgres sh -c 'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom' > backups/fluxer.dump
#
# The database and the role are both named fluxer and are fixed in
# docker-compose.yml. Keep -T. Without it Docker attaches a terminal to the
# command and the dump arrives corrupted, which is why the first five bytes are
# checked against the custom-format magic below rather than only the size.
# The postgres container's POSTGRES_USER and POSTGRES_DB follow
# FLUXER_POSTGRES_USERNAME and FLUXER_POSTGRES_DATABASE, so the command needs no
# names. Keep -T. Without it Docker attaches a terminal to the command and the
# dump arrives corrupted, which is why the first five bytes are checked against
# the custom-format magic below rather than only the size.
#
# The dump runs against the live stack. pg_dump reads inside one transaction, so
# it sees a consistent database without stopping anything. The volume copy below
@@ -1501,11 +1512,39 @@ $(fluxer_compose_error ' ')"
grep -qxF "$1" "$fluxer_scratch/all-services"
}
# The bundled postgres container takes POSTGRES_USER and POSTGRES_DB from
# FLUXER_POSTGRES_USERNAME and FLUXER_POSTGRES_DATABASE, and the image applies
# them only to an empty data directory. When those names point the apps at a
# database outside the stack, the bundled directory still holds the role and
# database it was first started with, so a dump that reads the container env asks
# for a role that is not there. The bundled service is idle in that shape and the
# dump skips it.
#
# By hand:
# grep -E '^FLUXER_POSTGRES_(HOST|URL)=' .env
fluxer_postgres_external() {
fluxer_pg_host=$(fluxer_compose_value FLUXER_POSTGRES_HOST)
if [ -n "$fluxer_pg_host" ] && [ "$fluxer_pg_host" != postgres ]; then
return 0
fi
fluxer_pg_url=$(fluxer_compose_value FLUXER_POSTGRES_URL)
[ -n "$fluxer_pg_url" ] || return 1
fluxer_pg_url_host=${fluxer_pg_url#*://}
fluxer_pg_url_host=${fluxer_pg_url_host%%[/?]*}
fluxer_pg_url_host=${fluxer_pg_url_host##*@}
fluxer_pg_url_host=${fluxer_pg_url_host%%:*}
[ "$fluxer_pg_url_host" != postgres ]
}
fluxer_dump_postgres() {
if ! fluxer_stack_defines_service postgres; then
fluxer_say 'Skipping the database dump. This stack defines no postgres service, so its database runs outside the stack and only the operator of that database can dump it.'
return 0
fi
if fluxer_postgres_external; then
fluxer_say 'Skipping the database dump. FLUXER_POSTGRES_HOST or FLUXER_POSTGRES_URL points the stack at a database outside it, so the bundled postgres service is idle and only the operator of that database can dump it.'
return 0
fi
if ! fluxer_postgres_running; then
fluxer_say 'Postgres is not running. Starting it for the dump.'
if ! $fluxer_engine compose up -d --wait postgres; then
@@ -1514,7 +1553,7 @@ fluxer_dump_postgres() {
fi
fluxer_dump_path="$fluxer_record/$FLUXER_DUMP_FILE"
fluxer_say 'Dumping the database.'
if ! $fluxer_engine compose exec -T postgres pg_dump -U fluxer -d fluxer --format=custom > "$fluxer_dump_path"; then
if ! $fluxer_engine compose exec -T postgres sh -c 'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom' > "$fluxer_dump_path"; then
rm -f "$fluxer_dump_path"
fluxer_fail 7 'pg_dump failed. The instance is untouched.'
fi
@@ -1627,7 +1666,17 @@ fluxer_backup() {
}
fluxer_postgres_major() {
sed -n 's/^[[:space:]]*image:[[:space:]]*postgres:\([0-9][0-9]*\).*/\1/p' "$1" | head -n 1
fluxer_pg_image=$(sed -n 's/^[[:space:]]*image:[[:space:]]*\(postgres:[^[:space:]]*\).*/\1/p' "$1" | head -n 1)
if [ -z "$fluxer_pg_image" ]; then
fluxer_pg_image=$(sed -n 's/^[[:space:]]*image:[[:space:]]*${FLUXER_POSTGRES_IMAGE:-\([^}]*\)}.*/\1/p' "$1" | head -n 1)
if [ -n "$fluxer_pg_image" ] && [ -n "$(fluxer_compose_value FLUXER_POSTGRES_IMAGE)" ]; then
fluxer_pg_image=$(fluxer_compose_value FLUXER_POSTGRES_IMAGE)
fi
fi
fluxer_pg_image=${fluxer_pg_image%%@*}
case ${fluxer_pg_image##*/} in
*:*) printf '%s\n' "${fluxer_pg_image##*:}" | sed -n 's/^\([0-9][0-9]*\).*/\1/p' ;;
esac
}
# A newer Postgres major does not read the data directory an older major wrote,
fluxer_gateway/Dockerfile
+1 -3
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
FROM erlang:28-slim AS build
ARG LOGGER_LEVEL=info
ARG RUST_TOOLCHAIN=1.98.1
ENV PATH="/root/.cargo/bin:${PATH}"
@@ -15,7 +14,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
gcc \
g++ \
libc6-dev \
gettext-base \
ca-certificates \
&& rm -rf /var/lib/apt/lists/*
@@ -29,7 +27,7 @@ COPY . .
WORKDIR /usr/src/app/fluxer_gateway
RUN rebar3 compile --deps_only
RUN LOGGER_LEVEL=${LOGGER_LEVEL} envsubst < config/sys.config.template > config/sys.config && \
RUN cp config/sys.config.template config/sys.config && \
rebar3 as prod release
FROM erlang:28-slim
fluxer_gateway/scripts/docker_entrypoint.sh
+2 -2
View File
@@ -50,8 +50,8 @@ clamp_int() {
echo "$value"
}
: "${FLUXER_ERLANG_SCHEDULERS_MIN:=2}"
: "${FLUXER_ERLANG_SCHEDULERS_MAX:=16}"
is_positive_int "${FLUXER_ERLANG_SCHEDULERS_MIN:-}" || FLUXER_ERLANG_SCHEDULERS_MIN=2
is_positive_int "${FLUXER_ERLANG_SCHEDULERS_MAX:-}" || FLUXER_ERLANG_SCHEDULERS_MAX=16
: "${FLUXER_ERLANG_NODE_NAME:[email protected]}"
: "${FLUXER_ERLANG_DIST_PORT:=8081}"
fluxer_gateway/src/gateway/fluxer_gateway_config.erl
+36 -52
View File
@@ -3,7 +3,7 @@
-module(fluxer_gateway_config).
-typing([eqwalizer]).
-export([load/0, build_config/1, optional_string/1]).
-export([load/0, build_config/1, optional_string/1, env_value/1]).
-export_type([config/0]).
-type config() :: map().
@@ -31,8 +31,7 @@ env_config() ->
<<"internal">> => env_internal_config(),
<<"public">> => env_public_config(),
<<"proxy">> => env_proxy_config(),
<<"services">> => env_services_config(),
<<"telemetry">> => env_telemetry_config()
<<"services">> => env_services_config()
}.
-spec env_internal_config() -> map().
@@ -82,7 +81,7 @@ env_gateway_base_config() ->
<<"push_outbox_request_timeout_ms">> => env_int(
"FLUXER_GATEWAY_PUSH_OUTBOX_REQUEST_TIMEOUT_MS", 100000
),
<<"logger_level">> => env_binary("FLUXER_GATEWAY_LOGGER_LEVEL", <<"info">>),
<<"logger_level">> => env_optional_binary("FLUXER_GATEWAY_LOGGER_LEVEL"),
<<"api_rpc_endpoint">> => env_optional_binary("FLUXER_GATEWAY_API_RPC_ENDPOINT"),
<<"cluster_enabled">> => env_bool("FLUXER_GATEWAY_CLUSTER_ENABLED", false),
<<"cluster_discovery_dns_name">> => env_optional_binary(
@@ -109,7 +108,6 @@ env_gateway_base_config() ->
<<"presence_push_buffer_max_bytes">> => env_int(
"FLUXER_GATEWAY_PRESENCE_PUSH_BUFFER_MAX_BYTES", 1048576
),
<<"shutdown_drain_wait_ms">> => env_int("FLUXER_GATEWAY_SHUTDOWN_DRAIN_WAIT_MS", 5000),
<<"gateway_http_rpc_max_concurrency">> => env_int(
"FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY", 512
),
@@ -131,19 +129,11 @@ env_nats_config() ->
<<"auth_token">> => env_string("FLUXER_NATS_AUTH_TOKEN", "")
}.
-spec env_telemetry_config() -> map().
env_telemetry_config() ->
#{
<<"enabled">> => env_bool("FLUXER_TELEMETRY_ENABLED", false),
<<"environment">> => env_string("FLUXER_ENV", "development")
}.
-spec build_config(map()) -> config().
build_config(RawConfig) ->
Service = get_map(RawConfig, [<<"services">>, <<"gateway">>]),
Internal = get_map(RawConfig, [<<"internal">>]),
Nats = get_map(RawConfig, [<<"services">>, <<"nats">>]),
Telemetry = get_map(RawConfig, [<<"telemetry">>]),
Proxy = get_map(RawConfig, [<<"proxy">>]),
Public = get_map(RawConfig, [<<"public">>]),
lists:foldl(fun maps:merge/2, #{}, [
@@ -152,7 +142,7 @@ build_config(RawConfig) ->
build_sharding_config(Service),
build_http_config(Service),
build_cluster_config(Service, Public),
build_misc_config(Service, Telemetry)
build_misc_config(Service)
]).
-spec build_core_config(map(), map(), map(), map()) -> config().
@@ -178,9 +168,6 @@ build_core_config(Service, Internal, Nats, Proxy) ->
build_push_config(Service, Public) ->
#{
push_enabled => get_bool(Service, <<"push_enabled">>, true),
push_user_guild_settings_cache_mb =>
get_int(Service, <<"push_user_guild_settings_cache_mb">>, 1024),
push_blocked_ids_cache_mb => get_int(Service, <<"push_blocked_ids_cache_mb">>, 1024),
static_cdn_endpoint => public_endpoint(
get_binary(Service, <<"static_cdn_endpoint">>, <<"http://localhost:8088">>), Public
),
@@ -208,8 +195,7 @@ build_sharding_config(Service) ->
guild_counts_cache_shards => get_optional_int(Service, <<"guild_counts_cache_shards">>),
guild_shards => get_optional_int(Service, <<"guild_shards">>),
session_shards => get_optional_int(Service, <<"session_shards">>),
session_connect_max_queue => get_int(Service, <<"session_connect_max_queue">>, 1024),
shutdown_drain_wait_ms => get_int(Service, <<"shutdown_drain_wait_ms">>, 5000)
session_connect_max_queue => get_int(Service, <<"session_connect_max_queue">>, 1024)
}.
-spec build_http_config(map()) -> config().
@@ -247,19 +233,15 @@ build_cluster_config(Service, Public) ->
)
}.
-spec build_misc_config(map(), map()) -> config().
build_misc_config(Service, Telemetry) ->
-spec build_misc_config(map()) -> config().
build_misc_config(Service) ->
#{
handoff_enable_event_pause => get_bool(
Service, <<"handoff_enable_event_pause">>, false
),
voice_state_counts_sync_interval_ms =>
get_int(Service, <<"voice_state_counts_sync_interval_ms">>, 30000),
logger_level => get_log_level(Service, <<"logger_level">>, warning),
telemetry => #{
enabled => get_bool(Telemetry, <<"enabled">>, true),
environment => get_string(Telemetry, <<"environment">>, "development")
}
logger_level => get_log_level(Service, <<"logger_level">>, info)
}.
-spec get_map(map(), [binary()]) -> map().
@@ -271,21 +253,33 @@ get_map(Map, Keys) ->
-spec env_string(string(), string()) -> string().
env_string(Name, Default) ->
case os:getenv(Name) of
false -> Default;
"" -> Default;
case env_value(Name) of
undefined -> Default;
Value -> Value
end.
-spec env_value(string()) -> string() | undefined.
env_value(Name) ->
case os:getenv(Name) of
false -> undefined;
Value -> non_blank(Value)
end.
-spec non_blank(string()) -> string() | undefined.
non_blank(Value) ->
case string:trim(Value) of
"" -> undefined;
_ -> Value
end.
-spec env_binary(string(), binary()) -> binary().
env_binary(Name, Default) ->
characters_to_binary_or_default(env_string(Name, binary_to_list(Default)), Default).
-spec env_optional_binary(string()) -> binary() | undefined.
env_optional_binary(Name) ->
case os:getenv(Name) of
false -> undefined;
"" -> undefined;
case env_value(Name) of
undefined -> undefined;
Value -> characters_to_binary_or_default(Value, undefined)
end.
@@ -300,27 +294,17 @@ characters_to_binary_or_default(Value, Default) ->
-spec env_int(string(), integer()) -> integer().
env_int(Name, Default) ->
parse_env_int(Name, os:getenv(Name), Default).
case env_value(Name) of
undefined -> Default;
Value -> parse_int(Name, Value)
end.
-spec parse_env_int(string(), false | string(), integer()) -> integer().
parse_env_int(_Name, false, Default) ->
Default;
parse_env_int(_Name, "", Default) ->
Default;
parse_env_int(Name, Value, Default) ->
parse_int(Name, Value, Default).
-spec parse_int(string(), string(), integer()) -> integer().
parse_int(Name, Value, Default) ->
case string:trim(Value) of
"" ->
Default;
Trimmed ->
try list_to_integer(Trimmed) of
Parsed -> Parsed
catch
error:badarg -> erlang:error({invalid_integer_env, Name, Value})
end
-spec parse_int(string(), string()) -> integer().
parse_int(Name, Value) ->
try list_to_integer(string:trim(Value)) of
Parsed -> Parsed
catch
error:badarg -> erlang:error({invalid_integer_env, Name, Value})
end.
-spec env_bool(string(), boolean()) -> boolean().
fluxer_gateway/src/gateway/fluxer_gateway_env.erl
+2 -26
View File
@@ -60,36 +60,16 @@ build_config() ->
-spec apply_system_config(config()) -> ok.
apply_system_config(Config) ->
apply_logger_config(Config),
store_environment(Config).
-spec apply_logger_config(config()) -> ok.
apply_logger_config(Config) ->
LoggerLevel = resolve_logger_level(Config),
_ = logger:set_primary_config(level, LoggerLevel),
_ = logger:set_handler_config(default, level, LoggerLevel),
ok.
-spec store_environment(config()) -> ok.
store_environment(Config) ->
Telemetry = maps:get(telemetry, Config, #{}),
Environment = maps:get(environment, Telemetry, <<"unknown">>),
EnvBin = ensure_binary(Environment),
persistent_term:put({fluxer_config, environment}, EnvBin),
ok.
-spec ensure_binary(term()) -> binary().
ensure_binary(Value) when is_binary(Value) -> Value;
ensure_binary(Value) when is_list(Value) -> characters_to_binary_or_unknown(Value);
ensure_binary(Value) when is_atom(Value) -> atom_to_binary(Value, utf8);
ensure_binary(_) -> <<"unknown">>.
-spec resolve_logger_level(config()) -> logger_level().
resolve_logger_level(Config) ->
Default = normalize_logger_level(maps:get(logger_level, Config, info)),
case os:getenv("LOGGER_LEVEL") of
false -> Default;
"" -> Default;
case fluxer_gateway_config:env_value("LOGGER_LEVEL") of
undefined -> Default;
Value -> parse_logger_level(Value, Default)
end.
@@ -117,7 +97,3 @@ normalize_logger_level(critical) -> critical;
normalize_logger_level(alert) -> alert;
normalize_logger_level(emergency) -> emergency;
normalize_logger_level(_) -> info.
-spec characters_to_binary_or_unknown(term()) -> binary().
characters_to_binary_or_unknown(Value) ->
type_conv:ensure_binary(Value, <<"unknown">>).
fluxer_gateway/src/gateway/gateway_build_info.erl
+3 -7
View File
@@ -9,13 +9,9 @@
-spec version() -> binary().
version() ->
case os:getenv("BUILD_VERSION") of
false ->
<<"dev">>;
"" ->
<<"dev">>;
Value ->
ensure_binary(Value)
case fluxer_gateway_config:env_value("BUILD_VERSION") of
undefined -> <<"dev">>;
Value -> ensure_binary(Value)
end.
-spec version_headers(#{binary() => binary()}) -> #{binary() => binary()}.
fluxer_gateway/src/gateway/gateway_timings.erl
+2 -2
View File
@@ -523,8 +523,8 @@ pod_name() ->
first_runtime_name([], Fallback) ->
Fallback;
first_runtime_name([Name | Rest], Fallback) ->
case os:getenv(Name) of
false -> first_runtime_name(Rest, Fallback);
case fluxer_gateway_config:env_value(Name) of
undefined -> first_runtime_name(Rest, Fallback);
Value -> normalize_key(Value)
end.
fluxer_gateway/test/fluxer_gateway_config_tests.erl
+26 -2
View File
@@ -117,14 +117,12 @@ presence_push_buffer_env_defaults_test() ->
env_only_http_runtime_config_test() ->
with_envs(
[
{"FLUXER_GATEWAY_SHUTDOWN_DRAIN_WAIT_MS", "1234"},
{"FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY", "42"},
{"FLUXER_GATEWAY_HTTP_FAILURE_THRESHOLD", "9"},
{"FLUXER_GATEWAY_HTTP_RECOVERY_TIMEOUT_MS", "6000"}
],
fun() ->
Config = fluxer_gateway_config:load(),
?assertEqual(1234, maps:get(shutdown_drain_wait_ms, Config)),
?assertEqual(42, maps:get(gateway_http_rpc_max_concurrency, Config)),
?assertEqual(9, maps:get(gateway_http_failure_threshold, Config)),
?assertEqual(6000, maps:get(gateway_http_recovery_timeout_ms, Config))
@@ -158,6 +156,32 @@ env_int_falls_back_to_the_default_for_an_empty_value_test() ->
?assertEqual(512, maps:get(gateway_http_rpc_max_concurrency, Config))
end).
blank_env_values_fall_back_to_the_defaults_test() ->
with_envs(
[
{"FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY", " "},
{"FLUXER_CLIENT_IP_HEADER_NAME", " "},
{"FLUXER_NATS_URL", "\t"},
{"FLUXER_GATEWAY_API_RPC_ENDPOINT", " "},
{"FLUXER_GATEWAY_LOGGER_LEVEL", " "},
{"FLUXER_GATEWAY_PUSH_ENABLED", " "}
],
fun() ->
Config = fluxer_gateway_config:load(),
?assertEqual(512, maps:get(gateway_http_rpc_max_concurrency, Config)),
?assertEqual(<<"x-forwarded-for">>, maps:get(client_ip_header, Config)),
?assertEqual("nats://nats:4222", maps:get(nats_core_url, Config)),
?assertEqual(undefined, maps:get(api_rpc_endpoint, Config)),
?assertEqual(info, maps:get(logger_level, Config)),
?assertEqual(true, maps:get(push_enabled, Config))
end
).
logger_level_env_test() ->
with_env("FLUXER_GATEWAY_LOGGER_LEVEL", "Debug", fun() ->
?assertEqual(debug, maps:get(logger_level, fluxer_gateway_config:load()))
end).
rpc_concurrency_key_defaults_test() ->
Config = fluxer_gateway_config:build_config(#{}),
?assertEqual(512, maps:get(gateway_nats_rpc_max_handlers, Config)),
fluxer_gifs/src/media_proxy.rs
+7 -8
View File
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use fluxer_svc::config::optional_env;
use url::Url;
#[derive(Clone)]
@@ -25,19 +26,17 @@ impl MediaProxyUrlBuilder {
}
pub fn from_env() -> anyhow::Result<Self> {
let endpoint = std::env::var("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT")
.or_else(|_| std::env::var("FLUXER_MEDIA_ENDPOINT"))
.unwrap_or_default();
if endpoint.trim().is_empty() {
let Some(endpoint) = optional_env("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT")
.or_else(|| optional_env("FLUXER_MEDIA_ENDPOINT"))
else {
anyhow::bail!(
"gifs shard requires FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT or FLUXER_MEDIA_ENDPOINT"
);
}
};
let secret_key = std::env::var("FLUXER_MEDIA_PROXY_SECRET_KEY").unwrap_or_default();
if secret_key.trim().is_empty() {
let Some(secret_key) = optional_env("FLUXER_MEDIA_PROXY_SECRET_KEY") else {
anyhow::bail!("gifs shard requires FLUXER_MEDIA_PROXY_SECRET_KEY");
}
};
let endpoint = fluxer_common::config::normalize_public_endpoint_from_env(
endpoint.trim_end_matches('/'),
fluxer_media_proxy/Cargo.toml
+1 -1
View File
@@ -49,7 +49,7 @@ thiserror = "2.0.20"
tokio = {version = "1.53.1", features = ["fs", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time"]}
tokio-util = {version = "0.7.19", features = ["io"]}
tracing = "0.1.44"
tracing-subscriber = {version = "0.3.23", features = ["env-filter", "fmt", "json"]}
tracing-subscriber = {version = "0.3.23", features = ["fmt", "json"]}
url = "2.5.8"
wyhash = "0.6.0"
fluxer_media_proxy/src/config/mod.rs
+14 -11
View File
@@ -8,7 +8,7 @@ use crate::constants;
use crate::secret::{SecretBytes, SecretString};
use http::HeaderValue;
use parse::{
EnvMap, decode_upload_relay_secret, default_native_transform_concurrency, non_empty,
EnvMap, decode_upload_relay_secret, default_native_transform_concurrency,
parse_allowed_origins, parse_attachment_url_secrets, parse_bool, parse_bucket_style, parse_f32,
parse_mode_env, parse_policy_mode, parse_storage_backend, parse_u16, parse_u64, parse_usize,
validate_read_endpoint,
@@ -128,7 +128,6 @@ pub struct UploadRelayConfig {
#[derive(Clone, Debug)]
pub struct Config {
pub node_env: String,
pub bind_host: String,
pub port: u16,
pub(crate) secret_key: SecretString,
@@ -174,7 +173,6 @@ impl Config {
})?;
Ok(Self {
node_env: env.get("NODE_ENV").unwrap_or("development").to_owned(),
bind_host: env
.get("FLUXER_MEDIA_PROXY_HOST")
.unwrap_or("0.0.0.0")
@@ -185,15 +183,16 @@ impl Config {
8080,
)?,
secret_key,
public_endpoint: non_empty(env.get("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT")).map(
|endpoint| {
public_endpoint: env
.get("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT")
.map(str::trim)
.map(|endpoint| {
fluxer_common::config::normalize_public_endpoint(
endpoint.trim_end_matches('/'),
&public_base_domain,
public_port,
)
},
),
}),
mode,
read_only: parse_bool(
"FLUXER_MEDIA_PROXY_READ_ONLY",
@@ -234,12 +233,16 @@ impl StorageConfig {
.get("FLUXER_S3_BUCKET_CDN")
.map(ToOwned::to_owned)
.unwrap_or_else(|| "cdn".to_owned());
let s3_read_endpoint = non_empty(env.get("FLUXER_S3_READ_ENDPOINT"));
let s3_read_endpoint = env
.get("FLUXER_S3_READ_ENDPOINT")
.map(|v| v.trim().to_owned());
if let Some(endpoint) = s3_read_endpoint.as_deref() {
validate_read_endpoint(endpoint)?;
}
let s3_read_bucket =
non_empty(env.get("FLUXER_S3_READ_BUCKET")).unwrap_or_else(|| bucket_cdn.clone());
let s3_read_bucket = env
.get("FLUXER_S3_READ_BUCKET")
.map(|v| v.trim().to_owned())
.unwrap_or_else(|| bucket_cdn.clone());
let s3_read_bucket_style = parse_bucket_style(env.get("FLUXER_S3_READ_BUCKET_STYLE"))?
.unwrap_or(if s3_force_path_style {
BucketStyle::Path
@@ -248,7 +251,7 @@ impl StorageConfig {
});
let s3_read_signed = parse_bool(
"FLUXER_S3_READ_SIGNED",
non_empty(env.get("FLUXER_S3_READ_SIGNED")).as_deref(),
env.get("FLUXER_S3_READ_SIGNED").map(str::trim),
)?
.unwrap_or(false);
Ok(Self {
fluxer_media_proxy/src/config/parse.rs
+3 -8
View File
@@ -26,6 +26,7 @@ impl EnvMap {
self.0
.iter()
.find_map(|(k, v)| (k == key).then_some(v.as_str()))
.filter(|v| !v.trim().is_empty())
}
}
@@ -150,14 +151,8 @@ fn is_origin_host(host: url::Host<&str>) -> bool {
}
}
pub(super) fn non_empty(raw: Option<&str>) -> Option<String> {
raw.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
pub(super) fn parse_bucket_style(raw: Option<&str>) -> anyhow::Result<Option<BucketStyle>> {
let Some(raw) = non_empty(raw) else {
let Some(raw) = raw.map(str::trim) else {
return Ok(None);
};
match raw.to_ascii_lowercase().as_str() {
@@ -210,7 +205,7 @@ pub(super) fn decode_upload_relay_secret(
raw: Option<&str>,
mode: DeploymentMode,
) -> anyhow::Result<SecretBytes> {
let Some(raw) = raw.map(str::trim).filter(|s| !s.is_empty()) else {
let Some(raw) = raw.map(str::trim) else {
anyhow::ensure!(
!mode.serves_upload_relay(),
"FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 is required in upload and relay modes"
fluxer_media_proxy/src/config/tests/attachment_signature.rs
+1 -1
View File
@@ -53,7 +53,7 @@ fn the_signature_mode_parses_every_variant_case_insensitively() {
#[test]
fn rejects_an_unknown_signature_mode() {
for raw in ["strict", "true", "1", "", "deny"] {
for raw in ["strict", "true", "1", "deny"] {
let err =
Config::load_from_iter(env_with(&[(MODE_KEY, raw), (SECRETS_KEY, FIRST)])).unwrap_err();
assert_eq!(
fluxer_media_proxy/src/config/tests/cors.rs
+1 -1
View File
@@ -45,7 +45,7 @@ fn cors_mode_parses_every_variant_case_insensitively() {
#[test]
fn rejects_an_unknown_cors_mode() {
for raw in ["strict", "true", "1", ""] {
for raw in ["strict", "true", "1"] {
let err = Config::load_from_iter(env_with(&[
("FLUXER_MEDIA_PROXY_CORS_MODE", raw),
(ORIGINS_KEY, "https://web.fluxer.app"),
fluxer_media_proxy/src/config/tests/mod.rs
+30 -2
View File
@@ -28,7 +28,6 @@ fn env_with<'a>(extra: &[(&'a str, &'a str)]) -> Vec<(&'a str, &'a str)> {
fn with_shared_runtime_env(release: &[(&str, &str)]) -> Vec<(String, String)> {
[
("NODE_ENV", "production"),
("FLUXER_ENV", "production"),
("FLUXER_MEDIA_PROXY_SECRET_KEY", "shared-runtime-secret"),
("FLUXER_S3_ENDPOINT", "https://ewr1.vultrobjects.com"),
@@ -68,6 +67,36 @@ fn default_config_matches_media_service() {
);
}
#[test]
fn blank_values_fall_back_to_the_defaults() {
let cfg = Config::load_from_iter(env_with(&[
("FLUXER_MEDIA_PROXY_MODE", ""),
("FLUXER_MEDIA_PROXY_HOST", " "),
("FLUXER_MEDIA_PROXY_PORT", ""),
("FLUXER_MEDIA_PROXY_READ_ONLY", ""),
("FLUXER_MEDIA_PROXY_STORAGE_BACKEND", ""),
("FLUXER_MEDIA_PROXY_NSFW_THRESHOLD", " "),
("FLUXER_MEDIA_PROXY_CORS_MODE", ""),
("FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE", " "),
("FLUXER_S3_REGION", ""),
("FLUXER_S3_BUCKET_CDN", ""),
]))
.unwrap();
assert_eq!(DeploymentMode::Mp, cfg.mode);
assert_eq!("0.0.0.0", cfg.bind_host);
assert_eq!(8080, cfg.port);
assert!(!cfg.read_only);
assert_eq!(StorageBackend::Local, cfg.storage.backend);
assert_eq!(0.85, cfg.media.nsfw_threshold);
assert_eq!(PolicyMode::Off, cfg.cors.mode);
assert_eq!(PolicyMode::Off, cfg.attachment_signature.mode);
assert_eq!("us-east-1", cfg.storage.s3_region);
assert_eq!("cdn", cfg.storage.bucket_cdn);
let err = Config::load_from_iter([("FLUXER_MEDIA_PROXY_SECRET_KEY", " ")]).unwrap_err();
assert!(err.to_string().contains("FLUXER_MEDIA_PROXY_SECRET_KEY"));
}
#[test]
fn canonical_media_proxy_env_overrides_apply() {
let cfg = Config::load_from_iter([
@@ -282,7 +311,6 @@ fn production_media_proxy_release_env_loads() {
]))
.unwrap();
assert_eq!("production", cfg.node_env);
assert_eq!(DeploymentMode::Mp, cfg.mode);
assert_eq!(PolicyMode::Enforce, cfg.cors.mode);
assert_eq!(
fluxer_media_proxy/src/healthcheck.rs
+5 -9
View File
@@ -2,24 +2,20 @@
use anyhow::Context as _;
use std::{
env,
net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr},
time::Duration,
};
pub async fn run() -> anyhow::Result<()> {
let addr = target(
env::var("FLUXER_MEDIA_PROXY_HOST").ok().as_deref(),
env::var("FLUXER_MEDIA_PROXY_PORT").ok().as_deref(),
fluxer_common::config::env_value("FLUXER_MEDIA_PROXY_HOST").as_deref(),
fluxer_common::config::env_value("FLUXER_MEDIA_PROXY_PORT").as_deref(),
)?;
probe(addr).await
}
fn target(host: Option<&str>, port: Option<&str>) -> anyhow::Result<SocketAddr> {
let host = host
.map(str::trim)
.filter(|host| !host.is_empty())
.unwrap_or("127.0.0.1");
let host = host.map(str::trim).unwrap_or("127.0.0.1");
let ip = host
.parse::<IpAddr>()
.with_context(|| format!("FLUXER_MEDIA_PROXY_HOST is not an IP address: {host}"))?;
@@ -28,7 +24,7 @@ fn target(host: Option<&str>, port: Option<&str>) -> anyhow::Result<SocketAddr>
IpAddr::V6(ip) if ip.is_unspecified() => IpAddr::V6(Ipv6Addr::LOCALHOST),
ip => ip,
};
let port = match port.map(str::trim).filter(|port| !port.is_empty()) {
let port = match port.map(str::trim) {
Some(port) => port
.parse::<u16>()
.with_context(|| format!("FLUXER_MEDIA_PROXY_PORT is not a port number: {port}"))?,
@@ -72,7 +68,7 @@ mod tests {
);
assert_eq!(
"[::1]:8080".parse::<SocketAddr>().unwrap(),
target(Some("::"), Some("")).unwrap()
target(Some("::"), None).unwrap()
);
assert!(target(Some("0.0.0.0"), Some("nope")).is_err());
}
fluxer_media_proxy/src/main.rs
+2 -2
View File
@@ -2,7 +2,7 @@
use clap::Parser;
use fluxer_media_proxy::{cli, healthcheck, run};
use tracing_subscriber::{EnvFilter, layer::SubscriberExt, util::SubscriberInitExt};
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
#[tokio::main(flavor = "multi_thread")]
async fn main() -> anyhow::Result<()> {
@@ -12,7 +12,7 @@ async fn main() -> anyhow::Result<()> {
}
tracing_subscriber::registry()
.with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")))
.with(fluxer_common::config::env_filter("info"))
.with(tracing_subscriber::fmt::layer().json())
.init();
fluxer_media_proxy/src/server/middleware.rs
+1 -4
View File
@@ -125,10 +125,7 @@ pub(in crate::server) fn build_version() -> &'static str {
static BUILD_VERSION: OnceLock<String> = OnceLock::new();
BUILD_VERSION
.get_or_init(|| {
std::env::var("BUILD_VERSION")
.ok()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| "dev".to_owned())
fluxer_common::config::env_value("BUILD_VERSION").unwrap_or_else(|| "dev".to_owned())
})
.as_str()
}
fluxer_media_proxy/src/storage/tests/mod.rs
-1
View File
@@ -36,7 +36,6 @@ pub(crate) type CapturedRequest = (Method, http::Uri, HeaderMap, Bytes);
fn test_config(root: &Path) -> Config {
Config {
node_env: "test".to_owned(),
bind_host: "127.0.0.1".to_owned(),
port: 0,
secret_key: SecretString::new("secret".to_owned()),
fluxer_push/Cargo.toml
-1
View File
@@ -34,5 +34,4 @@ sha2 = "0.11.0"
thiserror = "2.0.20"
tokio = { version = "1.53.1", features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] }
tracing = "0.1.44"
tracing-subscriber = { version = "0.3.23", features = ["env-filter", "fmt", "json"] }
url = "2.5.8"
fluxer_push/src/healthcheck.rs
+4 -8
View File
@@ -3,25 +3,21 @@
use crate::config::Mode;
use anyhow::Context as _;
use std::{
env,
net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr},
time::Duration,
};
pub async fn run(mode: Mode) -> anyhow::Result<()> {
let addr = target(
env::var("FLUXER_PUSH_SERVICE_HOST").ok().as_deref(),
env::var("FLUXER_PUSH_SERVICE_PORT").ok().as_deref(),
fluxer_svc::config::optional_env("FLUXER_PUSH_SERVICE_HOST").as_deref(),
fluxer_svc::config::optional_env("FLUXER_PUSH_SERVICE_PORT").as_deref(),
mode,
)?;
probe(addr).await
}
fn target(host: Option<&str>, port: Option<&str>, mode: Mode) -> anyhow::Result<SocketAddr> {
let host = host
.map(str::trim)
.filter(|host| !host.is_empty())
.unwrap_or("127.0.0.1");
let host = host.map(str::trim).unwrap_or("127.0.0.1");
let ip = host
.parse::<IpAddr>()
.with_context(|| format!("FLUXER_PUSH_SERVICE_HOST is not an IP address: {host}"))?;
@@ -30,7 +26,7 @@ fn target(host: Option<&str>, port: Option<&str>, mode: Mode) -> anyhow::Result<
IpAddr::V6(ip) if ip.is_unspecified() => IpAddr::V6(Ipv6Addr::LOCALHOST),
ip => ip,
};
let port = match port.map(str::trim).filter(|port| !port.is_empty()) {
let port = match port.map(str::trim) {
Some(port) => port
.parse::<u16>()
.with_context(|| format!("FLUXER_PUSH_SERVICE_PORT is not a port number: {port}"))?,
fluxer_push/src/main.rs
+1 -5
View File
@@ -2,7 +2,6 @@
use clap::Parser;
use fluxer_push::{cli, healthcheck, run};
use tracing_subscriber::{EnvFilter, layer::SubscriberExt, util::SubscriberInitExt};
#[tokio::main(flavor = "multi_thread")]
async fn main() -> anyhow::Result<()> {
@@ -11,10 +10,7 @@ async fn main() -> anyhow::Result<()> {
return healthcheck::run(args.mode).await;
}
tracing_subscriber::registry()
.with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")))
.with(tracing_subscriber::fmt::layer().json())
.init();
fluxer_svc::init_tracing();
let cfg = cli::load_config(&args)?;
run(cfg).await
fluxer_svc/src/config.rs
+28 -9
View File
@@ -20,7 +20,6 @@ pub struct ServiceConfig {
pub nats_auth_token: Option<String>,
pub cache_max_entries: u64,
pub cache_ttl: Duration,
pub cache_hard_ttl: Duration,
pub max_concurrent_requests: usize,
pub scylla_hosts: Vec<String>,
pub scylla_keyspace: String,
@@ -114,12 +113,6 @@ impl ServiceConfig {
.transpose()?
.unwrap_or(30_000);
let cache_hard_ttl_ms = optional_from(&get, "FLUXER_SVC_CACHE_HARD_TTL_MS")
.map(|v| v.parse::<u64>())
.transpose()?
.unwrap_or(600_000)
.max(cache_ttl_ms);
let cassandra_port = optional_from(&get, "FLUXER_CASSANDRA_PORT")
.map(|v| v.parse::<u16>())
.transpose()?
@@ -175,7 +168,6 @@ impl ServiceConfig {
.transpose()?
.unwrap_or(100_000),
cache_ttl: Duration::from_millis(cache_ttl_ms),
cache_hard_ttl: Duration::from_millis(cache_hard_ttl_ms),
max_concurrent_requests,
scylla_hosts,
scylla_keyspace: optional_from(&get, "FLUXER_CASSANDRA_KEYSPACE")
@@ -218,7 +210,7 @@ fn optional_from<F>(get: &F, name: &str) -> Option<String>
where
F: Fn(&str) -> Option<String>,
{
get(name).filter(|v| !v.is_empty())
get(name).filter(|v| !v.trim().is_empty())
}
pub fn parse_hosts(hosts: &str) -> Vec<String> {
@@ -359,6 +351,33 @@ mod tests {
assert!(cfg.postgres_prepared_statements);
}
#[test]
fn blank_values_fall_back_to_the_defaults() {
let cfg = config_from_pairs(&[
("FLUXER_SVC_NAME", "messages"),
("FLUXER_SVC_MODE", ""),
("FLUXER_SVC_PORT", " "),
("FLUXER_SVC_MAX_CONCURRENT_REQUESTS", ""),
("FLUXER_NATS_AUTH_TOKEN", " "),
("FLUXER_POSTGRES_HOST", ""),
("FLUXER_POSTGRES_PASSWORD", " "),
("FLUXER_POSTGRES_MAX_CONNECTIONS", ""),
("FLUXER_POSTGRES_PREPARED_STATEMENTS", " "),
]);
assert_eq!(Mode::Router, cfg.mode);
assert_eq!(8090, cfg.listen_addr.port());
assert_eq!(
MESSAGES_MAX_CONCURRENT_REQUESTS,
cfg.max_concurrent_requests
);
assert_eq!(None, cfg.nats_auth_token);
assert_eq!("127.0.0.1", cfg.postgres_host);
assert_eq!(Some("fluxer".to_owned()), cfg.postgres_password);
assert_eq!(20, cfg.postgres_max_connections);
assert!(cfg.postgres_prepared_statements);
}
#[test]
fn reads_postgres_database_env() {
let cfg = config_from_pairs(&[
fluxer_svc/src/lib.rs
+3 -1
View File
@@ -20,7 +20,9 @@ pub fn init_tracing() {
tracing_subscriber::fmt()
.json()
.with_env_filter(
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")),
config::optional_env("RUST_LOG")
.and_then(|filter| EnvFilter::try_new(filter).ok())
.unwrap_or_else(|| EnvFilter::new("info")),
)
.try_init()
.ok();
fluxer_svc/src/router.rs
-1
View File
@@ -942,7 +942,6 @@ mod tests {
nats_auth_token: None,
cache_max_entries: 100,
cache_ttl: Duration::from_secs(30),
cache_hard_ttl: Duration::from_secs(600),
max_concurrent_requests,
scylla_hosts: Vec::new(),
scylla_keyspace: "fluxer".to_owned(),
fluxer_svc/src/server.rs
+1 -4
View File
@@ -81,10 +81,7 @@ fn build_version() -> &'static str {
static BUILD_VERSION: OnceLock<String> = OnceLock::new();
BUILD_VERSION
.get_or_init(|| {
std::env::var("BUILD_VERSION")
.ok()
.filter(|v| !v.trim().is_empty())
.unwrap_or_else(|| "dev".to_owned())
crate::config::optional_env("BUILD_VERSION").unwrap_or_else(|| "dev".to_owned())
})
.as_str()
}
fluxer_svc/src/shard.rs
-1
View File
@@ -436,7 +436,6 @@ mod tests {
nats_auth_token: None,
cache_max_entries: 100,
cache_ttl: Duration::from_secs(30),
cache_hard_ttl: Duration::from_secs(600),
max_concurrent_requests,
scylla_hosts: Vec::new(),
scylla_keyspace: "fluxer".to_owned(),
fluxer_unfurl/src/resolvers/klipy.rs
+2 -8
View File
@@ -4,6 +4,7 @@ use super::{ResolveContext, Resolver, ResolverResult};
use crate::http_fetch;
use crate::media_proxy::{MediaMetadata, embed_media_flags};
use crate::types::{EmbedMedia, EmbedProvider, MessageEmbed};
use fluxer_svc::config::optional_env;
use std::future::Future;
use std::pin::Pin;
use std::time::Duration;
@@ -136,14 +137,7 @@ fn klipy_resource(kind: &str) -> &'static str {
}
fn klipy_api_key() -> Option<String> {
std::env::var("FLUXER_KLIPY_API_KEY")
.ok()
.filter(|key| !key.is_empty())
.or_else(|| {
std::env::var("KLIPY_API_KEY")
.ok()
.filter(|key| !key.is_empty())
})
optional_env("FLUXER_KLIPY_API_KEY").or_else(|| optional_env("KLIPY_API_KEY"))
}
async fn resolve_media_via_api(
fluxer_unfurl/src/resolvers/youtube.rs
+2 -9
View File
@@ -5,8 +5,8 @@ use crate::http_fetch;
use crate::media_proxy::embed_media_flags;
use crate::text_limits;
use crate::types::{EmbedAuthor, EmbedMedia, EmbedProvider, MessageEmbed};
use fluxer_svc::config::optional_env;
use serde::Deserialize;
use std::env;
use std::future::Future;
use std::pin::Pin;
use std::time::Duration;
@@ -252,14 +252,7 @@ struct YouTubeThumbnail {
}
fn youtube_api_key() -> Option<String> {
env::var("FLUXER_YOUTUBE_API_KEY")
.ok()
.filter(|key| !key.is_empty())
.or_else(|| {
env::var("YOUTUBE_API_KEY")
.ok()
.filter(|key| !key.is_empty())
})
optional_env("FLUXER_YOUTUBE_API_KEY").or_else(|| optional_env("YOUTUBE_API_KEY"))
}
fn build_youtube_api_url(video_id: &str, api_key: &str) -> anyhow::Result<Url> {
fluxer_unfurl/src/shard_impl.rs
+6 -11
View File
@@ -8,6 +8,7 @@ use crate::embed_normalizer::normalize_embeds;
use crate::media_proxy::MediaProxyClient;
use crate::resolvers::{self, ResolveContext, ResolverResult};
use crate::types::{InvalidatedResponse, NsfwMode, UnfurlRequest, UnfurlResponse, UnfurlResult};
use fluxer_svc::config::optional_env;
use fluxer_svc::shard::ShardService;
use moka::future::Cache;
use std::sync::Arc;
@@ -32,19 +33,13 @@ impl UnfurlShard {
let resolvers = resolvers::build_resolver_chain();
let media_proxy_endpoint = std::env::var("FLUXER_MEDIA_PROXY_ENDPOINT")
.ok()
.filter(|v| !v.is_empty());
let media_proxy_secret = std::env::var("FLUXER_MEDIA_PROXY_SECRET_KEY")
.ok()
.filter(|v| !v.is_empty());
let media_proxy_public_endpoint = std::env::var("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT")
.ok()
.filter(|v| !v.is_empty())
let media_proxy_endpoint = optional_env("FLUXER_MEDIA_PROXY_ENDPOINT");
let media_proxy_secret = optional_env("FLUXER_MEDIA_PROXY_SECRET_KEY");
let media_proxy_public_endpoint = optional_env("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT")
.map(|v| fluxer_common::config::normalize_public_endpoint_from_env(&v));
let static_cdn_endpoint = fluxer_common::config::normalize_public_endpoint_from_env(
&std::env::var("FLUXER_UNFURL_STATIC_CDN_ENDPOINT")
.or_else(|_| std::env::var("FLUXER_STATIC_CDN_ENDPOINT"))
&optional_env("FLUXER_UNFURL_STATIC_CDN_ENDPOINT")
.or_else(|| optional_env("FLUXER_STATIC_CDN_ENDPOINT"))
.unwrap_or_default(),
);
let (media_proxy_endpoint, media_proxy_secret) = match (
packages/config/src/ConfigLoader.ts
+8 -48
View File
@@ -2,7 +2,7 @@
import {createECDH} from 'node:crypto';
import {isConfigObject} from '@fluxer/config/src/config_loader/ConfigObject';
import {buildNamedFluxerEnvOverrides} from '@fluxer/config/src/config_loader/EnvironmentOverrides';
import {buildNamedFluxerEnvOverrides, readEnvValue} from '@fluxer/config/src/config_loader/EnvironmentOverrides';
import {
buildUrl,
type DerivedEndpoints,
@@ -31,9 +31,7 @@ function defaultConfig(): MasterConfig {
base_domain: '',
public_origin: '',
public_scheme: 'http',
internal_scheme: 'http',
public_port: 8088,
internal_port: 8088,
static_cdn_domain: '',
invite_domain: '',
gift_domain: '',
@@ -46,18 +44,13 @@ function defaultConfig(): MasterConfig {
media: '',
static_cdn: '',
admin: '',
docs: '',
marketing: '',
invite: '',
gift: '',
},
internal: {
kv: 'redis://localhost:6379/0',
kv_provider: 'redis',
kv_mode: 'standalone',
kv_cluster_nodes: [],
kv_cluster_nat_map: {},
api: 'http://127.0.0.1:8080',
media_proxy: 'http://127.0.0.1:8082',
},
database: {
@@ -109,18 +102,6 @@ function defaultConfig(): MasterConfig {
presigned_harvest_downloads_enabled: true,
unfurl_ignored_hosts: [],
app_origin_aliases: [],
embeds: {
oembed_html_enabled: false,
oembed_html_allow_untrusted_on_self_hosted: false,
oembed_html_allowed_hosts: [],
cache_default_ttl_seconds: 86_400,
cache_max_ttl_seconds: 604_800,
cache_min_ttl_seconds: 300,
cache_respect_remote_ttl: true,
},
content_moderation: {
nsfw_threshold: 0.7,
},
storage_change_feed: {
enabled: false,
stream: 'STORAGE_CHANGES',
@@ -132,10 +113,7 @@ function defaultConfig(): MasterConfig {
auth_token: '',
},
media_proxy: {
host: '0.0.0.0',
port: 8082,
secret_key: '',
mode: 'upload',
upload_relay: {
endpoint: 'http://localhost:8088/media',
secret_base64: '',
@@ -148,19 +126,12 @@ function defaultConfig(): MasterConfig {
},
},
gateway: {
port: 8771,
rpc_auth_token: '',
},
admin: {
port: 3020,
base_path: '/admin',
secret_key_base: '',
oauth_client_secret: '',
},
app_proxy: {
port: 8773,
assets_dir: 'fluxer_app/dist',
},
},
auth: {
sudo_mode_secret: '',
@@ -200,7 +171,6 @@ function defaultConfig(): MasterConfig {
api_secret: '',
url: '',
internal_url: '',
webhook_url: '',
},
search: {
engine: 'elasticsearch',
@@ -253,10 +223,6 @@ function defaultConfig(): MasterConfig {
enabled: false,
apps: [],
},
fcm: {
enabled: false,
apps: [],
},
},
},
instance: {
@@ -314,13 +280,10 @@ function requireString(value: string | undefined, envName: string): void {
}
}
function validateUploadRelaySecret(value: string, mode: string): void {
function validateUploadRelaySecret(value: string): void {
const trimmed = value.trim();
if (trimmed.length === 0) {
if (mode === 'upload') {
throw new Error('FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 is required in upload mode');
}
return;
throw new Error('FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 is required');
}
if (!/^[A-Za-z0-9+/]+={0,2}$/u.test(trimmed)) {
throw new Error('FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 must be base64');
@@ -537,9 +500,7 @@ function validatePublicEndpoints(endpoints: DerivedEndpoints): void {
function normalizeConfig(config: MasterConfig): MasterConfig {
assertOneOf(config.env, ['development', 'production', 'test'], 'FLUXER_ENV');
assertOneOf(config.domain.public_scheme, ['http', 'https'], 'FLUXER_PUBLIC_SCHEME');
assertOneOf(config.domain.internal_scheme, ['http', 'https'], 'FLUXER_INTERNAL_SCHEME');
assertOneOf(config.database.backend, ['postgres', 'cassandra'], 'FLUXER_DATABASE_BACKEND');
assertOneOf(config.internal.kv_provider, ['redis'], 'FLUXER_KV_PROVIDER');
assertOneOf(config.internal.kv_mode, ['standalone', 'cluster'], 'FLUXER_KV_MODE');
assertOneOf(config.integrations.email.provider, ['smtp', 'none'], 'FLUXER_EMAIL_PROVIDER');
assertOneOf(config.integrations.search.engine, ['elasticsearch', 'meilisearch'], 'FLUXER_SEARCH_ENGINE');
@@ -563,7 +524,7 @@ function normalizeConfig(config: MasterConfig): MasterConfig {
requireString(config.s3?.access_key_id, 'FLUXER_S3_ACCESS_KEY_ID');
requireString(config.s3?.secret_access_key, 'FLUXER_S3_SECRET_ACCESS_KEY');
requireString(config.services.media_proxy.secret_key, 'FLUXER_MEDIA_PROXY_SECRET_KEY');
validateUploadRelaySecret(config.services.media_proxy.upload_relay.secret_base64, config.services.media_proxy.mode);
validateUploadRelaySecret(config.services.media_proxy.upload_relay.secret_base64);
validateAttachmentUrlSecrets(config.services.media_proxy.attachment_urls.secrets_base64);
requireString(config.services.admin.secret_key_base, 'FLUXER_ADMIN_SECRET_KEY_BASE');
requireString(config.services.admin.oauth_client_secret, 'FLUXER_ADMIN_OAUTH_CLIENT_SECRET');
@@ -622,10 +583,6 @@ function applyPublicPort(config: MasterConfig, endpoints: DerivedEndpoints): Mas
endpoint: normalize(config.services.media_proxy.upload_relay.endpoint),
},
},
gateway: {
...config.services.gateway,
media_proxy_endpoint: normalizeOptional(config.services.gateway.media_proxy_endpoint),
},
},
auth: {
...config.auth,
@@ -703,7 +660,10 @@ export async function loadConfig(): Promise<MasterConfig> {
const endpoints = {...derived, ...(normalized.endpoint_overrides ?? {})};
validatePublicEndpoints(endpoints);
const withPublicPort = applyPublicPort(normalized, endpoints);
normalizePasskeys(withPublicPort, process.env.FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS === undefined);
normalizePasskeys(
withPublicPort,
readEnvValue(process.env, 'FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS') === undefined,
);
cachedConfig = withPublicPort;
return cachedConfig;
}
packages/config/src/EndpointDerivation.ts
-7
View File
@@ -1,13 +1,9 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
const DOCS_ENDPOINT = 'https://fluxer.dev';
export interface DomainConfig {
base_domain: string;
public_scheme: 'http' | 'https';
internal_scheme: 'http' | 'https';
public_port?: number;
internal_port?: number;
static_cdn_domain?: string;
invite_domain?: string;
gift_domain?: string;
@@ -29,7 +25,6 @@ export interface DerivedEndpoints {
media: string;
static_cdn: string;
admin: string;
docs: string;
marketing: string;
invite: string;
gift: string;
@@ -129,7 +124,6 @@ export function deriveDomain(
| 'media'
| 'static_cdn'
| 'admin'
| 'docs'
| 'marketing'
| 'invite'
| 'gift',
@@ -160,7 +154,6 @@ export function deriveEndpointsFromDomain(config: DomainConfig): DerivedEndpoint
? buildUrl('https', deriveDomain('static_cdn', config), undefined)
: buildUrl(public_scheme, deriveDomain('static_cdn', config), public_port),
admin: buildUrl(public_scheme, deriveDomain('admin', config), public_port, '/admin'),
docs: DOCS_ENDPOINT,
marketing: buildUrl(public_scheme, deriveDomain('marketing', config), public_port, '/marketing'),
invite: buildUrl(public_scheme, deriveDomain('invite', config), public_port, '/invite'),
gift: buildUrl(public_scheme, deriveDomain('gift', config), public_port, '/gift'),
packages/config/src/MasterConfig.ts
-49
View File
@@ -26,9 +26,7 @@ export interface MasterConfig {
base_domain: string;
public_origin: string;
public_scheme: PublicScheme;
internal_scheme: PublicScheme;
public_port: number;
internal_port: number;
static_cdn_domain: string;
invite_domain: string;
gift_domain: string;
@@ -37,12 +35,7 @@ export interface MasterConfig {
endpoints: DerivedEndpoints;
internal: {
kv: string;
kv_provider: 'redis';
kv_mode: 'standalone' | 'cluster';
kv_cluster_nodes: Array<{host: string; port: number}>;
kv_cluster_nat_map: Record<string, {host: string; port: number}>;
api: string;
gateway?: string;
media_proxy: string;
};
database: {
@@ -95,18 +88,6 @@ export interface MasterConfig {
presigned_harvest_downloads_enabled: boolean;
unfurl_ignored_hosts: Array<string>;
app_origin_aliases: Array<string>;
embeds: {
oembed_html_enabled: boolean;
oembed_html_allow_untrusted_on_self_hosted: boolean;
oembed_html_allowed_hosts: Array<string>;
cache_default_ttl_seconds: number;
cache_max_ttl_seconds: number;
cache_min_ttl_seconds: number;
cache_respect_remote_ttl: boolean;
};
content_moderation?: {
nsfw_threshold?: number;
};
worker?: {
mode?: 'all_lanes' | 'single_lane' | 'single_task';
lane?: 'realtime' | 'unfurl' | 'lifecycle' | 'batch';
@@ -131,10 +112,7 @@ export interface MasterConfig {
auth_token?: string;
};
media_proxy: {
host: string;
port: number;
secret_key: string;
mode: string;
upload_relay: {
endpoint: string;
secret_base64: string;
@@ -147,21 +125,12 @@ export interface MasterConfig {
};
};
gateway: {
port: number;
rpc_auth_token?: string;
media_proxy_endpoint?: string;
api_rpc_endpoint?: string;
};
admin: {
port: number;
base_path: string;
secret_key_base: string;
oauth_client_secret: string;
};
app_proxy: {
port: number;
assets_dir: string;
};
};
auth: {
sudo_mode_secret: string;
@@ -213,7 +182,6 @@ export interface MasterConfig {
api_secret: string;
url: string;
internal_url: string;
webhook_url: string;
default_region?: {
id: string;
name: string;
@@ -280,27 +248,10 @@ export interface MasterConfig {
key_id?: string;
private_key?: string;
private_key_path?: string;
default_environment?: 'production' | 'development';
apps?: Array<{
app_id?: string;
topic?: string;
environment?: 'production' | 'development';
project_id?: string;
}>;
};
fcm: {
enabled: boolean;
project_id?: string;
client_email?: string;
private_key?: string;
private_key_path?: string;
service_account_json_path?: string;
token_uri?: string;
apps?: Array<{
app_id?: string;
topic?: string;
environment?: 'production' | 'development';
project_id?: string;
}>;
};
};
packages/config/src/__tests__/ConfigLoader.test.ts
+26 -22
View File
@@ -22,8 +22,6 @@ const MINIMAL_ENV: Record<string, string> = {
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64: 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=',
FLUXER_ADMIN_SECRET_KEY_BASE: 'test-admin-secret',
FLUXER_ADMIN_OAUTH_CLIENT_SECRET: 'test-admin-oauth-secret',
FLUXER_APP_PROXY_PORT: '8773',
FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: 'http://127.0.0.1:8088/media',
FLUXER_GATEWAY_RPC_AUTH_TOKEN: 'test-gateway-token',
FLUXER_SUDO_MODE_SECRET: 'test-sudo-secret',
FLUXER_CONNECTION_INITIATION_SECRET: 'test-connection-secret',
@@ -243,9 +241,9 @@ describe('ConfigLoader', () => {
},
);
test('rejects an empty client API endpoint override', async () => {
test('an empty client API endpoint override falls back to the derived endpoint', async () => {
stubMinimalEnv({FLUXER_API_CLIENT_ENDPOINT: ''});
await expect(loadConfig()).rejects.toThrow('FLUXER_API_CLIENT_ENDPOINT is required');
expect((await loadConfig()).endpoints.api_client).toBe('http://localhost:8088/api');
});
test('defaults the passkey relying party to the deployment domain', async () => {
@@ -260,17 +258,36 @@ describe('ConfigLoader', () => {
expect(config.auth.passkeys.rp_id).toBe('chat.example.com');
});
test('uses only the app origin when the operator clears the default list', async () => {
test('a blank origin list keeps the built-in origins', async () => {
stubMinimalEnv({
FLUXER_BASE_DOMAIN: 'chat.example.com',
FLUXER_PUBLIC_SCHEME: 'https',
FLUXER_PUBLIC_PORT: '443',
FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS: '',
FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS: ' ',
});
const config = await loadConfig();
expect(config.auth.passkeys.additional_allowed_origins).toEqual(['https://chat.example.com']);
expect(config.auth.passkeys.additional_allowed_origins).toContain('https://web.fluxer.app');
expect(config.auth.passkeys.additional_allowed_origins).toContain('https://chat.example.com');
});
test('blank values fall back to the code defaults', async () => {
stubMinimalEnv({
FLUXER_API_PORT: '',
FLUXER_EMAIL_FROM_NAME: '',
FLUXER_KV_URL: ' ',
FLUXER_S3_FORCE_PATH_STYLE: '',
FLUXER_API_STORAGE_CHANGE_FEED_SKIP_BUCKETS: '',
});
const config = await loadConfig();
expect(config.services.api.port).toBe(8080);
expect(config.integrations.email.from_name).toBe('Fluxer');
expect(config.internal.kv).toBe('redis://localhost:6379/0');
expect(config.s3?.force_path_style).toBe(false);
expect(config.services.api.storage_change_feed?.skip_buckets).toBeUndefined();
});
test('keeps explicit passkey relying party values', async () => {
@@ -665,13 +682,11 @@ describe('ConfigLoader', () => {
expect((await loadConfig()).services.media_proxy.upload_relay.max_body_bytes).toBe(524_288_000);
});
test('rejects a missing upload relay secret in upload mode', async () => {
test('rejects a missing upload relay secret', async () => {
stubMinimalEnv();
vi.stubEnv('FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64', '');
await expect(loadConfig()).rejects.toThrow(
'FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 is required in upload mode',
);
await expect(loadConfig()).rejects.toThrow('FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 is required');
});
test('rejects a non-base64 upload relay secret', async () => {
@@ -686,15 +701,6 @@ describe('ConfigLoader', () => {
);
});
test('leaves the upload relay secret optional outside upload mode', async () => {
stubMinimalEnv({FLUXER_MEDIA_PROXY_MODE: 'mp'});
vi.stubEnv('FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64', '');
const config = await loadConfig();
expect(config.services.media_proxy.upload_relay.secret_base64).toBe('');
});
test('rejects a VAPID public key that is not a 65-byte uncompressed point', async () => {
const {privateKey} = generateVapidPair();
stubMinimalEnv({
@@ -753,7 +759,6 @@ describe('ConfigLoader', () => {
test('inserts the public port into every other public url the config carries', async () => {
stubMinimalEnv({
FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: 'http://localhost/media',
FLUXER_S3_PUBLIC_ENDPOINT: 'http://localhost/s3',
FLUXER_EMAIL_APP_BASE_URL: 'http://localhost',
FLUXER_AUTH_BLUESKY_CLIENT_URI: 'http://localhost',
@@ -764,7 +769,6 @@ describe('ConfigLoader', () => {
const config = await loadConfig();
expect(config.services.gateway.media_proxy_endpoint).toBe('http://localhost:8088/media');
expect(config.s3?.presigned_url_base).toBe('http://localhost:8088/s3');
expect(config.integrations.email.app_base_url).toBe('http://localhost:8088');
expect(config.auth.bluesky.client_uri).toBe('http://localhost:8088');
packages/config/src/__tests__/EndpointDerivation.test.ts
+2 -33
View File
@@ -34,7 +34,6 @@ describe('deriveDomain', () => {
const baseConfig: DomainConfig = {
base_domain: 'fluxer.dev',
public_scheme: 'https',
internal_scheme: 'http',
};
test.each([
'api',
@@ -44,7 +43,6 @@ describe('deriveDomain', () => {
'media',
'static_cdn',
'admin',
'docs',
'marketing',
'invite',
'gift',
@@ -72,9 +70,7 @@ const endpointScenarios: Array<EndpointScenario> = [
config: {
base_domain: 'localhost',
public_scheme: 'http',
internal_scheme: 'http',
public_port: 8088,
internal_port: 8088,
},
expected: {
api: 'http://localhost:8088/api',
@@ -84,7 +80,6 @@ const endpointScenarios: Array<EndpointScenario> = [
media: 'http://localhost:8088/media',
static_cdn: 'http://localhost:8088',
admin: 'http://localhost:8088/admin',
docs: 'https://fluxer.dev',
marketing: 'http://localhost:8088/marketing',
invite: 'http://localhost:8088/invite',
gift: 'http://localhost:8088/gift',
@@ -95,9 +90,7 @@ const endpointScenarios: Array<EndpointScenario> = [
config: {
base_domain: 'fluxer.app',
public_scheme: 'https',
internal_scheme: 'http',
public_port: 443,
internal_port: 8080,
},
expected: {
api: 'https://fluxer.app/api',
@@ -107,7 +100,6 @@ const endpointScenarios: Array<EndpointScenario> = [
media: 'https://fluxer.app/media',
static_cdn: 'https://fluxer.app',
admin: 'https://fluxer.app/admin',
docs: 'https://fluxer.dev',
marketing: 'https://fluxer.app/marketing',
invite: 'https://fluxer.app/invite',
gift: 'https://fluxer.app/gift',
@@ -118,9 +110,7 @@ const endpointScenarios: Array<EndpointScenario> = [
config: {
base_domain: 'staging.fluxer.dev',
public_scheme: 'https',
internal_scheme: 'http',
public_port: 8443,
internal_port: 8080,
},
expected: {
api: 'https://staging.fluxer.dev:8443/api',
@@ -130,7 +120,6 @@ const endpointScenarios: Array<EndpointScenario> = [
media: 'https://staging.fluxer.dev:8443/media',
static_cdn: 'https://staging.fluxer.dev:8443',
admin: 'https://staging.fluxer.dev:8443/admin',
docs: 'https://fluxer.dev',
marketing: 'https://staging.fluxer.dev:8443/marketing',
invite: 'https://staging.fluxer.dev:8443/invite',
gift: 'https://staging.fluxer.dev:8443/gift',
@@ -141,7 +130,6 @@ const endpointScenarios: Array<EndpointScenario> = [
config: {
base_domain: 'fluxer.app',
public_scheme: 'https',
internal_scheme: 'http',
public_port: 443,
static_cdn_domain: 'cdn.fluxer.app',
},
@@ -153,7 +141,6 @@ const endpointScenarios: Array<EndpointScenario> = [
media: 'https://fluxer.app/media',
static_cdn: 'https://cdn.fluxer.app',
admin: 'https://fluxer.app/admin',
docs: 'https://fluxer.dev',
marketing: 'https://fluxer.app/marketing',
invite: 'https://fluxer.app/invite',
gift: 'https://fluxer.app/gift',
@@ -164,7 +151,6 @@ const endpointScenarios: Array<EndpointScenario> = [
config: {
base_domain: 'fluxer.app',
public_scheme: 'https',
internal_scheme: 'http',
public_port: 443,
invite_domain: 'fluxer.gg',
gift_domain: 'fluxer.gift',
@@ -177,7 +163,6 @@ const endpointScenarios: Array<EndpointScenario> = [
media: 'https://fluxer.app/media',
static_cdn: 'https://fluxer.app',
admin: 'https://fluxer.app/admin',
docs: 'https://fluxer.dev',
marketing: 'https://fluxer.app/marketing',
invite: 'https://fluxer.gg/invite',
gift: 'https://fluxer.gift/gift',
@@ -188,7 +173,6 @@ const endpointScenarios: Array<EndpointScenario> = [
config: {
base_domain: 'canary.fluxer.app',
public_scheme: 'https',
internal_scheme: 'http',
public_port: 443,
static_cdn_domain: 'cdn-canary.fluxer.app',
},
@@ -200,7 +184,6 @@ const endpointScenarios: Array<EndpointScenario> = [
media: 'https://canary.fluxer.app/media',
static_cdn: 'https://cdn-canary.fluxer.app',
admin: 'https://canary.fluxer.app/admin',
docs: 'https://fluxer.dev',
marketing: 'https://canary.fluxer.app/marketing',
invite: 'https://canary.fluxer.app/invite',
gift: 'https://canary.fluxer.app/gift',
@@ -211,7 +194,6 @@ const endpointScenarios: Array<EndpointScenario> = [
config: {
base_domain: 'example.com',
public_scheme: 'http',
internal_scheme: 'http',
public_port: 80,
},
expected: {
@@ -222,7 +204,6 @@ const endpointScenarios: Array<EndpointScenario> = [
media: 'http://example.com/media',
static_cdn: 'http://example.com',
admin: 'http://example.com/admin',
docs: 'https://fluxer.dev',
marketing: 'http://example.com/marketing',
invite: 'http://example.com/invite',
gift: 'http://example.com/gift',
@@ -233,7 +214,6 @@ const endpointScenarios: Array<EndpointScenario> = [
config: {
base_domain: 'example.com',
public_scheme: 'https',
internal_scheme: 'http',
},
expected: {
api: 'https://example.com/api',
@@ -243,7 +223,6 @@ const endpointScenarios: Array<EndpointScenario> = [
media: 'https://example.com/media',
static_cdn: 'https://example.com',
admin: 'https://example.com/admin',
docs: 'https://fluxer.dev',
marketing: 'https://example.com/marketing',
invite: 'https://example.com/invite',
gift: 'https://example.com/gift',
@@ -254,7 +233,6 @@ const endpointScenarios: Array<EndpointScenario> = [
config: {
base_domain: '127.0.0.1',
public_scheme: 'http',
internal_scheme: 'http',
public_port: 8088,
},
expected: {
@@ -265,7 +243,6 @@ const endpointScenarios: Array<EndpointScenario> = [
media: 'http://127.0.0.1:8088/media',
static_cdn: 'http://127.0.0.1:8088',
admin: 'http://127.0.0.1:8088/admin',
docs: 'https://fluxer.dev',
marketing: 'http://127.0.0.1:8088/marketing',
invite: 'http://127.0.0.1:8088/invite',
gift: 'http://127.0.0.1:8088/gift',
@@ -276,7 +253,6 @@ const endpointScenarios: Array<EndpointScenario> = [
config: {
base_domain: 'localhost',
public_scheme: 'http',
internal_scheme: 'http',
public_port: 8088,
static_cdn_domain: 'cdn.example.com',
},
@@ -288,7 +264,6 @@ const endpointScenarios: Array<EndpointScenario> = [
media: 'http://localhost:8088/media',
static_cdn: 'https://cdn.example.com',
admin: 'http://localhost:8088/admin',
docs: 'https://fluxer.dev',
marketing: 'http://localhost:8088/marketing',
invite: 'http://localhost:8088/invite',
gift: 'http://localhost:8088/gift',
@@ -469,10 +444,7 @@ describe('endpoints derived from a public origin', () => {
test('an origin with a non-standard port ports every derived endpoint', () => {
const origin = parsePublicOrigin('https://chat.example.com:29080');
assert.ok(origin);
const endpoints = deriveEndpointsFromDomain({
...origin,
internal_scheme: 'http',
});
const endpoints = deriveEndpointsFromDomain(origin);
expect(endpoints.api_client).toBe('https://chat.example.com:29080/api');
expect(endpoints.app).toBe('https://chat.example.com:29080');
expect(endpoints.gateway).toBe('wss://chat.example.com:29080/gateway');
@@ -481,10 +453,7 @@ describe('endpoints derived from a public origin', () => {
test('an origin written with an explicit :443 derives portless endpoints', () => {
const origin = parsePublicOrigin('https://chat.example.com:443');
assert.ok(origin);
const endpoints = deriveEndpointsFromDomain({
...origin,
internal_scheme: 'http',
});
const endpoints = deriveEndpointsFromDomain(origin);
expect(endpoints.admin).toBe('https://chat.example.com/admin');
expect(endpoints.app).toBe('https://chat.example.com');
expect(endpoints.gateway).toBe('wss://chat.example.com/gateway');
packages/config/src/__tests__/EnvironmentOverrides.test.ts
+38 -51
View File
@@ -1,45 +1,8 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
buildNamedFluxerEnvOverrides,
parseEnvValue,
setNestedValue,
} from '@fluxer/config/src/config_loader/EnvironmentOverrides';
import {buildNamedFluxerEnvOverrides, setNestedValue} from '@fluxer/config/src/config_loader/EnvironmentOverrides';
import {describe, expect, test} from 'vitest';
describe('parseEnvValue', () => {
test('parses boolean true', () => {
expect(parseEnvValue('true')).toBe(true);
expect(parseEnvValue(' true ')).toBe(true);
});
test('parses boolean false', () => {
expect(parseEnvValue('false')).toBe(false);
expect(parseEnvValue(' false ')).toBe(false);
});
test('parses integers', () => {
expect(parseEnvValue('42')).toBe(42);
expect(parseEnvValue('-7')).toBe(-7);
expect(parseEnvValue('0')).toBe(0);
});
test('parses floats', () => {
expect(parseEnvValue('3.14')).toBe(3.14);
expect(parseEnvValue('-0.5')).toBe(-0.5);
});
test('parses JSON objects', () => {
expect(parseEnvValue('{"key": "value"}')).toEqual({key: 'value'});
});
test('parses JSON arrays', () => {
expect(parseEnvValue('[1, 2, 3]')).toEqual([1, 2, 3]);
});
test('rejects invalid JSON-like values', () => {
expect(() => parseEnvValue('{not json}')).toThrow('must be valid JSON');
});
test('returns raw string for plain strings', () => {
expect(parseEnvValue('hello')).toBe('hello');
expect(parseEnvValue('localhost')).toBe('localhost');
});
});
describe('setNestedValue', () => {
test('sets a top-level key', () => {
const target: Record<string, unknown> = {};
@@ -81,14 +44,13 @@ describe('setNestedValue', () => {
});
describe('buildNamedFluxerEnvOverrides', () => {
test('builds canonical split env overrides and preserves empty strings', () => {
test('builds canonical split env overrides', () => {
const overrides = buildNamedFluxerEnvOverrides({
FLUXER_BASE_DOMAIN: 'canonical.example',
FLUXER_API_ENDPOINT: 'https://canonical.example/api',
FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS: 'https://a.example, https://b.example',
FLUXER_S3_FORCE_PATH_STYLE: 'true',
FLUXER_AUTH_BLUESKY_KEYS: '[{"kid":"key-1","private_key_path":"/etc/fluxer/keys/bluesky.pem"}]',
FLUXER_ADMIN_BASE_PATH: '',
FLUXER_STRIPE_PRICE_MONTHLY_USD: 'price_monthly_usd',
});
@@ -100,17 +62,46 @@ describe('buildNamedFluxerEnvOverrides', () => {
bluesky: {keys: [{kid: 'key-1', private_key_path: '/etc/fluxer/keys/bluesky.pem'}]},
},
s3: {force_path_style: true},
services: {
admin: {base_path: ''},
},
integrations: {stripe: {prices: {monthly_usd: 'price_monthly_usd'}}},
});
});
test('maps the internal scheme and KV provider names', () => {
expect(buildNamedFluxerEnvOverrides({FLUXER_INTERNAL_SCHEME: 'https', FLUXER_KV_PROVIDER: 'redis'})).toMatchObject({
domain: {internal_scheme: 'https'},
internal: {kv_provider: 'redis'},
test.each(['', ' ', '\t\n'])('treats %j as unset for every value type', (blank) => {
expect(
buildNamedFluxerEnvOverrides({
FLUXER_BASE_DOMAIN: blank,
FLUXER_API_PORT: blank,
FLUXER_S3_FORCE_PATH_STYLE: blank,
FLUXER_API_IP_BAN_EXEMPT_IPS: blank,
FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS: blank,
FLUXER_LIVEKIT_DEFAULT_REGION: blank,
FLUXER_AUTH_BLUESKY_KEYS: blank,
FLUXER_EMAIL_FROM_NAME: blank,
}),
).toEqual({});
});
test('a blank canonical name falls through to its alias', () => {
expect(
buildNamedFluxerEnvOverrides({
FLUXER_NATS_URL: '',
FLUXER_NATS_CORE_URL: 'nats://alias',
FLUXER_IPINFO_API_KEY: ' ',
FLUXER_RISK_IPINFO_API_KEY: 'alias-key',
}),
).toMatchObject({
services: {nats: {core_url: 'nats://alias'}},
integrations: {ipinfo: {api_key: 'alias-key'}},
});
});
test('a blank alias is unset too', () => {
expect(buildNamedFluxerEnvOverrides({FLUXER_NATS_URL: '', FLUXER_NATS_CORE_URL: ' '})).toEqual({});
});
test('none clears the storage change feed skip list', () => {
expect(buildNamedFluxerEnvOverrides({FLUXER_API_STORAGE_CHANGE_FEED_SKIP_BUCKETS: ' None '})).toEqual({
services: {api: {storage_change_feed: {skip_buckets: []}}},
});
});
@@ -120,10 +111,6 @@ describe('buildNamedFluxerEnvOverrides', () => {
);
});
test('leaves the default in place for a blank integer override', () => {
expect(buildNamedFluxerEnvOverrides({FLUXER_API_PORT: ''})).toEqual({});
});
test('the canonical name wins over its alias regardless of declaration order', () => {
expect(
buildNamedFluxerEnvOverrides({
packages/config/src/__tests__/SelfHostingCompose.ts
-2
View File
@@ -42,8 +42,6 @@ export function serviceEnvironment(name: string): Record<string, string> {
return value === undefined ? {} : environment(value, `services.${name}.environment`);
}
export const sharedEnvironment = environment(compose['x-fluxer-env'], 'x-fluxer-env');
export function serviceList(name: string, key: string): Array<string> {
const value = composeService(name)[key];
assert.ok(Array.isArray(value), `services.${name}.${key} must be a list`);
packages/config/src/__tests__/SelfHostingComposeEnvironment.test.ts
+82 -20
View File
@@ -1,28 +1,90 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {serviceEnvironment, serviceNames, sharedEnvironment} from '@fluxer/config/src/__tests__/SelfHostingCompose';
import {serviceEnvironment, serviceNames} from '@fluxer/config/src/__tests__/SelfHostingCompose';
import {NAMED_FLUXER_ENV_NAMES} from '@fluxer/config/src/config_loader/EnvironmentOverrides';
import {describe, expect, test} from 'vitest';
describe('the shipped compose stack wires every service it starts', () => {
test('no service sizes a pool for a connection it cannot make', () => {
const pooledServices = serviceNames.filter((name) => 'FLUXER_POSTGRES_MAX_CONNECTIONS' in serviceEnvironment(name));
expect(pooledServices.length).toBeGreaterThan(0);
for (const name of pooledServices) {
expect(serviceEnvironment(name), name).toMatchObject({
FLUXER_DATABASE_BACKEND: 'postgres',
FLUXER_POSTGRES_HOST: expect.stringMatching(/\S/u),
FLUXER_POSTGRES_PORT: expect.stringMatching(/\S/u),
FLUXER_POSTGRES_DATABASE: expect.stringMatching(/\S/u),
FLUXER_POSTGRES_USERNAME: expect.stringMatching(/\S/u),
FLUXER_POSTGRES_PASSWORD: expect.stringMatching(/\S/u),
});
}
const API_SETTINGS_NOT_FORWARDED: Record<string, string> = {
FLUXER_CASSANDRA_HOSTS: 'the stack runs Postgres only',
FLUXER_CASSANDRA_PORT: 'the stack runs Postgres only',
FLUXER_CASSANDRA_KEYSPACE: 'the stack runs Postgres only',
FLUXER_CASSANDRA_LOCAL_DC: 'the stack runs Postgres only',
FLUXER_CASSANDRA_USERNAME: 'the stack runs Postgres only',
FLUXER_CASSANDRA_PASSWORD: 'the stack runs Postgres only',
FLUXER_API_WORKER_MODE: 'the one worker container runs every lane',
FLUXER_API_WORKER_LANE: 'the one worker container runs every lane',
FLUXER_API_WORKER_TASK: 'the one worker container runs every lane',
FLUXER_API_WORKER_ENABLE_CRON_SCHEDULER: 'the one worker container hosts cron',
FLUXER_RELAX_REGISTRATION_RATE_LIMITS: 'test and development only',
FLUXER_DISABLE_RATE_LIMITS: 'test and development only',
FLUXER_TEST_MODE_ENABLED: 'test and development only',
FLUXER_TEST_HARNESS_TOKEN: 'test and development only',
FLUXER_VALIDATE_RESPONSES: 'test and development only',
...Object.fromEntries(
['MONTHLY', 'YEARLY', 'GIFT_1_MONTH', 'GIFT_1_YEAR'].flatMap((slot) =>
['USD', 'EUR', 'BRL', 'DKK', 'INR', 'NOK', 'PLN', 'SEK', 'TRY'].map((currency) => [
`FLUXER_STRIPE_PRICE_${slot}_${currency}`,
'FLUXER_STRIPE_PRICES or the dashboard sets prices',
]),
),
),
};
const INPUT_NAMES: Record<string, string> = {
FLUXER_BASE_DOMAIN: 'FLUXER_DOMAIN',
FLUXER_POSTGRES_PASSWORD: 'POSTGRES_PASSWORD',
FLUXER_SEARCH_API_KEY: 'MEILI_MASTER_KEY',
FLUXER_S3_ACCESS_KEY_ID: 'FLUXER_S3_ACCESS_KEY',
FLUXER_S3_SECRET_ACCESS_KEY: 'FLUXER_S3_SECRET_KEY',
FLUXER_LIVEKIT_API_KEY: 'LIVEKIT_API_KEY',
FLUXER_LIVEKIT_API_SECRET: 'LIVEKIT_API_SECRET',
POSTGRES_DB: 'FLUXER_POSTGRES_DATABASE',
POSTGRES_USER: 'FLUXER_POSTGRES_USERNAME',
MEILI_ENV: 'FLUXER_MEILISEARCH_ENV',
MEILI_NO_ANALYTICS: 'FLUXER_MEILISEARCH_NO_ANALYTICS',
MEILI_MAX_INDEXING_MEMORY: 'FLUXER_MEILISEARCH_MAX_INDEXING_MEMORY',
GOMEMLIMIT: 'FLUXER_SEAWEEDFS_GOMEMLIMIT',
LIVEKIT_KEYS: 'LIVEKIT_API_KEY',
NODE_EXTRA_CA_CERTS: 'FLUXER_NODE_EXTRA_CA_CERTS',
};
const OWN_POOL_SIZES = new Set(['api', 'worker', 'users-shard', 'messages-shard']);
const INTERPOLATION = /^\$\{([A-Z][A-Z0-9_]*)/u;
function inputName(service: string, key: string): string {
if (key === 'FLUXER_POSTGRES_MAX_CONNECTIONS' && OWN_POOL_SIZES.has(service)) {
return `FLUXER_${service.toUpperCase().replace('-', '_')}_POSTGRES_MAX_CONNECTIONS`;
}
return INPUT_NAMES[key] ?? key;
}
function forwardedEntries(): Array<{service: string; key: string; name: string}> {
return serviceNames.flatMap((service) =>
Object.entries(serviceEnvironment(service)).flatMap(([key, value]) => {
const match = INTERPOLATION.exec(value.trim());
if (match == null) {
return [];
}
return [{service, key, name: match[1]}];
}),
);
}
describe('the shipped compose stack forwards settings from .env', () => {
test('the api is handed every setting its config loader reads', () => {
const api = serviceEnvironment('api');
const missing = NAMED_FLUXER_ENV_NAMES.filter((name) => !(name in api) && !(name in API_SETTINGS_NOT_FORWARDED));
expect(missing).toEqual([]);
expect(Object.keys(API_SETTINGS_NOT_FORWARDED).filter((name) => !NAMED_FLUXER_ENV_NAMES.includes(name))).toEqual(
[],
);
});
test('the shared block sets the client-IP trust the merged services read', () => {
expect(sharedEnvironment).toMatchObject({
FLUXER_TRUST_CLIENT_IP_HEADER: `\${FLUXER_TRUST_CLIENT_IP_HEADER:-true}`,
FLUXER_CLIENT_IP_HEADER_NAME: `\${FLUXER_CLIENT_IP_HEADER_NAME:-x-forwarded-for}`,
});
test('every forwarded setting is read from .env under the name the operator sets', () => {
const renamed = forwardedEntries()
.filter(({service, key, name}) => name !== inputName(service, key))
.map(({service, key, name}) => `${service}.${key} reads ${name}`);
expect(renamed).toEqual([]);
});
});
packages/config/src/__tests__/SelfHostingObjectStoreAuth.test.ts
+1 -1
View File
@@ -33,7 +33,7 @@ describe('the shipped object store checks the credentials the stack sends', () =
});
test('media-proxy signs its reads, which the store now refuses to serve unsigned', () => {
expect(serviceEnvironment('media-proxy').FLUXER_S3_READ_SIGNED).toBe('true');
expect(serviceEnvironment('media-proxy').FLUXER_S3_READ_SIGNED).toBe(`\${FLUXER_S3_READ_SIGNED:-true}`);
});
test('every service that reaches the store waits for the identity to exist', () => {
packages/config/src/config_loader/EnvironmentOverrides.ts
+19 -126
View File
@@ -17,7 +17,6 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
FLUXER_BASE_DOMAIN: {path: ['domain', 'base_domain']},
FLUXER_PUBLIC_ORIGIN: {path: ['domain', 'public_origin']},
FLUXER_PUBLIC_SCHEME: {path: ['domain', 'public_scheme']},
FLUXER_INTERNAL_SCHEME: {path: ['domain', 'internal_scheme']},
FLUXER_PUBLIC_PORT: {path: ['domain', 'public_port'], parse: parseInteger},
FLUXER_STATIC_CDN_DOMAIN: {path: ['domain', 'static_cdn_domain']},
FLUXER_INVITE_DOMAIN: {path: ['domain', 'invite_domain']},
@@ -29,7 +28,6 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
FLUXER_MEDIA_ENDPOINT: {path: ['endpoint_overrides', 'media']},
FLUXER_STATIC_CDN_ENDPOINT: {path: ['endpoint_overrides', 'static_cdn']},
FLUXER_ADMIN_ENDPOINT: {path: ['endpoint_overrides', 'admin']},
FLUXER_DOCS_ENDPOINT: {path: ['endpoint_overrides', 'docs']},
FLUXER_MARKETING_ENDPOINT: {path: ['endpoint_overrides', 'marketing']},
FLUXER_INVITE_ENDPOINT: {path: ['endpoint_overrides', 'invite']},
FLUXER_GIFT_ENDPOINT: {path: ['endpoint_overrides', 'gift']},
@@ -54,10 +52,7 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
FLUXER_POSTGRES_PREPARED_STATEMENTS: {path: ['database', 'postgres', 'prepared_statements'], parse: parseBoolean},
FLUXER_DATABASE_BACKEND: {path: ['database', 'backend']},
FLUXER_KV_URL: {path: ['internal', 'kv']},
FLUXER_KV_PROVIDER: {path: ['internal', 'kv_provider']},
FLUXER_KV_MODE: {path: ['internal', 'kv_mode']},
FLUXER_INTERNAL_API_ENDPOINT: {path: ['internal', 'api']},
FLUXER_INTERNAL_GATEWAY_ENDPOINT: {path: ['internal', 'gateway']},
FLUXER_INTERNAL_MEDIA_PROXY_ENDPOINT: {path: ['internal', 'media_proxy']},
FLUXER_S3_ENDPOINT: {path: ['s3', 'endpoint']},
FLUXER_S3_PUBLIC_ENDPOINT: {path: ['s3', 'presigned_url_base']},
@@ -104,46 +99,11 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
FLUXER_API_STORAGE_CHANGE_FEED_STREAM: {path: ['services', 'api', 'storage_change_feed', 'stream']},
FLUXER_API_STORAGE_CHANGE_FEED_SKIP_BUCKETS: {
path: ['services', 'api', 'storage_change_feed', 'skip_buckets'],
parse: parseCsv,
parse: parseBucketList,
},
FLUXER_API_UNFURL_IGNORED_HOSTS: {path: ['services', 'api', 'unfurl_ignored_hosts'], parse: parseCsv},
FLUXER_APP_ORIGIN_ALIASES: {path: ['services', 'api', 'app_origin_aliases'], parse: parseCsv},
FLUXER_API_EMBEDS_OEMBED_HTML_ENABLED: {
path: ['services', 'api', 'embeds', 'oembed_html_enabled'],
parse: parseBoolean,
},
FLUXER_API_EMBEDS_OEMBED_HTML_ALLOW_UNTRUSTED_ON_SELF_HOSTED: {
path: ['services', 'api', 'embeds', 'oembed_html_allow_untrusted_on_self_hosted'],
parse: parseBoolean,
},
FLUXER_API_EMBEDS_OEMBED_HTML_ALLOWED_HOSTS: {
path: ['services', 'api', 'embeds', 'oembed_html_allowed_hosts'],
parse: parseCsv,
},
FLUXER_API_EMBEDS_CACHE_DEFAULT_TTL_SECONDS: {
path: ['services', 'api', 'embeds', 'cache_default_ttl_seconds'],
parse: parseInteger,
},
FLUXER_API_EMBEDS_CACHE_MAX_TTL_SECONDS: {
path: ['services', 'api', 'embeds', 'cache_max_ttl_seconds'],
parse: parseInteger,
},
FLUXER_API_EMBEDS_CACHE_MIN_TTL_SECONDS: {
path: ['services', 'api', 'embeds', 'cache_min_ttl_seconds'],
parse: parseInteger,
},
FLUXER_API_EMBEDS_CACHE_RESPECT_REMOTE_TTL: {
path: ['services', 'api', 'embeds', 'cache_respect_remote_ttl'],
parse: parseBoolean,
},
FLUXER_API_CONTENT_MODERATION_NSFW_THRESHOLD: {
path: ['services', 'api', 'content_moderation', 'nsfw_threshold'],
parse: parseEnvValue,
},
FLUXER_MEDIA_PROXY_HOST: {path: ['services', 'media_proxy', 'host']},
FLUXER_MEDIA_PROXY_PORT: {path: ['services', 'media_proxy', 'port'], parse: parseInteger},
FLUXER_MEDIA_PROXY_SECRET_KEY: {path: ['services', 'media_proxy', 'secret_key']},
FLUXER_MEDIA_PROXY_MODE: {path: ['services', 'media_proxy', 'mode']},
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT: {path: ['services', 'media_proxy', 'upload_relay', 'endpoint']},
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64: {path: ['services', 'media_proxy', 'upload_relay', 'secret_base64']},
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_MAX_BODY_BYTES: {
@@ -162,47 +122,9 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
path: ['services', 'media_proxy', 'attachment_urls', 'secrets_base64'],
parse: parseCsv,
},
FLUXER_ADMIN_PORT: {path: ['services', 'admin', 'port'], parse: parseInteger},
FLUXER_ADMIN_BASE_PATH: {path: ['services', 'admin', 'base_path']},
FLUXER_ADMIN_SECRET_KEY_BASE: {path: ['services', 'admin', 'secret_key_base']},
FLUXER_ADMIN_OAUTH_CLIENT_SECRET: {path: ['services', 'admin', 'oauth_client_secret']},
FLUXER_APP_PROXY_PORT: {path: ['services', 'app_proxy', 'port'], parse: parseInteger},
FLUXER_STATIC_DIR: {path: ['services', 'app_proxy', 'assets_dir']},
FLUXER_GATEWAY_PORT: {path: ['services', 'gateway', 'port'], parse: parseInteger},
FLUXER_GATEWAY_ROLE: {path: ['services', 'gateway', 'gateway_role']},
FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: {path: ['services', 'gateway', 'media_proxy_endpoint']},
FLUXER_GATEWAY_API_RPC_ENDPOINT: {path: ['services', 'gateway', 'api_rpc_endpoint']},
FLUXER_GATEWAY_RPC_AUTH_TOKEN: {path: ['services', 'gateway', 'rpc_auth_token']},
FLUXER_GATEWAY_LOGGER_LEVEL: {path: ['services', 'gateway', 'logger_level']},
FLUXER_GATEWAY_HTTP_FAILURE_THRESHOLD: {
path: ['services', 'gateway', 'gateway_http_failure_threshold'],
parse: parseInteger,
},
FLUXER_GATEWAY_HTTP_RECOVERY_TIMEOUT_MS: {
path: ['services', 'gateway', 'gateway_http_recovery_timeout_ms'],
parse: parseInteger,
},
FLUXER_GATEWAY_HTTP_RPC_MAX_CONCURRENCY: {
path: ['services', 'gateway', 'gateway_http_rpc_max_concurrency'],
parse: parseInteger,
},
FLUXER_GATEWAY_NATS_RPC_MAX_HANDLERS: {
path: ['services', 'gateway', 'gateway_nats_rpc_max_handlers'],
parse: parseInteger,
},
FLUXER_GATEWAY_SHUTDOWN_DRAIN_WAIT_MS: {
path: ['services', 'gateway', 'shutdown_drain_wait_ms'],
parse: parseInteger,
},
FLUXER_GATEWAY_CLUSTER_ENABLED: {path: ['services', 'gateway', 'cluster_enabled'], parse: parseEnvValue},
FLUXER_GATEWAY_CLUSTER_DISCOVERY_DNS_NAME: {path: ['services', 'gateway', 'cluster_discovery_dns_name']},
FLUXER_GATEWAY_CLUSTER_DISCOVERY_NODE_BASENAME: {
path: ['services', 'gateway', 'cluster_discovery_node_basename'],
},
FLUXER_GATEWAY_CLUSTER_DISCOVERY_POLL_INTERVAL_MS: {
path: ['services', 'gateway', 'cluster_discovery_poll_interval_ms'],
parse: parseInteger,
},
FLUXER_SUDO_MODE_SECRET: {path: ['auth', 'sudo_mode_secret']},
FLUXER_CONNECTION_INITIATION_SECRET: {path: ['auth', 'connection_initiation_secret']},
FLUXER_SSO_ALLOW_PRIVATE_ADDRESSES: {path: ['auth', 'sso_allow_private_addresses'], parse: parseBoolean},
@@ -238,7 +160,6 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
FLUXER_LIVEKIT_API_SECRET: {path: ['integrations', 'voice', 'api_secret']},
FLUXER_LIVEKIT_URL: {path: ['integrations', 'voice', 'url']},
FLUXER_LIVEKIT_INTERNAL_URL: {path: ['integrations', 'voice', 'internal_url']},
FLUXER_LIVEKIT_WEBHOOK_URL: {path: ['integrations', 'voice', 'webhook_url']},
FLUXER_LIVEKIT_DEFAULT_REGION: {path: ['integrations', 'voice', 'default_region'], parse: parseJsonObject},
FLUXER_SEARCH_ENGINE: {path: ['integrations', 'search', 'engine']},
FLUXER_SEARCH_URL: {path: ['integrations', 'search', 'url']},
@@ -272,10 +193,6 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
FLUXER_STRIPE_PRICE_YEARLY_PLN: {path: ['integrations', 'stripe', 'prices', 'yearly_pln']},
FLUXER_STRIPE_PRICE_YEARLY_SEK: {path: ['integrations', 'stripe', 'prices', 'yearly_sek']},
FLUXER_STRIPE_PRICE_YEARLY_TRY: {path: ['integrations', 'stripe', 'prices', 'yearly_try']},
FLUXER_STRIPE_PRICE_VISIONARY_USD: {path: ['integrations', 'stripe', 'prices', 'visionary_usd']},
FLUXER_STRIPE_PRICE_VISIONARY_EUR: {path: ['integrations', 'stripe', 'prices', 'visionary_eur']},
FLUXER_STRIPE_PRICE_GIFT_VISIONARY_USD: {path: ['integrations', 'stripe', 'prices', 'gift_visionary_usd']},
FLUXER_STRIPE_PRICE_GIFT_VISIONARY_EUR: {path: ['integrations', 'stripe', 'prices', 'gift_visionary_eur']},
FLUXER_STRIPE_PRICE_GIFT_1_MONTH_USD: {path: ['integrations', 'stripe', 'prices', 'gift_1_month_usd']},
FLUXER_STRIPE_PRICE_GIFT_1_MONTH_EUR: {path: ['integrations', 'stripe', 'prices', 'gift_1_month_eur']},
FLUXER_STRIPE_PRICE_GIFT_1_MONTH_SEK: {path: ['integrations', 'stripe', 'prices', 'gift_1_month_sek']},
@@ -323,16 +240,7 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
FLUXER_PUSH_APNS_KEY_ID: {path: ['integrations', 'push', 'apns', 'key_id']},
FLUXER_PUSH_APNS_PRIVATE_KEY: {path: ['integrations', 'push', 'apns', 'private_key']},
FLUXER_PUSH_APNS_PRIVATE_KEY_PATH: {path: ['integrations', 'push', 'apns', 'private_key_path']},
FLUXER_PUSH_APNS_DEFAULT_ENVIRONMENT: {path: ['integrations', 'push', 'apns', 'default_environment']},
FLUXER_PUSH_APNS_APPS: {path: ['integrations', 'push', 'apns', 'apps'], parse: parseJsonArray},
FLUXER_PUSH_FCM_ENABLED: {path: ['integrations', 'push', 'fcm', 'enabled'], parse: parseBoolean},
FLUXER_PUSH_FCM_PROJECT_ID: {path: ['integrations', 'push', 'fcm', 'project_id']},
FLUXER_PUSH_FCM_CLIENT_EMAIL: {path: ['integrations', 'push', 'fcm', 'client_email']},
FLUXER_PUSH_FCM_PRIVATE_KEY: {path: ['integrations', 'push', 'fcm', 'private_key']},
FLUXER_PUSH_FCM_PRIVATE_KEY_PATH: {path: ['integrations', 'push', 'fcm', 'private_key_path']},
FLUXER_PUSH_FCM_SERVICE_ACCOUNT_JSON_PATH: {path: ['integrations', 'push', 'fcm', 'service_account_json_path']},
FLUXER_PUSH_FCM_TOKEN_URI: {path: ['integrations', 'push', 'fcm', 'token_uri']},
FLUXER_PUSH_FCM_APPS: {path: ['integrations', 'push', 'fcm', 'apps'], parse: parseJsonArray},
FLUXER_SELF_HOSTED: {path: ['instance', 'self_hosted'], parse: parseBoolean},
FLUXER_AUTO_JOIN_INVITE_CODE: {path: ['instance', 'auto_join_invite_code']},
FLUXER_VISIONARIES_GUILD_ID: {path: ['instance', 'visionaries_guild_id']},
@@ -394,50 +302,27 @@ function parseBoolean(raw: string): boolean {
}
function parseJson(raw: string): unknown {
const trimmed = raw.trim();
if (trimmed.length === 0) return undefined;
try {
return JSON.parse(trimmed);
return JSON.parse(raw);
} catch {
throw new Error('must be valid JSON');
}
}
function parseJsonObject(raw: string): ConfigObject | undefined {
function parseJsonObject(raw: string): ConfigObject {
const value = parseJson(raw);
if (value === undefined || isConfigObject(value)) return value;
if (isConfigObject(value)) return value;
throw new Error('must be a JSON object');
}
function parseJsonArray(raw: string): Array<unknown> | undefined {
function parseJsonArray(raw: string): Array<unknown> {
const value = parseJson(raw);
if (value === undefined || Array.isArray(value)) return value;
if (Array.isArray(value)) return value;
throw new Error('must be a JSON array');
}
export function parseEnvValue(raw: string): unknown {
function parseInteger(raw: string): number {
const trimmed = raw.trim();
const lower = trimmed.toLowerCase();
if (lower === 'true' || lower === 'false') return parseBoolean(trimmed);
if (/^-?\d+$/.test(trimmed)) {
return parseInteger(trimmed);
}
if (/^-?\d+\.\d+$/.test(trimmed)) {
const value = Number(trimmed);
if (!Number.isFinite(value)) throw new Error('must be a finite number');
return value;
}
if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
return parseJson(trimmed);
}
return raw;
}
function parseInteger(raw: string): number | undefined {
const trimmed = raw.trim();
if (trimmed.length === 0) {
return undefined;
}
if (!/^-?\d+$/.test(trimmed)) {
throw new Error(`must be an integer, got ${JSON.stringify(raw)}`);
}
@@ -453,6 +338,10 @@ function parseCsv(raw: string): Array<string> {
.filter((part) => part.length > 0);
}
function parseBucketList(raw: string): Array<string> {
return raw.trim().toLowerCase() === 'none' ? [] : parseCsv(raw);
}
function parsePasskeyOrigins(raw: string): Array<string> {
if (/\p{Cc}/u.test(raw)) {
throw new Error('must not contain control characters');
@@ -486,11 +375,18 @@ const NAMED_FLUXER_ENV_ALIASES: Record<string, string | undefined> = {
FLUXER_IPINFO_API_KEY: 'FLUXER_RISK_IPINFO_API_KEY',
};
export const NAMED_FLUXER_ENV_NAMES = Object.keys(NAMED_FLUXER_ENV_OVERRIDES);
export function readEnvValue(env: NodeJS.ProcessEnv, name: string): string | undefined {
const value = env[name];
return value === undefined || value.trim().length === 0 ? undefined : value;
}
export function buildNamedFluxerEnvOverrides(env: NodeJS.ProcessEnv): ConfigObject {
const overrides: ConfigObject = {};
for (const [envKey, mapping] of Object.entries(NAMED_FLUXER_ENV_OVERRIDES)) {
const alias = NAMED_FLUXER_ENV_ALIASES[envKey];
const raw = env[envKey] ?? (alias === undefined ? undefined : env[alias]);
const raw = readEnvValue(env, envKey) ?? (alias === undefined ? undefined : readEnvValue(env, alias));
if (raw === undefined) {
continue;
}
@@ -500,9 +396,6 @@ export function buildNamedFluxerEnvOverrides(env: NodeJS.ProcessEnv): ConfigObje
} catch (error) {
throw new Error(`${envKey} ${error instanceof Error ? error.message : String(error)}`);
}
if (parsed === undefined) {
continue;
}
setNestedValue(overrides, mapping.path, parsed);
}
return overrides;
packages/constants/src/Timeouts.ts
-1
View File
@@ -1,5 +1,4 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
export const DEFAULT_KV_TIMEOUT_MS = 5000;
export const DEFAULT_HTTP_WORKER_TIMEOUT_MS = 30000;
export const DEFAULT_SEARCH_CLIENT_TIMEOUT_MS = 30000;
packages/errors/package.json
-1
View File
@@ -12,7 +12,6 @@
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@fluxer/config": "workspace:*",
"@fluxer/constants": "workspace:*",
"@fluxer/hono_types": "workspace:*",
"@fluxer/i18n": "workspace:*",
packages/errors/src/domains/core/MaxBookmarksError.ts
+3 -12
View File
@@ -1,24 +1,15 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {getConfig} from '@fluxer/config/src/ConfigLoader';
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
import {BadRequestError} from '@fluxer/errors/src/domains/core/BadRequestError';
export class MaxBookmarksError extends BadRequestError {
constructor(params: {
maxBookmarks: number;
isPremium?: boolean;
}) {
const {maxBookmarks, isPremium} = params;
const config = getConfig();
const selfHosted = 'self_hosted' in config ? config.self_hosted : false;
constructor(params: {maxBookmarks: number}) {
const {maxBookmarks} = params;
super({
code: APIErrorCodes.MAX_BOOKMARKS,
messageVariables: {count: maxBookmarks},
data: {
max_bookmarks: maxBookmarks,
...(selfHosted || isPremium === undefined ? {} : {is_premium: isPremium}),
},
data: {max_bookmarks: maxBookmarks},
});
}
}
packages/logger/src/Logger.ts
+1 -1
View File
@@ -20,7 +20,7 @@ function isPinoLevel(value: string): value is pino.Level {
}
function resolveEnvironment(options: LoggerOptions): string {
return options.environment ?? process.env.FLUXER_ENV ?? 'production';
return options.environment ?? (process.env.FLUXER_ENV?.trim() || 'production');
}
function resolveLevel(options: LoggerOptions, isDev: boolean): pino.Level {
pnpm-lock.yaml
Generated
-9
View File
@@ -1026,15 +1026,9 @@ importers:
fluxer_api/pkgs/worker:
dependencies:
'@fluxer/constants':
specifier: workspace:*
version: link:../../../packages/constants
'@fluxer/logger':
specifier: workspace:*
version: link:../../../packages/logger
itty-time:
specifier: 'catalog:'
version: 2.0.2
devDependencies:
'@types/node':
specifier: 'catalog:'
@@ -1969,9 +1963,6 @@ importers:
packages/errors:
dependencies:
'@fluxer/config':
specifier: workspace:*
version: link:../config
'@fluxer/constants':
specifier: workspace:*
version: link:../constants
tools/dev/src/dev.rs
-1
View File
@@ -542,7 +542,6 @@ pub fn task_table() -> Result<BTreeMap<&'static str, DevTask>> {
Some(format!("{public_url}/media")),
),
("FLUXER_STATIC_CDN_ENDPOINT".to_owned(), Some(public_url)),
("RELEASE_CHANNEL".to_owned(), Some("canary".to_owned())),
],
});
insert(DevTask {
tools/dev/src/tunnel.rs
-4
View File
@@ -98,10 +98,6 @@ pub fn public_url_env(public_url: &str) -> Result<Vec<(String, String)>> {
"FLUXER_LIVEKIT_URL".to_owned(),
format!("{gateway_base}/livekit"),
),
(
"FLUXER_LIVEKIT_WEBHOOK_URL".to_owned(),
format!("{base}/api/webhooks/livekit"),
),
(
"FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT".to_owned(),
format!("{base}/media"),