From 1b22d14f3d5a30f94429ffbfd326596383eb897d Mon Sep 17 00:00:00 2001 From: Hampus Date: Tue, 8 Sep 2026 23:36:52 +0200 Subject: [PATCH] feat(self-hosting): run postgres or the object store outside (#2620) --- deploy/self-hosting/.env.example | 15 ++++++++++ deploy/self-hosting/docker-compose.yml | 18 ++++++------ .../src/content/docs/operator/upgrading.mdx | 19 ++++++++++++ fluxer_docs/src/installer/install.ps1 | 29 +++++++++++++++++-- fluxer_docs/src/installer/install.sh | 25 ++++++++++++++++ 5 files changed, 95 insertions(+), 11 deletions(-) diff --git a/deploy/self-hosting/.env.example b/deploy/self-hosting/.env.example index 298884654..ec85e094f 100644 --- a/deploy/self-hosting/.env.example +++ b/deploy/self-hosting/.env.example @@ -117,6 +117,21 @@ FLUXER_IMAGE_TAG=v1 POSTGRES_PASSWORD=CHANGE_ME MEILI_MASTER_KEY=CHANGE_ME +# The stack ships its own Postgres and its own object store, and points at both +# by service name. Set these to run either one outside the stack. Leave them +# unset and the bundled services are used. Taking a service out of the stack +# means an upgrade skips the backup step that reaches into it, and backing that +# store up belongs to whoever runs it. +#FLUXER_POSTGRES_HOST=db.example.com +#FLUXER_POSTGRES_PORT=5432 +#FLUXER_POSTGRES_DATABASE=fluxer +#FLUXER_POSTGRES_USERNAME=fluxer +#FLUXER_POSTGRES_SSL=true +#FLUXER_S3_ENDPOINT=https://s3.eu-central-1.amazonaws.com +#FLUXER_S3_PUBLIC_ENDPOINT=https://cdn.example.com +#FLUXER_S3_REGION=eu-central-1 +#FLUXER_S3_FORCE_PATH_STYLE=false + FLUXER_S3_ACCESS_KEY=fluxer FLUXER_S3_SECRET_KEY=CHANGE_ME diff --git a/deploy/self-hosting/docker-compose.yml b/deploy/self-hosting/docker-compose.yml index 66498cdf8..437ba7999 100644 --- a/deploy/self-hosting/docker-compose.yml +++ b/deploy/self-hosting/docker-compose.yml @@ -2,12 +2,12 @@ name: fluxer x-fluxer-postgres-env: &fluxer-postgres-env FLUXER_DATABASE_BACKEND: postgres - FLUXER_POSTGRES_HOST: postgres - FLUXER_POSTGRES_PORT: "5432" - FLUXER_POSTGRES_DATABASE: fluxer - FLUXER_POSTGRES_USERNAME: fluxer + FLUXER_POSTGRES_HOST: ${FLUXER_POSTGRES_HOST:-postgres} + FLUXER_POSTGRES_PORT: "${FLUXER_POSTGRES_PORT:-5432}" + FLUXER_POSTGRES_DATABASE: ${FLUXER_POSTGRES_DATABASE:-fluxer} + FLUXER_POSTGRES_USERNAME: ${FLUXER_POSTGRES_USERNAME:-fluxer} FLUXER_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env} - FLUXER_POSTGRES_SSL: "false" + FLUXER_POSTGRES_SSL: "${FLUXER_POSTGRES_SSL:-false}" FLUXER_POSTGRES_PREPARED_STATEMENTS: ${FLUXER_POSTGRES_PREPARED_STATEMENTS:-true} x-fluxer-env: &fluxer-env @@ -35,12 +35,12 @@ x-fluxer-env: &fluxer-env FLUXER_SEARCH_URL: http://meilisearch:7700 FLUXER_SEARCH_API_KEY: ${MEILI_MASTER_KEY:?set MEILI_MASTER_KEY in .env} - FLUXER_S3_ENDPOINT: http://seaweedfs:8333 - FLUXER_S3_PUBLIC_ENDPOINT: http://seaweedfs:8333 - FLUXER_S3_REGION: us-east-1 + FLUXER_S3_ENDPOINT: ${FLUXER_S3_ENDPOINT:-http://seaweedfs:8333} + FLUXER_S3_PUBLIC_ENDPOINT: ${FLUXER_S3_PUBLIC_ENDPOINT:-${FLUXER_S3_ENDPOINT:-http://seaweedfs:8333}} + FLUXER_S3_REGION: ${FLUXER_S3_REGION:-us-east-1} FLUXER_S3_ACCESS_KEY_ID: ${FLUXER_S3_ACCESS_KEY:?set FLUXER_S3_ACCESS_KEY in .env} FLUXER_S3_SECRET_ACCESS_KEY: ${FLUXER_S3_SECRET_KEY:?set FLUXER_S3_SECRET_KEY in .env} - FLUXER_S3_FORCE_PATH_STYLE: "true" + FLUXER_S3_FORCE_PATH_STYLE: "${FLUXER_S3_FORCE_PATH_STYLE:-true}" FLUXER_S3_BUCKET_CDN: fluxer FLUXER_S3_BUCKET_UPLOADS: fluxer-uploads FLUXER_S3_BUCKET_DOWNLOADS: fluxer-downloads diff --git a/fluxer_docs/src/content/docs/operator/upgrading.mdx b/fluxer_docs/src/content/docs/operator/upgrading.mdx index 68a14658a..03a10e512 100644 --- a/fluxer_docs/src/content/docs/operator/upgrading.mdx +++ b/fluxer_docs/src/content/docs/operator/upgrading.mdx @@ -212,6 +212,25 @@ Put a dump on a timer as well. On Linux and macOS, this crontab line writes one `/srv/fluxer` stands for the directory holding `.env`. Write it as an absolute path, because cron does not expand `~`. An unescaped `%` in a crontab is a newline, which is why every one of them has a backslash. +## Run a data store outside the stack + +The stack ships its own Postgres and its own object store and points at both by service name. `.env` moves either one somewhere else, and the bundled service is used when the line is absent: + +```ini +FLUXER_POSTGRES_HOST=db.example.com +FLUXER_POSTGRES_PORT=5432 +FLUXER_POSTGRES_SSL=true +FLUXER_S3_ENDPOINT=https://s3.eu-central-1.amazonaws.com +FLUXER_S3_REGION=eu-central-1 +FLUXER_S3_FORCE_PATH_STYLE=false +``` + +`FLUXER_S3_PUBLIC_ENDPOINT` follows `FLUXER_S3_ENDPOINT` when it is not set on its own, and the credentials stay `FLUXER_S3_ACCESS_KEY` and `FLUXER_S3_SECRET_KEY`. + +Pointing the stack elsewhere leaves the bundled service defined and running with nothing reading it. Take it out with an override file listed in `COMPOSE_FILE`, which [Keep a local compose change](#keep-a-local-compose-change) describes, rather than by editing `docker-compose.yml`, which the Place step replaces on every upgrade. + +An upgrade backs up what the stack holds. The Dump step reaches into the `postgres` service and the Copy step reaches into the `seaweedfs` volume, so a stack that defines neither has neither step to run. The run says which one it skipped and goes on. Backing up a store outside the stack belongs to whoever runs it, and the record the upgrade writes holds no dump of it, so take one before upgrading if a rollback would need it. + ## Roll back A rollback puts the previous release back: diff --git a/fluxer_docs/src/installer/install.ps1 b/fluxer_docs/src/installer/install.ps1 index 4cc0ecbdf..6a6ee864e 100644 --- a/fluxer_docs/src/installer/install.ps1 +++ b/fluxer_docs/src/installer/install.ps1 @@ -1281,7 +1281,28 @@ function Test-FluxerDumpHeader([string]$Path) { # The volume copy measures its volume and refuses when the disk is short. This step does not, # because the size of a custom-format dump is not knowable before pg_dump writes it. Point # -BackupDir at a drive with room for the database. +# The bundled data stores are services in the stack file. An operator who points the stack at a +# database or an object store outside it takes those services out, and the two backup steps that +# reach into them then have nothing to reach. That is a supported shape rather than a fault, so +# each step says what it skipped and the upgrade goes on. Backing up a store outside the stack +# belongs to whoever runs it. +$script:FluxerStackServices = $null + +function Test-FluxerStackDefinesService([string]$Name, [string]$TargetDir) { + if ($null -eq $script:FluxerStackServices) { + if ((Invoke-FluxerComposeQuery '--services') -ne 0) { + Stop-Fluxer "docker compose config --services failed in $TargetDir, so the services this stack defines cannot be read. Compose printed:`n$(Get-FluxerComposeError ' ')" $FluxerExitUnhealthy + } + $script:FluxerStackServices = @(Get-FluxerComposeServices) + } + return $script:FluxerStackServices -contains $Name +} + function Backup-FluxerDatabase([string]$Record, [string]$TargetDir) { + if (-not (Test-FluxerStackDefinesService 'postgres' $TargetDir)) { + Write-FluxerLine 'Skipping the database dump. This stack defines no postgres service, so its database runs outside the stack and only the operator of that database can dump it.' + return + } if (-not (Test-FluxerPostgresRunning)) { Write-FluxerLine 'Postgres is not running. Starting it for the dump.' if ((Invoke-FluxerDocker @('compose', 'up', '-d', '--wait', 'postgres')) -ne 0) { @@ -1340,7 +1361,11 @@ function Get-FluxerFreeKb([string]$Path) { # docker compose stop # docker run --rm -v fluxer_seaweedfs-data:/data -v "${PWD}\backups:/backup" alpine tar czf /backup/seaweedfs-data.tgz -C /data . # docker compose up -d -function Copy-FluxerVolumes([string]$Record, [string]$Project) { +function Copy-FluxerVolumes([string]$Record, [string]$Project, [string]$TargetDir) { + if (-not (Test-FluxerStackDefinesService 'seaweedfs' $TargetDir)) { + Write-FluxerLine 'Skipping the uploads copy. This stack defines no seaweedfs service, so its objects live outside the stack and only the operator of that store can copy them.' + return + } $present = @() foreach ($volume in $FluxerBackupVolumes) { $full = "${Project}_$volume" @@ -1391,7 +1416,7 @@ function Backup-FluxerInstance([string]$Record, [string]$TargetDir, [string]$Pro Write-FluxerLine 'Skipping the uploads copy. -NoVolumeBackup was given.' return } - Copy-FluxerVolumes $Record $Project + Copy-FluxerVolumes $Record $Project $TargetDir } # A newer Postgres major does not read the data directory an older major wrote, so moving between diff --git a/fluxer_docs/src/installer/install.sh b/fluxer_docs/src/installer/install.sh index b1bb4602a..c43f0e478 100644 --- a/fluxer_docs/src/installer/install.sh +++ b/fluxer_docs/src/installer/install.sh @@ -1351,7 +1351,28 @@ fluxer_postgres_running() { # step does not, because the size of a custom-format dump is not knowable before # pg_dump writes it. Point --backup-dir at a filesystem with room for the # database. +# The bundled data stores are services in the stack file. An operator who points +# the stack at a database or an object store outside it takes those services out, +# and the two backup steps that reach into them then have nothing to reach. That +# is a supported shape rather than a fault, so each step says what it skipped and +# the upgrade goes on. Backing up a store outside the stack belongs to whoever +# runs it. +fluxer_stack_defines_service() { + if [ ! -f "$fluxer_scratch/all-services" ]; then + if ! fluxer_compose_services > "$fluxer_scratch/all-services"; then + rm -f "$fluxer_scratch/all-services" + fluxer_fail 6 "docker compose config --services failed in $opt_dir, so the services this stack defines cannot be read. Compose printed: +$(fluxer_compose_error ' ')" + fi + fi + grep -qxF "$1" "$fluxer_scratch/all-services" +} + fluxer_dump_postgres() { + if ! fluxer_stack_defines_service postgres; then + fluxer_say 'Skipping the database dump. This stack defines no postgres service, so its database runs outside the stack and only the operator of that database can dump it.' + return 0 + fi if ! fluxer_postgres_running; then fluxer_say 'Postgres is not running. Starting it for the dump.' if ! $fluxer_engine compose up -d --wait postgres; then @@ -1400,6 +1421,10 @@ fluxer_free_kb() { # docker run --rm -v fluxer_seaweedfs-data:/data -v "$PWD/backups:/backup" alpine tar czf /backup/seaweedfs-data.tgz -C /data . # docker compose up -d fluxer_copy_volumes() { + if ! fluxer_stack_defines_service seaweedfs; then + fluxer_say 'Skipping the uploads copy. This stack defines no seaweedfs service, so its objects live outside the stack and only the operator of that store can copy them.' + return 0 + fi fluxer_backup_volumes > "$fluxer_scratch/backup-volumes" : > "$fluxer_scratch/copy-volumes" fluxer_copy_any=0