feat(self-host): add an overlay that turns off bundled seaweedfs (#3041)

deploy/self-hosting/.env.example

@@ -84,6 +84,11 @@ MEILI_MASTER_KEY=CHANGE_ME
 #FLUXER_S3_BUCKET_UPLOADS=fluxer-uploads
 #FLUXER_S3_BUCKET_REPORTS=fluxer-reports
 #FLUXER_S3_BUCKET_HARVESTS=fluxer-harvests
+# With the object store outside the stack, add this overlay to COMPOSE_FILE and
+# the bundled seaweedfs no longer starts. Put it after any other overlay, such as
+# docker-compose.yml:docker-compose.proxy.yml:external-object-store.compose.yml.
+# Needs Compose 2.24.4 or newer.
+#COMPOSE_FILE=docker-compose.yml:external-object-store.compose.yml
 
 # The other bundled services, pointed elsewhere. Removing a service from the
 # stack belongs in an override file, since an upgrade replaces docker-compose.yml.

deploy/self-hosting/external-object-store.compose.yml

@@ -0,0 +1,14 @@
+services:
+  seaweedfs:
+    profiles: [bundled-object-store]
+  seaweedfs-init:
+    profiles: [bundled-object-store]
+  api:
+    depends_on:
+      seaweedfs-init: !reset null
+  worker:
+    depends_on:
+      seaweedfs-init: !reset null
+  media-proxy:
+    depends_on:
+      seaweedfs-init: !reset null

fluxer_docs/src/content/docs/operator/configuration.mdx

@@ -463,7 +463,7 @@ Keep persistent storage and the bundled `noeviction` policy to protect deletion
 
 ## Object storage
 
-Every uploaded file lands in an S3-compatible object store, which the stack provides with SeaweedFS. `FLUXER_S3_ACCESS_KEY_ID` and `FLUXER_S3_SECRET_ACCESS_KEY` are required. The rest are optional.
+Every uploaded file lands in an S3-compatible object store, which the stack provides with SeaweedFS. `FLUXER_S3_ACCESS_KEY_ID` and `FLUXER_S3_SECRET_ACCESS_KEY` are required. The rest are optional. [Run a data store outside the stack](/operator/upgrading/#run-a-data-store-outside-the-stack) has the overlay that stops the bundled SeaweedFS once the stack points at another store.
 
 #### `FLUXER_S3_ENDPOINT`
 

fluxer_docs/src/content/docs/operator/get-started.mdx

@@ -136,7 +136,7 @@ powershell -ExecutionPolicy Bypass -File .\install.ps1 -Domain chat.example.com
 
 The run prints one line per phase and ends with the URL to open.
 
-`~/fluxer` then holds `docker-compose.yml`, `docker-compose.proxy.yml`, `tunnel.compose.yml`, `Caddyfile` and `.env.example`, plus a `.env` readable only by you. Every value that ships as `CHANGE_ME` in `.env.example` is a fresh random value. Every command from here on runs in that directory.
+`~/fluxer` then holds `docker-compose.yml`, `docker-compose.proxy.yml`, `tunnel.compose.yml`, `external-object-store.compose.yml`, `Caddyfile` and `.env.example`, plus a `.env` readable only by you. Every value that ships as `CHANGE_ME` in `.env.example` is a fresh random value. Every command from here on runs in that directory.
 
 ### Serving on another port
 

fluxer_docs/src/content/docs/operator/upgrading.mdx

@@ -12,7 +12,7 @@ An upgrade moves an instance to a newer release. `install.sh --update` does all
 
 | Requirement | Value |
 | --- | --- |
-| Working directory | The one holding `.env`, `docker-compose.yml`, `docker-compose.proxy.yml`, `tunnel.compose.yml` and `Caddyfile` |
+| Working directory | The one holding `.env` and the stack files |
 | Compose commands | Behind your own reverse proxy, set `COMPOSE_FILE` in `.env` once and every command picks up the overlay |
 | Free disk | Room for one database dump, one copy of the uploads, and the new images beside the old ones |
 | Downtime | A minute or two for the recreate, plus however long the uploads copy takes with the stack stopped |
@@ -79,7 +79,7 @@ Read the [Linux and macOS installer](https://fluxer.dev/install.sh) or [Windows
 
 ## Keep a local compose change
 
-An upgrade replaces the stack files, including direct edits to `docker-compose.yml`, `docker-compose.proxy.yml`, `tunnel.compose.yml` or the `Caddyfile`. Use `--dry-run` to see which files will change.
+An upgrade replaces the stack files, including direct edits to `docker-compose.yml`, `docker-compose.proxy.yml`, `tunnel.compose.yml`, `external-object-store.compose.yml` or the `Caddyfile`. Use `--dry-run` to see which files will change.
 
 The supported way to hold a local choice is a separate file, listed in `COMPOSE_FILE` in `.env`:
 
@@ -251,7 +251,23 @@ FLUXER_S3_FORCE_PATH_STYLE=false
 
 `FLUXER_KV_URL`, `FLUXER_NATS_URL`, `FLUXER_SEARCH_URL` and `FLUXER_LIVEKIT_INTERNAL_URL` move the other bundled services the same way, and `FLUXER_NATS_JETSTREAM_URL` and `FLUXER_SVC_NATS_URL` follow `FLUXER_NATS_URL` when they are not set on their own.
 
-Pointing the stack elsewhere leaves the bundled service defined and running with nothing reading it. Take it out with an override file listed in `COMPOSE_FILE`, which [Keep a local compose change](#keep-a-local-compose-change) describes, rather than by editing `docker-compose.yml`, which the Place step replaces on every upgrade.
+Pointing the stack elsewhere leaves the bundled service defined and running with nothing reading it. The object store has a shipped overlay for that. `external-object-store.compose.yml` keeps `seaweedfs` and `seaweedfs-init` from starting, and `api`, `worker` and `media-proxy` stop waiting for `seaweedfs-init`. Add it to `COMPOSE_FILE` in `.env`, after any other overlay:
+
+```ini
+COMPOSE_FILE=docker-compose.yml:external-object-store.compose.yml
+```
+
+Behind a reverse proxy the line is `docker-compose.yml:docker-compose.proxy.yml:external-object-store.compose.yml`. The overlay needs Compose 2.24.4 or newer. An upgrade refreshes it with the other stack files, skips the uploads copy and does not wait for `seaweedfs-init`.
+
+Compose leaves containers that already exist in place, so remove the two old ones once after the switch:
+
+```bash
+docker compose rm -sf seaweedfs seaweedfs-init
+```
+
+The `seaweedfs-data` volume stays until you delete it. Copy its objects to the new store before you delete it.
+
+The other bundled services have no shipped overlay. Take one out with an override file listed in `COMPOSE_FILE`, which [Keep a local compose change](#keep-a-local-compose-change) describes, rather than by editing `docker-compose.yml`, which the Place step replaces on every upgrade.
 
 The installer backs up only the bundled database and object store. Arrange separate backups for external stores before upgrading, and keep them with the release's backup record.
 

fluxer_docs/src/installer/install.ps1

@@ -104,6 +104,7 @@ $FluxerStackFiles = @(
 	'docker-compose.yml'
 	'docker-compose.proxy.yml'
 	'tunnel.compose.yml'
+	'external-object-store.compose.yml'
 	'Caddyfile'
 	'.env.example'
 )
@@ -670,6 +671,7 @@ function Move-FluxerStackFiles([string]$StagingDir, [string]$TargetDir) {
 	foreach ($name in $FluxerStackFiles) {
 		Move-Item -LiteralPath (Join-Path $StagingDir $name) -Destination (Join-Path $TargetDir (Get-FluxerPlacedName $name)) -Force
 	}
+	$script:FluxerStackServices = $null
 }
 
 # The same file by hand, which is what the caller of this function does in one pass:
@@ -785,7 +787,7 @@ function Wait-FluxerStack([string]$Lead) {
 	$deadline = (Get-Date).AddSeconds($FluxerReadyTimeoutSeconds)
 	$reportAt = (Get-Date).AddSeconds($FluxerReadyReportSeconds)
 	while ((Get-Date) -lt $deadline) {
-		$rows = @(Get-FluxerComposeRows)
+		$rows = @(Get-FluxerComposeRows | Where-Object { Test-FluxerStackDefinesService (Get-FluxerProperty $_ 'Service') (Get-Location).Path })
 		if ($rows.Count -gt 0) {
 			$ready = Measure-FluxerReadyRows $rows
 			if ($ready -eq $rows.Count) {
@@ -1877,6 +1879,13 @@ function Resolve-FluxerComposeBase([string]$TargetDir, [string]$EnvPath) {
 	}
 }
 
+function Get-FluxerOverlayAbsence([string]$Name) {
+	if ($Name -eq 'external-object-store.compose.yml') {
+		return 'Without it the bundled seaweedfs starts again and api, worker and media-proxy wait for it.'
+	}
+	return "Without $Name the edge container binds 80 and 443 and requests its own certificate."
+}
+
 function Assert-FluxerComposeFiles([string]$TargetDir, [string]$EnvPath) {
 	$setting = Get-FluxerComposeSetting $EnvPath
 	$value = $setting.Value
@@ -1897,7 +1906,7 @@ function Assert-FluxerComposeFiles([string]$TargetDir, [string]$EnvPath) {
 			continue
 		}
 		if ($FluxerStackFiles -contains $name) {
-			Stop-Fluxer "COMPOSE_FILE from $source names $name and $path is not there, so every docker compose command in $TargetDir fails and this run stops before it changes anything. This script downloads $name, and an instance set up before it existed does not hold that file yet. Put it in place and run this again:`n  Invoke-WebRequest -Uri $FluxerRawBase/$Ref/$FluxerStackPath/$name -OutFile $path -UseBasicParsing`nLeave the COMPOSE_FILE line as it is. Without $name the edge container binds 80 and 443 and requests its own certificate." $FluxerExitPrerequisite
+			Stop-Fluxer "COMPOSE_FILE from $source names $name and $path is not there, so every docker compose command in $TargetDir fails and this run stops before it changes anything. This script downloads $name, and an instance set up before it existed does not hold that file yet. Put it in place and run this again:`n  Invoke-WebRequest -Uri $FluxerRawBase/$Ref/$FluxerStackPath/$name -OutFile $path -UseBasicParsing`nLeave the COMPOSE_FILE line as it is. $(Get-FluxerOverlayAbsence $name)" $FluxerExitPrerequisite
 		}
 		Stop-Fluxer "COMPOSE_FILE from $source names $name and $path is not there, so every docker compose command in $TargetDir fails. This script does not download $name. Put that file back, or take it out of the COMPOSE_FILE line." $FluxerExitPrerequisite
 	}

fluxer_docs/src/installer/install.sh

@@ -58,9 +58,9 @@ FLUXER_MIN_ENGINE='24.0.0'
 # as written, healthcheck conditions and all.
 FLUXER_MIN_PODMAN='5.0.0'
 FLUXER_MIN_COMPOSE='2.20.2'
-# Both overlays this script downloads use the !override tag, which Compose learned
+# Every overlay this script downloads uses the !override or !reset tag, which
-# in 2.24.4. A stack that loads neither runs on the lower minimum, so the higher
+# Compose 2.24.4 reads. A stack that loads none runs on the lower minimum, so the
-# one is required only once COMPOSE_FILE names more than one file.
+# higher one is required only once COMPOSE_FILE names more than one file.
 FLUXER_MIN_COMPOSE_OVERLAY='2.24.4'
 FLUXER_READY_TIMEOUT=600
 FLUXER_READY_INTERVAL=5
@@ -127,6 +127,7 @@ fluxer_stack_files() {
 docker-compose.yml
 docker-compose.proxy.yml
 tunnel.compose.yml
+external-object-store.compose.yml
 Caddyfile
 .env.example
 FILES
@@ -949,6 +950,7 @@ fluxer_stack_ready() {
 	fluxer_service_count=0
 	while read -r fluxer_service fluxer_status fluxer_health fluxer_code; do
 		[ -n "$fluxer_service" ] || continue
+		fluxer_stack_defines_service "$fluxer_service" || continue
 		fluxer_service_count=$((fluxer_service_count + 1))
 		case $fluxer_status in
 			running)
@@ -1276,6 +1278,13 @@ fluxer_resolve_compose_base() {
 	esac
 }
 
+fluxer_overlay_absence() {
+	case $1 in
+		external-object-store.compose.yml) printf '%s' 'Without it the bundled seaweedfs starts again and api, worker and media-proxy wait for it.' ;;
+		*) printf '%s' "Without $1 the edge container binds 80 and 443 and requests its own certificate." ;;
+	esac
+}
+
 fluxer_require_compose_files() {
 	fluxer_read_compose_setting
 	[ -n "$fluxer_compose_file" ] || return 0
@@ -1302,13 +1311,13 @@ fluxer_require_compose_files() {
 		if fluxer_stack_files | grep -qxF "$fluxer_name"; then
 			fluxer_fail 2 "COMPOSE_FILE from $fluxer_compose_from names $fluxer_name and $fluxer_path is not there, so every $fluxer_engine compose command in $opt_dir fails and this run stops before it changes anything. This script downloads $fluxer_name, and an instance set up before it existed does not hold that file yet. Put it in place and run this again:
   curl -fsSL --proto '=https' --tlsv1.2 -o $fluxer_path $FLUXER_RAW_BASE/$opt_ref/$FLUXER_STACK_PATH/$fluxer_name
-Leave the COMPOSE_FILE line as it is. Without $fluxer_name the edge container binds 80 and 443 and requests its own certificate."
+Leave the COMPOSE_FILE line as it is. $(fluxer_overlay_absence "$fluxer_name")"
 		fi
 		fluxer_fail 2 "COMPOSE_FILE from $fluxer_compose_from names $fluxer_name and $fluxer_path is not there, so every $fluxer_engine compose command in $opt_dir fails. This script does not download $fluxer_name. Put that file back, or take it out of the COMPOSE_FILE line."
 	done
 	if [ "$fluxer_compose_count" -gt 1 ] &&
 		! fluxer_version_ge "$fluxer_compose_version" "$FLUXER_MIN_COMPOSE_OVERLAY"; then
-		fluxer_fail 2 "COMPOSE_FILE from $fluxer_compose_from loads $fluxer_compose_count files and this host runs Compose $fluxer_compose_version. Every overlay this script downloads uses the !override tag, which needs Compose $FLUXER_MIN_COMPOSE_OVERLAY or newer. Upgrade Compose, or load only $fluxer_compose_base."
+		fluxer_fail 2 "COMPOSE_FILE from $fluxer_compose_from loads $fluxer_compose_count files and this host runs Compose $fluxer_compose_version. Every overlay this script downloads uses the !override or !reset tag, which needs Compose $FLUXER_MIN_COMPOSE_OVERLAY or newer. Upgrade Compose, or load only $fluxer_compose_base."
 	fi
 }