fix(self-hosting): adapt the upgrade to existing instances (#2550)

deploy/self-hosting/.env.example

@@ -189,6 +189,16 @@ FLUXER_DISCOVERY_ENABLED=true
 #FLUXER_VALKEY_MEMORY_LIMIT=256mb
 #FLUXER_NATS_MEMORY_LIMIT=256mb
 #FLUXER_MEILISEARCH_MEMORY_LIMIT=768mb
+# LiveKit finds the address browsers dial by asking a STUN server. A host that
+# cannot reach one over UDP stops with "could not resolve external IP", and the
+# address is then set by hand: put it in FLUXER_LIVEKIT_NODE_IP and turn the
+# lookup off. Point the two STUN entries at another server to keep the lookup
+# and leave Google out of it.
+#FLUXER_LIVEKIT_USE_EXTERNAL_IP=true
+#FLUXER_LIVEKIT_NODE_IP=203.0.113.10
+#FLUXER_LIVEKIT_STUN_PRIMARY=stun.l.google.com:19302
+#FLUXER_LIVEKIT_STUN_SECONDARY=stun1.l.google.com:19302
+
 #FLUXER_SEAWEEDFS_MEMORY_LIMIT=512mb
 #FLUXER_SEAWEEDFS_INIT_MEMORY_LIMIT=128mb
 #FLUXER_LIVEKIT_MEMORY_LIMIT=512mb

deploy/self-hosting/docker-compose.yml

@@ -332,10 +332,11 @@ services:
         rtc:
           tcp_port: ${FLUXER_LIVEKIT_TCP_PORT:-7881}
           udp_port: ${FLUXER_LIVEKIT_UDP_PORT:-7882}
-          use_external_ip: true
+          use_external_ip: ${FLUXER_LIVEKIT_USE_EXTERNAL_IP:-true}
+          node_ip: "${FLUXER_LIVEKIT_NODE_IP:-}"
           stun_servers:
-            - stun.l.google.com:19302
+            - ${FLUXER_LIVEKIT_STUN_PRIMARY:-stun.l.google.com:19302}
-            - stun1.l.google.com:19302
+            - ${FLUXER_LIVEKIT_STUN_SECONDARY:-stun1.l.google.com:19302}
         webhook:
           api_key: ${LIVEKIT_API_KEY:?set LIVEKIT_API_KEY in .env}
           urls:

fluxer_docs/src/content/docs/operator/configuration.mdx

@@ -18,6 +18,15 @@ The installer in [Get started](/operator/get-started/) writes `.env` for you and
 
 Compose reads `.env` and passes the values it names into containers. A name reaches a container only when `docker-compose.yml` lists it, either in the shared `x-fluxer-env` block or in that service's own `environment` block. No service declares `env_file`, so a name in `.env` that appears in neither block never arrives, whatever it is set to.
 
+A `$` inside a value is a variable reference to Compose, not a character. `POSTGRES_PASSWORD=ab$cd` reaches the container as `ab`, and every command against the stack prints `The "cd" variable is not set. Defaulting to a blank string.` first. Write the `$` as `$$`, or put single quotes around the whole value. Both deliver one literal `$`:
+
+```ini
+POSTGRES_PASSWORD=ab$$cd
+POSTGRES_PASSWORD='ab$cd'
+```
+
+Double quotes do not escape it. `docker compose config` prints a literal `$` back as `$$`, so a value that reads `ab$$cd` in that output is the correct one. The fourteen secrets the installer generates are hex or base64 and hold no `$`, so this reaches an instance through a password, an API key or an SMTP secret pasted in by hand.
+
 A container's environment is fixed when the container is created, and `api` and `worker` cache their configuration at first load. Either way a change needs the process restarted, which `docker compose up -d` does by recreating the service.
 
 Precedence, highest first:
@@ -1535,9 +1544,11 @@ Defaults to the crate version. The reported build. `BUILD_VERSION` is preferred
 
 `app-proxy` builds a per-request nonce-based policy for the client HTML and the assets it serves. Each variable appends sources to one directive on top of the built-in ones. All are empty by default, and Compose forwards all eleven.
 
+Every name below goes in `.env`. `app-proxy` reads its environment at container start, so a change takes effect on `docker compose up -d app-proxy` and not on `docker compose restart app-proxy`.
+
 `FLUXER_CSP_EXTRA_DEFAULT_SRC`, `FLUXER_CSP_EXTRA_CONNECT_SRC`, `FLUXER_CSP_EXTRA_IMG_SRC`, `FLUXER_CSP_EXTRA_MEDIA_SRC`, `FLUXER_CSP_EXTRA_FONT_SRC`, `FLUXER_CSP_EXTRA_SCRIPT_SRC`, `FLUXER_CSP_EXTRA_STYLE_SRC`, `FLUXER_CSP_EXTRA_FRAME_SRC`, `FLUXER_CSP_EXTRA_WORKER_SRC`, and `FLUXER_CSP_EXTRA_MANIFEST_SRC` take one or more sources separated by commas, spaces, tabs, or newlines. Blank entries and sources the directive already lists are dropped. `FLUXER_CSP_REPORT_URI` sets a single `report-uri` value.
 
-`object-src`, `base-uri`, and `frame-ancestors` are fixed and have no override. `app-proxy` reads the discovery document and adds the static CDN endpoint, the media endpoint, and the origins of the configured branding images, so a stack on one hostname needs no extra sources. The usual reason to set one is a voice server on another hostname, which needs its WebSocket origin in `FLUXER_CSP_EXTRA_CONNECT_SRC`.
+`object-src`, `base-uri`, and `frame-ancestors` are fixed and have no override. `app-proxy` reads the discovery document and adds the static CDN endpoint, the media endpoint, and the origins of the configured branding images, so a stack on one hostname needs no extra sources. The usual reason to set one is a voice server on another hostname, which needs its WebSocket origin in `FLUXER_CSP_EXTRA_CONNECT_SRC`. [Voice media does not use the proxy](/operator/reverse-proxy/#voice-media-does-not-use-the-proxy) has that line in place.
 
 A front proxy must not add a Content-Security-Policy of its own.
 

fluxer_docs/src/content/docs/operator/reverse-proxy.mdx

@@ -483,8 +483,10 @@ LiveKit signalling goes through `/livekit/*` like everything else. WebRTC media
 
 Both ports are published directly by the stack and must reach the host. A proxy or tunnel in front of `443` does nothing for them.
 
-Hosting LiveKit on a hostname other than `FLUXER_DOMAIN` means widening the Content-Security-Policy the web app runs under:
+Hosting LiveKit on a hostname other than `FLUXER_DOMAIN` means widening the Content-Security-Policy the web app runs under. The line goes in `.env`, beside `FLUXER_DOMAIN`:
 
 ```ini
 FLUXER_CSP_EXTRA_CONNECT_SRC=wss://livekit.example.com:7881
 ```
+
+`app-proxy` builds the policy and reads its environment at container start, so apply the change with `docker compose up -d app-proxy`. `docker compose restart app-proxy` reuses the existing container with its old environment. [Content Security Policy](/operator/configuration/#content-security-policy) has the other ten variables.

fluxer_docs/src/content/docs/operator/upgrading.mdx

@@ -46,20 +46,73 @@ A failed run leaves the instance running on the images it already had.
 
 `docker compose up -d` does not notice a changed `Caddyfile`. The script restarts `edge` by name to pick it up.
 
-The refreshed `docker-compose.yml` renames the `caddy` service to `edge`, and the two publish the same host ports. `--remove-orphans` takes the old container down in the same call, so the new one can bind them. Without it the step stops with `Bind for 0.0.0.0:443 failed`. It also removes any container in the project whose service the loaded Compose files no longer define, so keep `COMPOSE_FILE` the same across an upgrade.
+The refreshed `docker-compose.yml` renames the `caddy` service to `edge`, and the two publish the same host ports. `--remove-orphans` takes the old container down in the same call, so the new one can bind them. Without it the step stops with `Bind for 0.0.0.0:443 failed`. It also removes any container in the project whose service the loaded Compose files no longer define, so keep `COMPOSE_FILE` the same across an upgrade. [Keep a local compose change](#keep-a-local-compose-change) has the file layout that survives one, and [When the compose file list names a missing file](#when-the-compose-file-list-names-a-missing-file) has the failure a line naming an absent file produces.
 
 The rename also moves the `caddy-data` and `caddy-config` volumes to `edge-data` and `edge-config`, so the old two are left unused and the edge requests its certificate again on the first start.
 
-`FLUXER_IMAGE_TAG` is the only line in `.env` that either upgrade mode writes. Every secret is left as it is.
+`FLUXER_IMAGE_TAG` is the only line in `.env` that either upgrade mode rewrites once the instance holds every key the stack requires. An upgrade also writes a key the refreshed stack requires that `.env` does not hold at all, listed below, and it leaves every value already set as it is.
 
-The refreshed `docker-compose.yml` requires `FLUXER_ERLANG_COOKIE`. An `.env` written by an earlier installer has no such line, so add `FLUXER_ERLANG_COOKIE=$(openssl rand -hex 32)` to it before you upgrade. The value is 64 hex characters, which is what a fresh install writes. Until `.env` sets it, every Compose command against the stack stops with `set FLUXER_ERLANG_COOKIE in .env`.
+The refreshed `docker-compose.yml` requires `FLUXER_ERLANG_COOKIE`. An `.env` written by an earlier installer holds no such line, and `--update` writes one into `.env` before it reads anything, so an upgrade needs no edit for it. The value is 64 hex characters, which is what a fresh install writes. Until `.env` sets it, every Compose command against the stack stops with `set FLUXER_ERLANG_COOKIE in .env`.
 
-`api` and `media-proxy` now refuse to start unless `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64` decodes to at least 32 bytes. The line has been in `.env.example` for a while as `CHANGE_ME`, which decodes to 6 bytes, and nothing read it before, so an instance can be running today with the placeholder. Set it with `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64=$(openssl rand -base64 32)` before you upgrade. Both services read the same value, so give them one secret rather than two. A fresh install writes it for you. Without it `api` stops at boot with `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 is required for the API`.
+To write it by hand, run the generator in a shell:
+
+```bash
+printf '\nFLUXER_ERLANG_COOKIE=%s\n' "$(openssl rand -hex 32)" >> .env
+```
+
+Run that as a command. Pasting `FLUXER_ERLANG_COOKIE=$(openssl rand -hex 32)` into `.env` as text stores those characters as the value, because Compose reads a line literally and runs nothing in it. The leading newline is what keeps the key on its own line when the last line of `.env` ends without one, which an editor can leave behind and which would otherwise join the two.
+
+`api` and `media-proxy` now refuse to start unless `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64` decodes to at least 32 bytes. The line has been in `.env.example` for a while as `CHANGE_ME`, which decodes to 6 bytes, and nothing read it before, so an instance can be running today with the placeholder. `--update` replaces an absent or `CHANGE_ME` value with a generated one before it reads anything. Both services read the same value, so they take one secret rather than two, and writing it in one place is what gives them that. A `CHANGE_ME` value stops `api` at boot with `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 must decode to at least 32 bytes`, and an empty one with `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 is required for the API`.
+
+To write it by hand, run the generator in a shell the same way:
+
+```bash
+printf '\nFLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64=%s\n' "$(openssl rand -base64 32)" >> .env
+```
 
 ## The script is the reference
 
 Every step above sits in the script beside a comment holding the command that does that step alone and the reason the step exists. Read [https://fluxer.dev/install.sh](https://fluxer.dev/install.sh), or [https://fluxer.dev/install.ps1](https://fluxer.dev/install.ps1) for Windows.
 
+## Keep a local compose change
+
+The Place step replaces all five stack files with the copies at the ref. An edit made directly in `docker-compose.yml`, `docker-compose.proxy.yml`, `tunnel.compose.yml` or the `Caddyfile` is gone once that step runs. The run prints one line for the whole step and never names the files it replaced. `--dry-run` names every one of them, as `changes`, `unchanged` or `is new`.
+
+The supported way to hold a local choice is a separate file, listed in `COMPOSE_FILE` in `.env`:
+
+```ini
+COMPOSE_FILE=docker-compose.yml:docker-compose.proxy.yml:local.compose.yml
+```
+
+Compose merges the files left to right, so `local.compose.yml` wins over the ones before it. An upgrade refreshes the five stack files and nothing else, so a file outside that set survives every upgrade untouched. A record copies `.env` and those five files, and no other file from the working directory, so an override file is never backed up with them and belongs wherever the rest of the configuration lives.
+
+`.env` needs no such file. `FLUXER_IMAGE_TAG` is the only line either upgrade mode writes.
+
+[Enable the overlay](/operator/reverse-proxy/#enable-the-overlay) has the overlay this is most often used for, and [Docker labels](/operator/reverse-proxy/#docker-labels) has a worked third file.
+
+## When the compose file list names a missing file
+
+`COMPOSE_FILE` names files relative to the working directory, and Compose refuses to load the set when one of them is absent:
+
+```
+stat /srv/fluxer/docker-compose.proxy.yml: no such file or directory
+```
+
+Every `docker compose` command stops there. An upgrade checks the list before it runs any of them, so it stops with a sentence naming the file and the `curl` that puts it back, and writes nothing.
+
+This happens on an instance older than the installer. `docker-compose.proxy.yml` and `tunnel.compose.yml` are files the upgrade downloads, so a `COMPOSE_FILE` line naming one of them in a directory that never held it fails long before the Fetch step that would have supplied it.
+
+Download the missing file at the ref the upgrade moves to, then run the upgrade:
+
+```bash
+curl -fsSL --proto '=https' --tlsv1.2 -o docker-compose.proxy.yml \
+  https://raw.githubusercontent.com/fluxerapp/fluxer/main/deploy/self-hosting/docker-compose.proxy.yml
+```
+
+`main` is the ref for `FLUXER_IMAGE_TAG=v1` and for `latest`. [Match the images to the stack files](#match-the-images-to-the-stack-files) has the ref a pinned tag needs.
+
+Removing the line from `.env` also gets the run moving and is the wrong fix. Without the overlay the edge publishes `80` and `443` and takes them from the reverse proxy already on the host.
+
 ## Run the upgrade
 
 Keep the installer beside the stack files. [Get started](/operator/get-started/#step-4-bring-up-the-instance) has the download and the checksum check for a fresh copy.
@@ -98,6 +151,32 @@ The run ends by naming the record it wrote and the command that rolls back to it
 
 `--dir` and `--ref` mean what they mean on an install, and the installer derives an omitted `--ref` from the `FLUXER_IMAGE_TAG` line in `.env`. Every failure prints a sentence on standard error and exits non-zero, and the script header lists what each code means. A Postgres major version change is exit 3.
 
+## Voice signalling moved to /livekit
+
+The edge serves LiveKit at `/livekit/*`. An instance set up before that routing served it under `/gateway/livekit`, and that path is gone. Voice stops connecting after the upgrade and the run still reports success, because the readiness poll and the `/_health` probe never reach voice.
+
+Before:
+
+```
+wss://chat.example.com/gateway/livekit
+```
+
+After:
+
+```
+wss://chat.example.com/livekit
+```
+
+Only the path is wrong. `docker-compose.yml` derives `https://chat.example.com/livekit` when `.env` sets no `FLUXER_LIVEKIT_URL`, and the client rewrites a leading `http` to `ws` itself, so `https` and `wss` both work.
+
+The address browsers dial is a column on a voice server row in the database. No upgrade migrates that row, and it is corrected in two places.
+
+First `.env`. When it sets `FLUXER_LIVEKIT_URL`, put the new path in that line and run `docker compose up -d api`. `api` writes that value onto the default voice server row at every start, so a row corrected in the dashboard while `.env` still names the old path goes back to the old path on the next restart. Removing the line is also correct, and Compose then derives the URL from `FLUXER_PUBLIC_ORIGIN`, or from the scheme and the domain.
+
+Then the dashboard, at `https://chat.example.com/admin` under **Voice Servers**. The **Endpoint** field on that page holds the address. `api` writes one row only, the one in region `default` with server id `default-server-1`, and leaves every other row as it is. That row corrects itself on the first `api` start after `.env` is right, so the dashboard is for every other row. When that region or that server is absent, its log says `skipping config sync` and it writes nothing. A region added by hand, or one left from an older layout under another id, keeps the endpoint it already holds until that field is edited.
+
+Voice media is unaffected. It never went through the edge, and `7881/tcp` and `7882/udp` still reach the host directly.
+
 ## What the backup covers
 
 Each upgrade writes one record directory, named `record-` and a UTC stamp, under `backups` or wherever `--backup-dir` points. It holds:

fluxer_docs/src/installer/install.ps1

@@ -19,8 +19,9 @@
 # types to do that step by hand, and the reason the step exists.
 #
 # Read this file before running it. The default mode writes .env, which holds every secret the
-# instance has. The upgrade generates no secret and rewrites no secret. The only line either
+# instance has. The upgrade generates a secret only for a key the refreshed stack requires that
-# upgrade mode ever changes in .env is FLUXER_IMAGE_TAG, and only during a rollback that moves
+# .env does not hold, and rewrites no secret. The only value either
+# upgrade mode ever replaces in .env is FLUXER_IMAGE_TAG, and only during a rollback that moves
 # off a pinned tag.
 #
 # Rewriting a secret in .env against volumes that already exist is the one thing this script
@@ -136,6 +137,15 @@ $FluxerBackupVolumes = @(
 	'seaweedfs-data'
 )
 
+# The keys a refreshed stack requires that an .env written by an older installer does not hold.
+# Only keys with no state anywhere else are here. A value like POSTGRES_PASSWORD is matched by a
+# password stored inside the database, so minting a new one locks the stack out of its own data
+# and it is not listed. CHANGE_ME counts as absent.
+$FluxerUpgradeSecretKeys = @(
+	@{Name = 'FLUXER_ERLANG_COOKIE'; Kind = 'hex'}
+	@{Name = 'FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64'; Kind = 'base64'}
+)
+
 $FluxerSecretKeys = @(
 	@{Name = 'POSTGRES_PASSWORD'; Kind = 'hex'}
 	@{Name = 'MEILI_MASTER_KEY'; Kind = 'hex'}
@@ -240,21 +250,30 @@ function Test-FluxerVersionAtLeast([string]$Found, [string]$Minimum) {
 	return $true
 }
 
+# docker writes the reason a command failed to stderr and nothing else ever repeats it, so stderr
+# is kept rather than discarded. The two streams stay apart because every caller reads Text as
+# data, and one warning line on stderr would be one more line of JSON, one more image reference or
+# one more container ID.
 function Invoke-FluxerCapture([string[]]$CommandArgs) {
 	$previous = $ErrorActionPreference
 	$ErrorActionPreference = 'Continue'
-	$text = ''
+	$out = @()
+	$err = @()
 	$code = 0
 	try {
-		$output = & docker @CommandArgs 2>$null
+		$output = & docker @CommandArgs 2>&1
 		$code = $LASTEXITCODE
-		if ($null -ne $output) {
+		foreach ($item in @($output)) {
-			$text = (@($output) | ForEach-Object {[string]$_}) -join "`n"
+			if ($item -is [System.Management.Automation.ErrorRecord]) {
+				$err += [string]$item
+			} else {
+				$out += [string]$item
+			}
 		}
 	} finally {
 		$ErrorActionPreference = $previous
 	}
-	return @{Code = $code; Text = $text.Trim()}
+	return @{Code = $code; Text = (($out -join "`n").Trim()); Error = (($err -join "`n").Trim())}
 }
 
 function Invoke-FluxerDocker([string[]]$CommandArgs) {
@@ -753,19 +772,39 @@ function Get-FluxerComposeProject([string]$TargetDir) {
 	return ''
 }
 
-# The image references the stack resolves to, deduplicated. Compose expands them from
+$script:FluxerComposeOut = ''
-# docker-compose.yml and .env, so this is the same resolution docker compose pull performs. It
+$script:FluxerComposeErr = ''
-# names the images, not the versions of them.
+
+# One read-only Compose query, with what Compose printed on stderr kept.
+#
+# Compose loads the whole file set and interpolates every variable in it before it answers either
+# query below. A file that is not there, a file it cannot read and a required variable .env does
+# not set all fail at that point, and the stderr text is the only thing that says which of them it
+# was.
+function Invoke-FluxerComposeQuery([string]$Selector) {
+	$result = Invoke-FluxerCapture @('compose', 'config', $Selector)
+	$script:FluxerComposeOut = $result.Text
+	$script:FluxerComposeErr = $result.Error
+	return $result.Code
+}
+
+# What Compose printed on the last query, indented one level for the caller.
+function Get-FluxerComposeError([string]$Indent) {
+	if ($script:FluxerComposeErr.Length -eq 0) {
+		return "${Indent}nothing"
+	}
+	return (($script:FluxerComposeErr.Split("`n") | ForEach-Object {"$Indent$_"}) -join "`n")
+}
+
+# The image references the stack resolves to, deduplicated, from the last query. Compose expands
+# them from docker-compose.yml and .env, so this is the same resolution docker compose pull
+# performs. It names the images, not the versions of them.
 #
 # By hand:
 #   docker compose config --images
 function Get-FluxerComposeImages {
-	$result = Invoke-FluxerCapture @('compose', 'config', '--images')
-	if ($result.Code -ne 0 -or $result.Text.Length -eq 0) {
-		return @()
-	}
 	$refs = @()
-	foreach ($line in $result.Text.Split("`n")) {
+	foreach ($line in $script:FluxerComposeOut.Split("`n")) {
 		$candidate = $line.Trim()
 		if ($candidate.Length -gt 0) {
 			$refs += $candidate
@@ -839,6 +878,16 @@ function Get-FluxerPostgresMajor([string]$Path) {
 	return ''
 }
 
+# -Force, because Get-Item without it returns nothing for a file the filesystem marks hidden while
+# Test-Path still reports that file as present, and the pair reads as a file of zero bytes.
+function Get-FluxerFileLength([string]$Path) {
+	$item = Get-Item -LiteralPath $Path -Force -ErrorAction SilentlyContinue
+	if ($null -eq $item -or $item.PSIsContainer) {
+		return 0
+	}
+	return [long]$item.Length
+}
+
 function Test-FluxerSameFile([string]$Left, [string]$Right) {
 	if (-not (Test-Path -LiteralPath $Left)) {
 		return $false
@@ -876,12 +925,8 @@ function Get-FluxerChangedMounts([string]$TargetDir, [string]$StagingDir) {
 # By hand:
 #   docker compose config --services
 function Get-FluxerComposeServices {
-	$result = Invoke-FluxerCapture @('compose', 'config', '--services')
-	if ($result.Code -ne 0 -or $result.Text.Length -eq 0) {
-		return @()
-	}
 	$services = @()
-	foreach ($line in $result.Text.Split("`n")) {
+	foreach ($line in $script:FluxerComposeOut.Split("`n")) {
 		$candidate = $line.Trim()
 		if ($candidate.Length -gt 0) {
 			$services += $candidate
@@ -890,14 +935,17 @@ function Get-FluxerComposeServices {
 	return @($services)
 }
 
-function Restart-FluxerMounts($Entries) {
+function Restart-FluxerMounts($Entries, [string]$TargetDir) {
 	if (@($Entries).Count -eq 0) {
 		return
 	}
-	$services = Get-FluxerComposeServices
+	if ((Invoke-FluxerComposeQuery '--services') -ne 0) {
+		Stop-Fluxer "docker compose config --services failed in $TargetDir, so the services that mount a refreshed file cannot be restarted. Compose printed:`n$(Get-FluxerComposeError '  ')" $FluxerExitUnhealthy
+	}
+	$services = @(Get-FluxerComposeServices)
 	foreach ($entry in $Entries) {
 		if ($services -notcontains $entry.Service) {
-			Write-FluxerLine "Skipping the restart of $($entry.Service), because the docker-compose.yml in place defines no service by that name."
+			Write-FluxerLine "Skipping the restart of $($entry.Service), because the docker-compose.yml in $TargetDir defines no service by that name."
 			continue
 		}
 		Write-FluxerLine "Restarting $($entry.Service), because $($entry.Name) is mounted into it and up -d does not reload a mounted file."
@@ -939,6 +987,52 @@ function Get-FluxerRecordTag([string]$Record) {
 	return ([string]$lines[0]).Trim()
 }
 
+function Set-FluxerEnvValue([string]$EnvPath, [string]$Name, [string]$Value) {
+	$lines = @(Get-FluxerEnvLines $EnvPath)
+	$written = $false
+	$out = @()
+	foreach ($line in $lines) {
+		if ($line.StartsWith("$Name=")) {
+			$out += "$Name=$Value"
+			$written = $true
+		} else {
+			$out += $line
+		}
+	}
+	if (-not $written) {
+		$out += "$Name=$Value"
+	}
+	Write-FluxerEnvFile $EnvPath $out
+}
+
+# Step 0 of an upgrade: mint the values the refreshed stack requires.
+#
+# Compose stops on a ${NAME:?} it cannot resolve, so a key the running .env does not hold fails
+# every command this script runs before it reaches the step that would have reported it. Writing
+# the value first is what makes the rest of the run possible. This runs before the record, so the
+# .env the record saves is the one that works.
+function Add-FluxerRequiredSecrets([string]$EnvPath) {
+	foreach ($entry in $FluxerUpgradeSecretKeys) {
+		$current = Get-FluxerEnvValue $EnvPath $entry.Name
+		if ($current.Length -ne 0 -and $current -ne 'CHANGE_ME') {
+			continue
+		}
+		$value = ''
+		if ($entry.Kind -eq 'hex') {
+			$value = ConvertTo-FluxerHex (New-FluxerRandomBytes 32)
+		} elseif ($entry.Kind -eq 'base64') {
+			$value = [System.Convert]::ToBase64String((New-FluxerRandomBytes 32))
+		} else {
+			Stop-Fluxer "Unknown secret kind $($entry.Kind) for $($entry.Name)." $FluxerExitSecret
+		}
+		if ($value.Length -eq 0) {
+			Stop-Fluxer "Generated an empty value for $($entry.Name)." $FluxerExitSecret
+		}
+		Set-FluxerEnvValue $EnvPath $entry.Name $value
+		Write-FluxerLine "Wrote $($entry.Name) into .env. The refreshed stack requires it and this instance held no usable value."
+	}
+}
+
 function Get-FluxerNewestRecord([string]$BackupRoot) {
 	if (-not (Test-Path -LiteralPath $BackupRoot)) {
 		return ''
@@ -970,13 +1064,21 @@ function Write-FluxerTextFile([string]$Path, [string[]]$Lines) {
 #
 # By hand:
 #   docker compose images
-function Save-FluxerVersionRecord([string]$Record, [string]$EnvPath) {
+# The record directory is created once both refusals above have passed, so a run
-	Write-FluxerTextFile (Join-Path $Record $FluxerTagFile) @((Get-FluxerEnvValue $EnvPath 'FLUXER_IMAGE_TAG'))
+# that stops here leaves no record behind. A record with no images file would
+# otherwise sort newest and take the place of the last usable one, and a rollback
+# reads the newest.
+function Save-FluxerVersionRecord([string]$BackupRoot, [string]$EnvPath, [string]$TargetDir) {
+	$running = Get-FluxerRunningImageIds
+	if ((Invoke-FluxerComposeQuery '--images') -ne 0) {
+		Stop-Fluxer "docker compose config --images failed in $TargetDir, so the running version cannot be recorded. Compose printed:`n$(Get-FluxerComposeError '  ')" $FluxerExitPrerequisite
+	}
 	$refs = @(Get-FluxerComposeImages)
 	if ($refs.Count -eq 0) {
-		Stop-Fluxer 'docker compose config --images returned nothing, so the running version cannot be recorded.' $FluxerExitPrerequisite
+		Stop-Fluxer "docker compose config --images returned nothing in $TargetDir, so the running version cannot be recorded. The stack files there declare no service with an image." $FluxerExitPrerequisite
 	}
-	$running = Get-FluxerRunningImageIds
+	$Record = New-FluxerRecord $BackupRoot
+	Write-FluxerTextFile (Join-Path $Record $FluxerTagFile) @((Get-FluxerEnvValue $EnvPath 'FLUXER_IMAGE_TAG'))
 	$lines = @()
 	foreach ($reference in $refs) {
 		$id = Get-FluxerRunningImageId $running $reference
@@ -987,6 +1089,7 @@ function Save-FluxerVersionRecord([string]$Record, [string]$EnvPath) {
 	}
 	Write-FluxerTextFile (Join-Path $Record $FluxerImagesFile) $lines
 	Write-FluxerLine "Recorded $($lines.Count) image references in $Record."
+	return $Record
 }
 
 # .env and the stack files go into the record because nothing else regenerates them. .env holds
@@ -1071,8 +1174,18 @@ function Backup-FluxerDatabase([string]$Record, [string]$TargetDir) {
 	Write-FluxerLine "Dumped the database to $dump."
 }
 
+$script:FluxerVolumeError = ''
+
+function Get-FluxerVolumeError([string]$Indent) {
+	if ([string]::IsNullOrWhiteSpace($script:FluxerVolumeError)) {
+		return "${Indent}nothing"
+	}
+	return (($script:FluxerVolumeError -split "`n") | ForEach-Object {"$Indent$_"}) -join "`n"
+}
+
 function Get-FluxerVolumeSizeKb([string]$Volume) {
 	$result = Invoke-FluxerCapture @('run', '--rm', '-v', "${Volume}:/data:ro", $FluxerHelperImage, 'du', '-sk', '/data')
+	$script:FluxerVolumeError = $result.Error
 	if ($result.Code -ne 0) {
 		return -1
 	}
@@ -1100,27 +1213,32 @@ function Get-FluxerFreeKb([string]$Path) {
 #   docker run --rm -v fluxer_seaweedfs-data:/data -v "${PWD}\backups:/backup" alpine tar czf /backup/seaweedfs-data.tgz -C /data .
 #   docker compose up -d
 function Copy-FluxerVolumes([string]$Record, [string]$Project) {
+	$present = @()
 	foreach ($volume in $FluxerBackupVolumes) {
 		$full = "${Project}_$volume"
 		$inspect = Invoke-FluxerCapture @('volume', 'inspect', $full)
 		if ($inspect.Code -ne 0) {
-			Stop-Fluxer "The volume $full does not exist, so the uploads cannot be copied." $FluxerExitBackup
+			Stop-Fluxer "The volume $full does not exist, so the uploads cannot be copied. The stack declares that volume, so this is a project name other than $Project or a volume that was removed. Pass -NoVolumeBackup to take the database dump alone when the uploads of this instance live somewhere this script cannot reach." $FluxerExitBackup
 		}
 		$size = Get-FluxerVolumeSizeKb $full
 		if ($size -lt 0) {
-			Stop-Fluxer "Cannot measure the volume $full." $FluxerExitBackup
+			Stop-Fluxer "Cannot measure the volume $full, so the uploads copy cannot be sized. Pass -NoVolumeBackup to take the database dump alone. Docker printed:`n$(Get-FluxerVolumeError '  ')" $FluxerExitBackup
 		}
 		$free = Get-FluxerFreeKb $Record
 		$need = [long]($size * $FluxerVolumeHeadroomPercent / 100)
 		if ($free -lt $need) {
 			Stop-Fluxer "$full holds $([long]($size / 1024)) MB and $Record has $([long]($free / 1024)) MB free. Point -BackupDir at a drive with room, or pass -NoVolumeBackup to take the database dump alone." $FluxerExitBackup
 		}
+		$present += $volume
+	}
+	if ($present.Count -eq 0) {
+		return
 	}
 	Write-FluxerLine 'Stopping the stack for a consistent copy of the uploads.'
 	if ((Invoke-FluxerDocker @('compose', 'stop')) -ne 0) {
 		Stop-Fluxer 'docker compose stop failed.' $FluxerExitBackup
 	}
-	foreach ($volume in $FluxerBackupVolumes) {
+	foreach ($volume in $present) {
 		$full = "${Project}_$volume"
 		Write-FluxerLine "Copying $full."
 		$code = Invoke-FluxerDocker @('run', '--rm', '-v', "${full}:/data:ro", '-v', "${Record}:/backup", $FluxerHelperImage, 'tar', 'czf', "/backup/$volume.tgz", '-C', '/data', '.')
@@ -1166,7 +1284,7 @@ function Assert-FluxerPostgresMajor([string]$TargetDir, [string]$StagingDir) {
 
 # Puts one line of .env back, and proves it touched nothing else.
 #
-# FLUXER_IMAGE_TAG is the only key either upgrade mode writes. Every other line, which is every
+# FLUXER_IMAGE_TAG is the only key this function writes. Every other line, which is every
 # secret, is compared before the new file replaces the old one, so a rewrite that lost or changed
 # a secret cannot land.
 function Set-FluxerImageTag([string]$EnvPath, [string]$Tag) {
@@ -1199,8 +1317,16 @@ function Show-FluxerUpdatePlan([string]$TargetDir, [string]$EnvPath, [string]$Ba
 	Write-FluxerLine "  Ref:        $Ref"
 	Write-FluxerLine "  Image tag:  $(Get-FluxerEnvValue $EnvPath 'FLUXER_IMAGE_TAG') from .env"
 	Write-FluxerLine "  Backup dir: $BackupRoot"
-	Write-FluxerLine '  Running now:'
 	$running = Get-FluxerRunningImageIds
+	if ((Invoke-FluxerComposeQuery '--images') -ne 0) {
+		Write-FluxerLine '  Refusal:    docker compose config --images fails here, and step 1 of the upgrade reads that list'
+		Write-FluxerLine '  Compose said:'
+		Write-FluxerLine (Get-FluxerComposeError '    ')
+		Write-FluxerLine '  Outcome:    the run stops on step 1 and changes nothing'
+		Write-FluxerLine 'Nothing outside a temporary directory was written. Fix what Compose reports, then run this again.'
+		return
+	}
+	Write-FluxerLine '  Running now:'
 	foreach ($reference in Get-FluxerComposeImages) {
 		$id = Get-FluxerRunningImageId $running $reference
 		if ($id.Length -eq 0) {
@@ -1293,8 +1419,8 @@ function Show-FluxerRollbackPlan([string]$TargetDir, [string]$EnvPath, [string]$
 
 # The full upgrade, in the order that leaves a working instance behind at every point it can fail.
 function Invoke-FluxerUpgrade([string]$TargetDir, [string]$EnvPath, [string]$BackupRoot, [string]$Project) {
-	$record = New-FluxerRecord $BackupRoot
+	Add-FluxerRequiredSecrets $EnvPath
-	Save-FluxerVersionRecord $record $EnvPath
+	$record = Save-FluxerVersionRecord $BackupRoot $EnvPath $TargetDir
 	Save-FluxerCurrentFiles $record $TargetDir $EnvPath
 	Backup-FluxerInstance $record $TargetDir $Project
 
@@ -1339,7 +1465,7 @@ function Invoke-FluxerUpgrade([string]$TargetDir, [string]$EnvPath, [string]$Bac
 	if ((Invoke-FluxerDocker @('compose', 'up', '-d', '--remove-orphans')) -ne 0) {
 		Stop-Fluxer 'docker compose up -d failed. Read docker compose logs.' $FluxerExitUnhealthy
 	}
-	Restart-FluxerMounts $changedMounts
+	Restart-FluxerMounts $changedMounts $TargetDir
 	Wait-FluxerStack 'Waiting for every service to report ready.'
 	$domainValue = Get-FluxerEnvValue $EnvPath 'FLUXER_DOMAIN'
 	if ($domainValue.Length -gt 0) {
@@ -1425,7 +1551,7 @@ function Invoke-FluxerRollback([string]$TargetDir, [string]$EnvPath, [string]$Ba
 	}
 	# The mounted file came back from the record, so its service restarts. Comparing it first
 	# would save one restart and cost the reader a reason.
-	Restart-FluxerMounts $FluxerMountedFiles
+	Restart-FluxerMounts $FluxerMountedFiles $TargetDir
 	Wait-FluxerStack 'Waiting for every service to report ready.'
 	$domainValue = Get-FluxerEnvValue $EnvPath 'FLUXER_DOMAIN'
 	if ($domainValue.Length -gt 0) {
@@ -1441,6 +1567,88 @@ function Invoke-FluxerRollback([string]$TargetDir, [string]$EnvPath, [string]$Ba
 	exit 0
 }
 
+function Assert-FluxerInstance([string]$TargetDir, [string]$EnvPath) {
+	if (-not (Test-Path -LiteralPath $EnvPath)) {
+		Stop-Fluxer "No .env in $TargetDir. That directory holds no instance. Run install.ps1 with neither -Update nor -Rollback to set one up." $FluxerExitPrerequisite
+	}
+	$compose = Join-Path $TargetDir 'docker-compose.yml'
+	if (-not (Test-Path -LiteralPath $compose)) {
+		Stop-Fluxer "No docker-compose.yml in $TargetDir. That directory does not hold an instance." $FluxerExitPrerequisite
+	}
+	if ((Get-FluxerFileLength $compose) -eq 0) {
+		Stop-Fluxer "$compose is empty. A redirect that captured a failed download leaves that, and Compose refuses an empty compose file. Put the file back from a backup or from the record of the last upgrade, then run this again." $FluxerExitPrerequisite
+	}
+}
+
+# Compose reads COMPOSE_FILE from .env and loads every file it names before it answers anything, so
+# one file the directory does not hold fails every docker compose command run in it. What Compose
+# prints for that is a single stat line that names neither COMPOSE_FILE nor .env.
+#
+# Two of the files this script downloads sit in .env.example as a COMPOSE_FILE line to uncomment,
+# so an instance set up before this script existed can hold the line and not the file. An upgrade
+# reads the running images before it refreshes the stack files, so such an instance stops on the
+# first step and no re-run gets any further.
+#
+# The line stays. Without the file it names the edge container binds 80 and 443 and requests its
+# own certificate.
+#
+# By hand:
+#   Select-String COMPOSE_FILE .env
+# A value as Compose reads it: no surrounding quotes and no trailing blanks.
+# Get-FluxerEnvLines already drops a carriage return. Reading it any other way
+# invents a filename the operator cannot see and refuses a run that would have
+# worked.
+function Get-FluxerEnvScalar([string]$EnvPath, [string]$Name) {
+	$raw = (Get-FluxerEnvValue $EnvPath $Name).TrimEnd()
+	if ($raw.Length -ge 2) {
+		if (($raw[0] -eq '"' -and $raw[-1] -eq '"') -or ($raw[0] -eq "'" -and $raw[-1] -eq "'")) {
+			return $raw.Substring(1, $raw.Length - 2)
+		}
+	}
+	return $raw
+}
+
+function Assert-FluxerComposeFiles([string]$TargetDir, [string]$EnvPath) {
+	$value = ''
+	$source = 'the environment'
+	if ($null -ne $env:COMPOSE_FILE) {
+		$value = [string]$env:COMPOSE_FILE
+	}
+	if ($value.Length -eq 0) {
+		$value = Get-FluxerEnvScalar $EnvPath 'COMPOSE_FILE'
+		$source = $EnvPath
+	}
+	if ($value.Length -eq 0) {
+		return
+	}
+	$separator = ''
+	if ($null -ne $env:COMPOSE_PATH_SEPARATOR) {
+		$separator = [string]$env:COMPOSE_PATH_SEPARATOR
+	}
+	if ($separator.Length -eq 0) {
+		$separator = Get-FluxerEnvScalar $EnvPath 'COMPOSE_PATH_SEPARATOR'
+	}
+	if ($separator.Length -eq 0) {
+		$separator = [System.IO.Path]::PathSeparator
+	}
+	foreach ($name in $value.Split([string[]]$separator, [System.StringSplitOptions]::None)) {
+		if ($name.Length -eq 0) {
+			continue
+		}
+		$path = $name
+		if (-not [System.IO.Path]::IsPathRooted($path)) {
+			$path = Join-Path $TargetDir $name
+		}
+		if (Test-Path -LiteralPath $path) {
+			continue
+		}
+		if ($FluxerStackFiles -contains $name) {
+			Stop-Fluxer "COMPOSE_FILE from $source names $name and $path is not there, so every docker compose command in $TargetDir fails and this run stops before it changes anything. This script downloads $name, and an instance set up before it existed does not hold that file yet. Put it in place and run this again:`n  Invoke-WebRequest -Uri $FluxerRawBase/$Ref/$FluxerStackPath/$name -OutFile $path -UseBasicParsing`nLeave the COMPOSE_FILE line as it is. Without $name the edge container binds 80 and 443 and requests its own certificate." $FluxerExitPrerequisite
+		}
+		Stop-Fluxer "COMPOSE_FILE from $source names $name and $path is not there, so every docker compose command in $TargetDir fails. This script does not download $name. Put that file back, or take it out of the COMPOSE_FILE line." $FluxerExitPrerequisite
+	}
+}
+
 function Invoke-FluxerInstall {
 	if ($Help) {
 		Show-FluxerUsage
@@ -1478,9 +1686,18 @@ function Invoke-FluxerInstall {
 	Invoke-FluxerPreflight
 
 	$targetPath = $Dir
+	$adoptedCwd = $false
 	if ($targetPath.Length -eq 0) {
+		$here = (Get-Location).Path
+		$hereEnv = Join-Path $here '.env'
+		$hereIsFluxer = (Test-Path -LiteralPath $hereEnv) -and (@(Get-FluxerEnvLines $hereEnv | Where-Object {$_.StartsWith('FLUXER_')}).Count -gt 0)
+		if (($Update -or $Rollback) -and (Get-FluxerFileLength (Join-Path $here 'docker-compose.yml')) -gt 0 -and $hereIsFluxer) {
+			$targetPath = $here
+			$adoptedCwd = $true
+		} else {
 			$targetPath = Join-Path $HOME 'fluxer'
 		}
+	}
 	$targetDir = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($targetPath)
 	$envPath = Join-Path $targetDir '.env'
 	$backupPath = $BackupDir
@@ -1488,18 +1705,17 @@ function Invoke-FluxerInstall {
 		$backupPath = Join-Path $targetDir 'backups'
 	}
 	$backupRoot = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($backupPath)
+	if ($adoptedCwd) {
+		Write-FluxerLine "Acting on the instance in $targetDir, the working directory. Pass -Dir to name another."
+	}
 
 	if ($Update -or $Rollback) {
-		if (-not (Test-Path -LiteralPath $envPath)) {
+		Assert-FluxerInstance $targetDir $envPath
-			Stop-Fluxer "No .env in $targetDir. That directory holds no instance. Run install.ps1 with neither -Update nor -Rollback to set one up." $FluxerExitPrerequisite
-		}
-		if (-not (Test-Path -LiteralPath (Join-Path $targetDir 'docker-compose.yml'))) {
-			Stop-Fluxer "No docker-compose.yml in $targetDir. That directory does not hold an instance." $FluxerExitPrerequisite
-		}
 		if ($Ref.Length -eq 0) {
 			$script:Ref = Get-FluxerRefForTag (Get-FluxerEnvValue $envPath 'FLUXER_IMAGE_TAG')
 			Assert-FluxerDerivedRef $Ref $envPath
 		}
+		Assert-FluxerComposeFiles $targetDir $envPath
 		$project = Get-FluxerComposeProject $targetDir
 		if ($project.Length -eq 0) {
 			Stop-Fluxer "docker-compose.yml in $targetDir declares no project name, so the volume names cannot be derived." $FluxerExitPrerequisite

fluxer_docs/src/installer/install.sh

@@ -22,8 +22,9 @@
 # an operator types to do that step by hand, and the reason the step exists.
 #
 # Read this file before you run it. The default mode writes .env, which holds
-# every secret the instance has. The upgrade generates no secret and rewrites no
+# every secret the instance has. The upgrade generates a secret only for a key
-# secret. The only line either upgrade mode ever changes in .env is
+# the refreshed stack requires that .env does not hold, and rewrites no
+# secret. The only value either upgrade mode ever replaces in .env is
 # FLUXER_IMAGE_TAG, and only during a rollback that moves off a pinned tag.
 #
 # Rewriting a secret in .env against volumes that already exist is the one thing
@@ -338,8 +339,25 @@ while [ $# -gt 0 ]; do
 	esac
 done
 
+# An upgrade acts on an instance that already exists, and the directory holding
+# one is recognisable: a compose file with content, an .env beside it, and at
+# least one FLUXER_ key in that .env. Taking the working directory when it holds
+# those is what an operator standing in their own instance means, and it is the
+# only case where the default is not ~/fluxer.
+#
+# The FLUXER_ test is what keeps this off somebody else's stack. A compose file
+# and an .env together describe every Compose project there is, and an upgrade
+# replaces the stack files in the directory it acts on, so a looser test would
+# overwrite a project that has nothing to do with Fluxer.
+#
+# An install writes a new instance, so it never adopts the working directory.
 if [ -z "$opt_dir" ]; then
-	if [ -n "${HOME:-}" ]; then
+	if { [ "$opt_update" -eq 1 ] || [ "$opt_rollback" -eq 1 ]; } &&
+		[ -s "$(pwd)/docker-compose.yml" ] && [ -e "$(pwd)/.env" ] &&
+		grep -q '^FLUXER_' "$(pwd)/.env" 2>/dev/null; then
+		opt_dir="$(pwd)"
+		fluxer_say "Acting on the instance in $opt_dir, the working directory. Pass --dir to name another."
+	elif [ -n "${HOME:-}" ]; then
 		opt_dir="$HOME/fluxer"
 	else
 		opt_dir="$(pwd)/fluxer"
@@ -886,6 +904,83 @@ fluxer_env_value() {
 	sed -n "s/^$1=\\(.*\\)\$/\\1/p" "$opt_dir/.env" | head -n 1
 }
 
+# The keys a refreshed stack requires that an .env written by an older installer
+# does not hold. Only keys with no state anywhere else are here. A value like
+# POSTGRES_PASSWORD is matched by a password stored inside the database, so
+# minting a new one locks the stack out of its own data and it is not listed.
+#
+# CHANGE_ME counts as absent. .env.example shipped the relay secret with that
+# value for a while, and nothing read it until the API began refusing to start
+# on a value under 32 bytes.
+fluxer_upgrade_secret_keys() {
+	cat <<'KEYS'
+FLUXER_ERLANG_COOKIE hex
+FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 base64
+KEYS
+}
+
+# One key written into .env, through a temporary file that replaces the whole
+# file at once.
+#
+# Appending is not an option. A .env whose last line has no newline takes an
+# appended line onto the end of that line instead, which reads as one key whose
+# value ends in the name of the next, and destroys the secret that was there.
+# awk ends every line it prints, so reading the file and writing it back gives
+# the missing newline as a side effect.
+#
+# The replacement is a rename, so an interrupt leaves the old file whole rather
+# than a truncated one, and this runs before the record exists.
+fluxer_env_set() {
+	fluxer_old_umask=$(umask)
+	umask 077
+	awk -v fluxer_k="$1" -v fluxer_v="$2" '
+		index($0, fluxer_k "=") == 1 && !fluxer_done {print fluxer_k "=" fluxer_v; fluxer_done = 1; next}
+		{print}
+		END {if (!fluxer_done) print fluxer_k "=" fluxer_v}
+	' "$opt_dir/.env" > "$fluxer_scratch/env.set"
+	if [ ! -s "$fluxer_scratch/env.set" ]; then
+		umask "$fluxer_old_umask"
+		fluxer_fail 5 "Rewriting $1 into .env produced an empty file. Nothing was written."
+	fi
+	if ! grep -q "^$1=" "$fluxer_scratch/env.set"; then
+		umask "$fluxer_old_umask"
+		fluxer_fail 5 "Rewriting $1 into .env did not write that key. Nothing was written."
+	fi
+	mv "$fluxer_scratch/env.set" "$opt_dir/.env"
+	chmod 600 "$opt_dir/.env"
+	umask "$fluxer_old_umask"
+}
+
+# Step 0 of an upgrade: mint the values the refreshed stack requires.
+#
+# Compose stops on a ${NAME:?} it cannot resolve, so a key the running .env does
+# not hold fails every command this script runs before it reaches the step that
+# would have reported it. Writing the value first is what makes the rest of the
+# run possible, and a generated secret is as good as a hand-written one for
+# every key listed above.
+#
+# This runs before the record, so the .env the record saves is the one that
+# works and a rollback does not reintroduce the gap.
+fluxer_fill_required_secrets() {
+	fluxer_upgrade_secret_keys > "$fluxer_scratch/upgrade-keys"
+	while read -r fluxer_key fluxer_kind; do
+		[ -n "$fluxer_key" ] || continue
+		fluxer_current=$(fluxer_env_value "$fluxer_key")
+		case ${fluxer_current:-} in
+			''|CHANGE_ME) ;;
+			*) continue ;;
+		esac
+		case $fluxer_kind in
+			hex) fluxer_new=$(openssl rand -hex 32) ;;
+			base64) fluxer_new=$(openssl rand -base64 32) ;;
+			*) fluxer_fail 5 "Unknown secret kind $fluxer_kind for $fluxer_key." ;;
+		esac
+		[ -n "$fluxer_new" ] || fluxer_fail 5 "Generated an empty value for $fluxer_key."
+		fluxer_env_set "$fluxer_key" "$fluxer_new"
+		fluxer_say "Wrote $fluxer_key into .env. The refreshed stack requires it and this instance held no usable value."
+	done < "$fluxer_scratch/upgrade-keys"
+}
+
 fluxer_require_instance() {
 	if [ ! -e "$opt_dir/.env" ]; then
 		fluxer_fail 2 "No .env in $opt_dir. That directory holds no instance. Run install.sh with neither --update nor --rollback to set one up."
@@ -893,6 +988,105 @@ fluxer_require_instance() {
 	if [ ! -e "$opt_dir/docker-compose.yml" ]; then
 		fluxer_fail 2 "No docker-compose.yml in $opt_dir. That directory does not hold an instance."
 	fi
+	if [ ! -s "$opt_dir/docker-compose.yml" ]; then
+		fluxer_fail 2 "$opt_dir/docker-compose.yml is empty. A redirect that captured a failed download leaves that, and Compose refuses an empty compose file. Put the file back from a backup or from the record of the last upgrade, then run this again."
+	fi
+}
+
+# Compose reads COMPOSE_FILE from .env and loads every file it names before it
+# answers anything, so one file the directory does not hold fails every docker
+# compose command run in it. What Compose prints for that is a single stat line
+# that names neither COMPOSE_FILE nor .env.
+#
+# Two of the files this script downloads sit in .env.example as a COMPOSE_FILE
+# line to uncomment, so an instance set up before this script existed can hold
+# the line and not the file. An upgrade reads the running images before it
+# refreshes the stack files, so such an instance stops on the first step and no
+# re-run gets any further.
+#
+# The line stays. Without the file it names the edge container binds 80 and 443
+# and requests its own certificate.
+#
+# By hand:
+#   grep COMPOSE_FILE .env
+# A value as Compose reads it: no surrounding quotes, no carriage return from an
+# editor that writes CRLF, no trailing blanks. Reading it any other way invents a
+# filename the operator cannot see and refuses a run that would have worked.
+fluxer_env_scalar() {
+	fluxer_scalar=$(fluxer_env_value "$1" | tr -d '\r')
+	fluxer_scalar=${fluxer_scalar%"${fluxer_scalar##*[! 	]}"}
+	case $fluxer_scalar in
+		\"*\") fluxer_scalar=${fluxer_scalar#\"}; fluxer_scalar=${fluxer_scalar%\"} ;;
+		"'"*"'") fluxer_scalar=${fluxer_scalar#"'"}; fluxer_scalar=${fluxer_scalar%"'"} ;;
+	esac
+	printf '%s' "$fluxer_scalar"
+}
+
+fluxer_require_compose_files() {
+	fluxer_compose_file=${COMPOSE_FILE:-}
+	fluxer_compose_from="the environment"
+	if [ -z "$fluxer_compose_file" ]; then
+		fluxer_compose_file=$(fluxer_env_scalar COMPOSE_FILE)
+		fluxer_compose_from="$opt_dir/.env"
+	fi
+	[ -n "$fluxer_compose_file" ] || return 0
+	fluxer_path_sep=${COMPOSE_PATH_SEPARATOR:-}
+	if [ -z "$fluxer_path_sep" ]; then
+		fluxer_path_sep=$(fluxer_env_scalar COMPOSE_PATH_SEPARATOR)
+	fi
+	if [ -z "$fluxer_path_sep" ]; then
+		fluxer_path_sep=':'
+	fi
+	fluxer_rest=$fluxer_compose_file
+	while [ -n "$fluxer_rest" ]; do
+		case $fluxer_rest in
+			*"$fluxer_path_sep"*)
+				fluxer_name=${fluxer_rest%%"$fluxer_path_sep"*}
+				fluxer_rest=${fluxer_rest#*"$fluxer_path_sep"}
+				;;
+			*)
+				fluxer_name=$fluxer_rest
+				fluxer_rest=''
+				;;
+		esac
+		[ -n "$fluxer_name" ] || continue
+		case $fluxer_name in
+			/*) fluxer_path=$fluxer_name ;;
+			*) fluxer_path="$opt_dir/$fluxer_name" ;;
+		esac
+		[ ! -e "$fluxer_path" ] || continue
+		if fluxer_stack_files | grep -qxF "$fluxer_name"; then
+			fluxer_fail 2 "COMPOSE_FILE from $fluxer_compose_from names $fluxer_name and $fluxer_path is not there, so every docker compose command in $opt_dir fails and this run stops before it changes anything. This script downloads $fluxer_name, and an instance set up before it existed does not hold that file yet. Put it in place and run this again:
+  curl -fsSL --proto '=https' --tlsv1.2 -o $fluxer_path $FLUXER_RAW_BASE/$opt_ref/$FLUXER_STACK_PATH/$fluxer_name
+Leave the COMPOSE_FILE line as it is. Without $fluxer_name the edge container binds 80 and 443 and requests its own certificate."
+		fi
+		fluxer_fail 2 "COMPOSE_FILE from $fluxer_compose_from names $fluxer_name and $fluxer_path is not there, so every docker compose command in $opt_dir fails. This script does not download $fluxer_name. Put that file back, or take it out of the COMPOSE_FILE line."
+	done
+}
+
+# One read-only Compose query, with what Compose printed on stderr kept.
+#
+# Compose loads the whole file set and interpolates every variable in it before
+# it answers either query below. A file that is not there, a file it cannot
+# read and a required variable .env does not set all fail at that point, and the
+# stderr text is the only thing that says which of them it was.
+#
+# The query writes to files rather than through a pipe, because the status of a
+# pipeline is the status of its last command and the query is not the last
+# command.
+fluxer_compose_query() {
+	fluxer_query_status=0
+	docker compose config "$1" > "$fluxer_scratch/compose-out" 2> "$fluxer_scratch/compose-err" || fluxer_query_status=$?
+	return "$fluxer_query_status"
+}
+
+# What Compose printed on the last query, indented one level for the caller.
+fluxer_compose_error() {
+	if [ -s "$fluxer_scratch/compose-err" ]; then
+		sed "s/^/$1/" "$fluxer_scratch/compose-err"
+	else
+		printf '%snothing\n' "$1"
+	fi
 }
 
 # The image references the stack resolves to, one per line, deduplicated. Compose
@@ -902,7 +1096,8 @@ fluxer_require_instance() {
 # By hand:
 #   docker compose config --images
 fluxer_compose_images() {
-	docker compose config --images 2>/dev/null | sort -u
+	fluxer_compose_query --images || return 1
+	sort -u "$fluxer_scratch/compose-out"
 }
 
 # The image ID each container was actually created from, against the reference
@@ -945,12 +1140,16 @@ fluxer_recorded_id_for() {
 # By hand:
 #   docker compose images
 fluxer_record_state() {
-	printf '%s\n' "$(fluxer_env_value FLUXER_IMAGE_TAG)" > "$fluxer_record/$FLUXER_TAG_FILE"
 	fluxer_running_image_ids > "$fluxer_scratch/running"
-	fluxer_compose_images > "$fluxer_scratch/refs"
+	if ! fluxer_compose_images > "$fluxer_scratch/refs"; then
-	if [ ! -s "$fluxer_scratch/refs" ]; then
+		fluxer_fail 2 "docker compose config --images failed in $opt_dir, so the running version cannot be recorded. Compose printed:
-		fluxer_fail 2 "docker compose config --images returned nothing in $opt_dir, so the running version cannot be recorded."
+$(fluxer_compose_error '  ')"
 	fi
+	if [ ! -s "$fluxer_scratch/refs" ]; then
+		fluxer_fail 2 "docker compose config --images returned nothing in $opt_dir, so the running version cannot be recorded. The stack files there declare no service with an image."
+	fi
+	fluxer_prepare_record
+	printf '%s\n' "$(fluxer_env_value FLUXER_IMAGE_TAG)" > "$fluxer_record/$FLUXER_TAG_FILE"
 	: > "$fluxer_record/$FLUXER_IMAGES_FILE"
 	while read -r fluxer_ref; do
 		[ -n "$fluxer_ref" ] || continue
@@ -1026,8 +1225,18 @@ fluxer_dump_postgres() {
 	fluxer_say "Dumped the database to $fluxer_dump_path."
 }
 
+# What the sizing container printed, indented one level for the caller.
+fluxer_volume_error() {
+	if [ -s "$fluxer_scratch/volume-err" ]; then
+		sed "s/^/$1/" "$fluxer_scratch/volume-err"
+	else
+		printf '%snothing\n' "$1"
+	fi
+}
+
 fluxer_volume_size_kb() {
-	docker run --rm -v "$1:/data:ro" "$FLUXER_HELPER_IMAGE" du -sk /data 2>/dev/null | awk 'NR==1 {print $1}'
+	docker run --rm -v "$1:/data:ro" "$FLUXER_HELPER_IMAGE" du -sk /data 2> "$fluxer_scratch/volume-err" |
+		awk 'NR==1 {print $1}'
 }
 
 fluxer_free_kb() {
@@ -1046,16 +1255,19 @@ fluxer_free_kb() {
 #   docker compose up -d
 fluxer_copy_volumes() {
 	fluxer_backup_volumes > "$fluxer_scratch/backup-volumes"
+	: > "$fluxer_scratch/copy-volumes"
 	fluxer_copy_any=0
 	while read -r fluxer_volume; do
 		[ -n "$fluxer_volume" ] || continue
 		fluxer_full="${fluxer_project}_${fluxer_volume}"
 		if ! docker volume inspect "$fluxer_full" >/dev/null 2>&1; then
-			fluxer_fail 7 "The volume $fluxer_full does not exist, so the uploads cannot be copied."
+			fluxer_fail 7 "The volume $fluxer_full does not exist, so the uploads cannot be copied. The stack declares that volume, so this is a project name other than $fluxer_project or a volume that was removed. Pass --no-volume-backup to take the database dump alone when the uploads of this instance live somewhere this script cannot reach."
 		fi
 		fluxer_size=$(fluxer_volume_size_kb "$fluxer_full")
 		case ${fluxer_size:-} in
-			''|*[!0-9]*) fluxer_fail 7 "Cannot measure the volume $fluxer_full." ;;
+			''|*[!0-9]*)
+				fluxer_fail 7 "Cannot measure the volume $fluxer_full, so the uploads copy cannot be sized. Pass --no-volume-backup to take the database dump alone. Docker printed:
+$(fluxer_volume_error '  ')" ;;
 		esac
 		fluxer_free=$(fluxer_free_kb "$fluxer_record")
 		case ${fluxer_free:-} in
@@ -1065,6 +1277,7 @@ fluxer_copy_volumes() {
 		if [ "$fluxer_free" -lt "$fluxer_need" ]; then
 			fluxer_fail 7 "$fluxer_full holds $((fluxer_size / 1024)) MB and $fluxer_record has $((fluxer_free / 1024)) MB free. Point --backup-dir at a filesystem with room, or pass --no-volume-backup to take the database dump alone."
 		fi
+		printf '%s\n' "$fluxer_volume" >> "$fluxer_scratch/copy-volumes"
 		fluxer_copy_any=1
 	done < "$fluxer_scratch/backup-volumes"
 	if [ "$fluxer_copy_any" -eq 0 ]; then
@@ -1082,7 +1295,7 @@ fluxer_copy_volumes() {
 			docker compose up -d --remove-orphans || true
 			fluxer_fail 7 "Copying $fluxer_full failed. The stack is started again on the images it was running."
 		fi
-	done < "$fluxer_scratch/backup-volumes"
+	done < "$fluxer_scratch/copy-volumes"
 	fluxer_say 'Starting the stack again before the upgrade continues.'
 	if ! docker compose up -d --remove-orphans; then
 		fluxer_fail 7 "docker compose up -d failed in $opt_dir after the copy. Read docker compose logs there."
@@ -1153,12 +1366,16 @@ fluxer_note_mounted_changes() {
 # By hand:
 #   docker compose config --services
 fluxer_compose_services() {
-	docker compose config --services 2>/dev/null | sort -u
+	fluxer_compose_query --services || return 1
+	sort -u "$fluxer_scratch/compose-out"
 }
 
 fluxer_restart_mounted() {
 	[ -s "$fluxer_scratch/restart" ] || return 0
-	fluxer_compose_services > "$fluxer_scratch/services"
+	if ! fluxer_compose_services > "$fluxer_scratch/services"; then
+		fluxer_fail 6 "docker compose config --services failed in $opt_dir, so the services that mount a refreshed file cannot be restarted. Compose printed:
+$(fluxer_compose_error '  ')"
+	fi
 	while read -r fluxer_file fluxer_service; do
 		[ -n "$fluxer_service" ] || continue
 		if ! grep -qxF "$fluxer_service" "$fluxer_scratch/services"; then
@@ -1214,9 +1431,16 @@ fluxer_plan_update() {
 	fluxer_say "  ref           $opt_ref"
 	fluxer_say "  image tag     $(fluxer_env_value FLUXER_IMAGE_TAG) from .env"
 	fluxer_say "  backup dir    $opt_backup_dir"
-	fluxer_say '  running now'
 	fluxer_running_image_ids > "$fluxer_scratch/running"
-	fluxer_compose_images > "$fluxer_scratch/refs" || true
+	if ! fluxer_compose_images > "$fluxer_scratch/refs"; then
+		fluxer_say '  refusal       docker compose config --images fails here, and step 1 of the upgrade reads that list'
+		fluxer_say '  compose said'
+		fluxer_compose_error '    '
+		fluxer_say '  outcome       the run stops on step 1 and changes nothing'
+		fluxer_say 'Nothing outside a temporary directory was written. Fix what Compose reports, then run this again.'
+		return 0
+	fi
+	fluxer_say '  running now'
 	while read -r fluxer_ref; do
 		[ -n "$fluxer_ref" ] || continue
 		fluxer_id=$(fluxer_recorded_id_for "$fluxer_ref")
@@ -1307,7 +1531,7 @@ fluxer_plan_rollback() {
 fluxer_run_update() {
 	fluxer_open_scratch "$opt_dir"
 	fluxer_stack_files > "$fluxer_scratch/files"
-	fluxer_prepare_record
+	fluxer_fill_required_secrets
 	fluxer_record_state
 	fluxer_save_current_files
 	fluxer_backup
@@ -1357,7 +1581,7 @@ fluxer_run_update() {
 
 # Puts one line of .env back, and proves it touched nothing else.
 #
-# FLUXER_IMAGE_TAG is the only key either upgrade mode writes. Every other line,
+# FLUXER_IMAGE_TAG is the only key this function writes. Every other line,
 # which is every secret, is compared byte for byte before the new file replaces
 # the old one, so a rewrite that lost or changed a secret cannot land.
 fluxer_set_image_tag() {
@@ -1473,6 +1697,7 @@ fluxer_resolve_values
 if [ "$opt_update" -eq 1 ] || [ "$opt_rollback" -eq 1 ]; then
 	fluxer_require_instance
 	fluxer_resolve_ref
+	fluxer_require_compose_files
 	cd "$opt_dir"
 	fluxer_set_project
 	if [ "$opt_dry_run" -eq 1 ]; then