fix(installer): stop upgrades hanging after the uploads copy (#3478)

deploy/self-hosting/docker-compose.yml

@@ -215,7 +215,7 @@ x-fluxer-app-healthcheck: &fluxer-app-healthcheck
 x-fluxer-svc-healthcheck: &fluxer-svc-healthcheck
   <<: *fluxer-app-healthcheck
   start_period: ${FLUXER_SVC_HEALTHCHECK_START_PERIOD:-60s}
-  test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/8090 && printf 'GET /_health HTTP/1.0\\r\\n\\r\\n' >&3 && head -n 1 <&3 | grep -q ' 200 '"]
+  test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/8090 && printf \"GET /_health HTTP/1.0\\r\\n\\r\\n\" >&3 && head -n 1 <&3 | grep -q \" 200 \"'"]
 
 services:
   edge:
@@ -469,7 +469,7 @@ services:
       <<: *fluxer-env
       FLUXER_API_PORT: "8080"
       FLUXER_SETUP_TOKEN: ${FLUXER_SETUP_TOKEN:-}
-      NODE_OPTIONS: --enable-source-maps${FLUXER_API_NODE_HEAP_MB:+ --max-old-space-size=$FLUXER_API_NODE_HEAP_MB}${FLUXER_API_NODE_OPTIONS:+ $FLUXER_API_NODE_OPTIONS}
+      NODE_OPTIONS: --enable-source-maps${FLUXER_API_NODE_HEAP_MB:+ --max-old-space-size=${FLUXER_API_NODE_HEAP_MB:-}}${FLUXER_API_NODE_OPTIONS:+ ${FLUXER_API_NODE_OPTIONS:-}}
       NODE_EXTRA_CA_CERTS: ${FLUXER_NODE_EXTRA_CA_CERTS:-/etc/ssl/certs/ca-certificates.crt}
       FLUXER_POSTGRES_MAX_CONNECTIONS: "${FLUXER_API_POSTGRES_MAX_CONNECTIONS:-25}"
       FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED: "${FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED:-true}"
@@ -503,13 +503,13 @@ services:
     command: ["node", "dist/WorkerEntrypoint.js"]
     environment:
       <<: *fluxer-env
-      NODE_OPTIONS: --enable-source-maps${FLUXER_WORKER_NODE_HEAP_MB:+ --max-old-space-size=$FLUXER_WORKER_NODE_HEAP_MB}${FLUXER_WORKER_NODE_OPTIONS:+ $FLUXER_WORKER_NODE_OPTIONS}
+      NODE_OPTIONS: --enable-source-maps${FLUXER_WORKER_NODE_HEAP_MB:+ --max-old-space-size=${FLUXER_WORKER_NODE_HEAP_MB:-}}${FLUXER_WORKER_NODE_OPTIONS:+ ${FLUXER_WORKER_NODE_OPTIONS:-}}
       NODE_EXTRA_CA_CERTS: ${FLUXER_NODE_EXTRA_CA_CERTS:-/etc/ssl/certs/ca-certificates.crt}
       FLUXER_API_WORKER_MODE: all_lanes
       FLUXER_API_WORKER_ENABLE_CRON_SCHEDULER: "true"
       FLUXER_POSTGRES_MAX_CONNECTIONS: "${FLUXER_WORKER_POSTGRES_MAX_CONNECTIONS:-25}"
     healthcheck:
-      test: ["CMD", "node", "-e", "const age=Date.now()-require('node:fs').statSync('/tmp/fluxer-worker-heartbeat').mtimeMs;if(age>30000){console.error('worker heartbeat is '+Math.round(age)+'ms old');process.exit(1)}"]
+      test: ["CMD-SHELL", "node -e \"const age=Date.now()-require('node:fs').statSync('/tmp/fluxer-worker-heartbeat').mtimeMs;if(age>30000){console.error('worker heartbeat is '+Math.round(age)+'ms old');process.exit(1)}\""]
       <<: *fluxer-app-healthcheck
       retries: ${FLUXER_WORKER_HEALTHCHECK_RETRIES:-3}
     depends_on:
@@ -867,7 +867,7 @@ services:
       FLUXER_API_ENDPOINT: http://api:8080
       FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_STATIC_CDN_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}}
     healthcheck:
-      test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/8080 && printf 'GET /_health HTTP/1.0\\r\\n\\r\\n' >&3 && head -n 1 <&3 | grep -q ' 200 '"]
+      test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/8080 && printf \"GET /_health HTTP/1.0\\r\\n\\r\\n\" >&3 && head -n 1 <&3 | grep -q \" 200 \"'"]
       <<: *fluxer-app-healthcheck
       start_period: ${FLUXER_SVC_HEALTHCHECK_START_PERIOD:-60s}
     depends_on:

fluxer_docs/src/installer/install.ps1

@@ -1454,6 +1454,17 @@ function Get-FluxerFreeKb([string]$Path) {
 	return [long]($drive.AvailableFreeSpace / 1024)
 }
 
+function Start-FluxerStopped([string[]]$Ids) {
+	if ($Ids.Count -eq 0) {
+		return 0
+	}
+	$result = Invoke-FluxerCapture (@('start') + $Ids)
+	if ($result.Code -ne 0 -and $result.Error.Length -gt 0) {
+		Write-FluxerProblem $result.Error
+	}
+	return $result.Code
+}
+
 # Step 2 continued: the volume copy.
 #
 # A live copy can catch a file mid-write, so the stack stops for it. The stack comes back up on
@@ -1461,9 +1472,10 @@ function Get-FluxerFreeKb([string]$Path) {
 # working instance rather than a stopped one.
 #
 # By hand:
+#   docker compose ps -q > running
 #   docker compose stop
 #   docker run --rm -v fluxer_seaweedfs-data:/data -v "${PWD}\backups:/backup" alpine tar czf /backup/seaweedfs-data.tgz -C /data .
-#   docker compose up -d
+#   docker start (Get-Content running)
 function Copy-FluxerVolumes([string]$Record, [string]$Project, [string]$TargetDir) {
 	if (-not (Test-FluxerStackDefinesService 'seaweedfs' $TargetDir)) {
 		Write-FluxerLine 'Skipping the uploads copy. This stack defines no seaweedfs service, so its objects are outside the stack and only the operator of that store can copy them.'
@@ -1497,6 +1509,11 @@ function Copy-FluxerVolumes([string]$Record, [string]$Project, [string]$TargetDi
 		$tarExtension = 'tar'
 	}
 	Write-FluxerLine 'Stopping the stack for a consistent copy of the uploads.'
+	$running = Invoke-FluxerCapture @('compose', 'ps', '-q')
+	if ($running.Code -ne 0) {
+		Stop-Fluxer "docker compose ps failed, so the containers to start again after the copy cannot be read. Compose printed:`n$(Get-FluxerIndented $running.Error '  ')" $FluxerExitBackup
+	}
+	$stopped = @($running.Text.Split("`n") | ForEach-Object {$_.Trim()} | Where-Object {$_.Length -gt 0})
 	if ((Invoke-FluxerDocker @('compose', 'stop')) -ne 0) {
 		Stop-Fluxer 'docker compose stop failed.' $FluxerExitBackup
 	}
@@ -1505,13 +1522,13 @@ function Copy-FluxerVolumes([string]$Record, [string]$Project, [string]$TargetDi
 		Write-FluxerLine "Copying $full."
 		$code = Invoke-FluxerDocker @('run', '--rm', '-v', "${full}:/data:ro", '-v', "${Record}:/backup", $FluxerHelperImage, 'tar', $tarFlags, "/backup/$volume.$tarExtension", '-C', '/data', '.')
 		if ($code -ne 0) {
-			[void](Invoke-FluxerDocker @('compose', 'up', '-d', '--remove-orphans'))
+			[void](Start-FluxerStopped $stopped)
 			Stop-Fluxer "Copying $full failed. The stack is started again on the images it was running." $FluxerExitBackup
 		}
 	}
 	Write-FluxerLine 'Starting the stack again before the upgrade continues.'
-	if ((Invoke-FluxerDocker @('compose', 'up', '-d', '--remove-orphans')) -ne 0) {
+	if ((Start-FluxerStopped $stopped) -ne 0) {
-		Stop-Fluxer 'docker compose up -d failed after the copy. Read docker compose logs.' $FluxerExitBackup
+		Stop-Fluxer 'docker start failed after the copy. Read docker compose logs.' $FluxerExitBackup
 	}
 }
 

fluxer_docs/src/installer/install.sh

@@ -1686,6 +1686,11 @@ fluxer_free_kb() {
 	df -Pk "$1" 2>/dev/null | awk 'NR==2 {print $4}'
 }
 
+fluxer_start_stopped() {
+	[ -s "$fluxer_scratch/stopped" ] || return 0
+	xargs $fluxer_engine start < "$fluxer_scratch/stopped" > /dev/null
+}
+
 # Step 2 continued: the volume copy.
 #
 # A live copy can catch a file mid-write, so the stack stops for it. The stack
@@ -1693,9 +1698,10 @@ fluxer_free_kb() {
 # happens, so a failure here leaves a working instance rather than a stopped one.
 #
 # By hand:
+#   docker compose ps -q > running
 #   docker compose stop
 #   docker run --rm -v fluxer_seaweedfs-data:/data -v "$PWD/backups:/backup" alpine tar czf /backup/seaweedfs-data.tgz -C /data .
-#   docker compose up -d
+#   xargs docker start < running
 fluxer_copy_volumes() {
 	if ! fluxer_stack_defines_service seaweedfs; then
 		fluxer_say 'Skipping the uploads copy. This stack defines no seaweedfs service, so its objects are outside the stack and only the operator of that store can copy them.'
@@ -1738,6 +1744,9 @@ $(fluxer_volume_error '  ')" ;;
 		fluxer_tar_ext='tgz'
 	fi
 	fluxer_say 'Stopping the stack for a consistent copy of the uploads.'
+	if ! $fluxer_engine compose ps -q > "$fluxer_scratch/stopped"; then
+		fluxer_fail 7 "$fluxer_engine compose ps failed in $opt_dir, so the containers to start again after the copy cannot be read."
+	fi
 	if ! $fluxer_engine compose stop; then
 		fluxer_fail 7 "$fluxer_engine compose stop failed in $opt_dir."
 	fi
@@ -1746,13 +1755,13 @@ $(fluxer_volume_error '  ')" ;;
 		fluxer_full="${fluxer_project}_${fluxer_volume}"
 		fluxer_say "Copying $fluxer_full."
 		if ! $fluxer_engine run --rm -v "$fluxer_full:/data:ro" -v "$fluxer_record:/backup" "$FLUXER_HELPER_IMAGE" tar "$fluxer_tar_flags" "/backup/$fluxer_volume.$fluxer_tar_ext" -C /data .; then
-			$fluxer_engine compose up -d --remove-orphans || true
+			fluxer_start_stopped || true
 			fluxer_fail 7 "Copying $fluxer_full failed. The stack is started again on the images it was running."
 		fi
 	done < "$fluxer_scratch/copy-volumes"
 	fluxer_say 'Starting the stack again before the upgrade continues.'
-	if ! $fluxer_engine compose up -d --remove-orphans; then
+	if ! fluxer_start_stopped; then
-		fluxer_fail 7 "$fluxer_engine compose up -d failed in $opt_dir after the copy. Read $fluxer_engine compose logs there."
+		fluxer_fail 7 "$fluxer_engine start failed in $opt_dir after the copy. Read $fluxer_engine compose logs there."
 	fi
 }