From 812919759aceeb38402522b908e5eed2af328ea7 Mon Sep 17 00:00:00 2001 From: Hampus Date: Wed, 14 Oct 2026 20:37:03 +0200 Subject: [PATCH] fix(installer): stop upgrades hanging after the uploads copy (#3478) --- deploy/self-hosting/docker-compose.yml | 10 +++++----- fluxer_docs/src/installer/install.ps1 | 25 +++++++++++++++++++++---- fluxer_docs/src/installer/install.sh | 17 +++++++++++++---- 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/deploy/self-hosting/docker-compose.yml b/deploy/self-hosting/docker-compose.yml index bfbaba678..cf8e3622c 100644 --- a/deploy/self-hosting/docker-compose.yml +++ b/deploy/self-hosting/docker-compose.yml @@ -215,7 +215,7 @@ x-fluxer-app-healthcheck: &fluxer-app-healthcheck x-fluxer-svc-healthcheck: &fluxer-svc-healthcheck <<: *fluxer-app-healthcheck start_period: ${FLUXER_SVC_HEALTHCHECK_START_PERIOD:-60s} - test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/8090 && printf 'GET /_health HTTP/1.0\\r\\n\\r\\n' >&3 && head -n 1 <&3 | grep -q ' 200 '"] + test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/8090 && printf \"GET /_health HTTP/1.0\\r\\n\\r\\n\" >&3 && head -n 1 <&3 | grep -q \" 200 \"'"] services: edge: @@ -469,7 +469,7 @@ services: <<: *fluxer-env FLUXER_API_PORT: "8080" FLUXER_SETUP_TOKEN: ${FLUXER_SETUP_TOKEN:-} - NODE_OPTIONS: --enable-source-maps${FLUXER_API_NODE_HEAP_MB:+ --max-old-space-size=$FLUXER_API_NODE_HEAP_MB}${FLUXER_API_NODE_OPTIONS:+ $FLUXER_API_NODE_OPTIONS} + NODE_OPTIONS: --enable-source-maps${FLUXER_API_NODE_HEAP_MB:+ --max-old-space-size=${FLUXER_API_NODE_HEAP_MB:-}}${FLUXER_API_NODE_OPTIONS:+ ${FLUXER_API_NODE_OPTIONS:-}} NODE_EXTRA_CA_CERTS: ${FLUXER_NODE_EXTRA_CA_CERTS:-/etc/ssl/certs/ca-certificates.crt} FLUXER_POSTGRES_MAX_CONNECTIONS: "${FLUXER_API_POSTGRES_MAX_CONNECTIONS:-25}" FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED: "${FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED:-true}" @@ -503,13 +503,13 @@ services: command: ["node", "dist/WorkerEntrypoint.js"] environment: <<: *fluxer-env - NODE_OPTIONS: --enable-source-maps${FLUXER_WORKER_NODE_HEAP_MB:+ --max-old-space-size=$FLUXER_WORKER_NODE_HEAP_MB}${FLUXER_WORKER_NODE_OPTIONS:+ $FLUXER_WORKER_NODE_OPTIONS} + NODE_OPTIONS: --enable-source-maps${FLUXER_WORKER_NODE_HEAP_MB:+ --max-old-space-size=${FLUXER_WORKER_NODE_HEAP_MB:-}}${FLUXER_WORKER_NODE_OPTIONS:+ ${FLUXER_WORKER_NODE_OPTIONS:-}} NODE_EXTRA_CA_CERTS: ${FLUXER_NODE_EXTRA_CA_CERTS:-/etc/ssl/certs/ca-certificates.crt} FLUXER_API_WORKER_MODE: all_lanes FLUXER_API_WORKER_ENABLE_CRON_SCHEDULER: "true" FLUXER_POSTGRES_MAX_CONNECTIONS: "${FLUXER_WORKER_POSTGRES_MAX_CONNECTIONS:-25}" healthcheck: - test: ["CMD", "node", "-e", "const age=Date.now()-require('node:fs').statSync('/tmp/fluxer-worker-heartbeat').mtimeMs;if(age>30000){console.error('worker heartbeat is '+Math.round(age)+'ms old');process.exit(1)}"] + test: ["CMD-SHELL", "node -e \"const age=Date.now()-require('node:fs').statSync('/tmp/fluxer-worker-heartbeat').mtimeMs;if(age>30000){console.error('worker heartbeat is '+Math.round(age)+'ms old');process.exit(1)}\""] <<: *fluxer-app-healthcheck retries: ${FLUXER_WORKER_HEALTHCHECK_RETRIES:-3} depends_on: @@ -867,7 +867,7 @@ services: FLUXER_API_ENDPOINT: http://api:8080 FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_STATIC_CDN_ENDPOINT:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}} healthcheck: - test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/8080 && printf 'GET /_health HTTP/1.0\\r\\n\\r\\n' >&3 && head -n 1 <&3 | grep -q ' 200 '"] + test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/8080 && printf \"GET /_health HTTP/1.0\\r\\n\\r\\n\" >&3 && head -n 1 <&3 | grep -q \" 200 \"'"] <<: *fluxer-app-healthcheck start_period: ${FLUXER_SVC_HEALTHCHECK_START_PERIOD:-60s} depends_on: diff --git a/fluxer_docs/src/installer/install.ps1 b/fluxer_docs/src/installer/install.ps1 index 08b87e949..233b5fb68 100644 --- a/fluxer_docs/src/installer/install.ps1 +++ b/fluxer_docs/src/installer/install.ps1 @@ -1454,6 +1454,17 @@ function Get-FluxerFreeKb([string]$Path) { return [long]($drive.AvailableFreeSpace / 1024) } +function Start-FluxerStopped([string[]]$Ids) { + if ($Ids.Count -eq 0) { + return 0 + } + $result = Invoke-FluxerCapture (@('start') + $Ids) + if ($result.Code -ne 0 -and $result.Error.Length -gt 0) { + Write-FluxerProblem $result.Error + } + return $result.Code +} + # Step 2 continued: the volume copy. # # A live copy can catch a file mid-write, so the stack stops for it. The stack comes back up on @@ -1461,9 +1472,10 @@ function Get-FluxerFreeKb([string]$Path) { # working instance rather than a stopped one. # # By hand: +# docker compose ps -q > running # docker compose stop # docker run --rm -v fluxer_seaweedfs-data:/data -v "${PWD}\backups:/backup" alpine tar czf /backup/seaweedfs-data.tgz -C /data . -# docker compose up -d +# docker start (Get-Content running) function Copy-FluxerVolumes([string]$Record, [string]$Project, [string]$TargetDir) { if (-not (Test-FluxerStackDefinesService 'seaweedfs' $TargetDir)) { Write-FluxerLine 'Skipping the uploads copy. This stack defines no seaweedfs service, so its objects are outside the stack and only the operator of that store can copy them.' @@ -1497,6 +1509,11 @@ function Copy-FluxerVolumes([string]$Record, [string]$Project, [string]$TargetDi $tarExtension = 'tar' } Write-FluxerLine 'Stopping the stack for a consistent copy of the uploads.' + $running = Invoke-FluxerCapture @('compose', 'ps', '-q') + if ($running.Code -ne 0) { + Stop-Fluxer "docker compose ps failed, so the containers to start again after the copy cannot be read. Compose printed:`n$(Get-FluxerIndented $running.Error ' ')" $FluxerExitBackup + } + $stopped = @($running.Text.Split("`n") | ForEach-Object {$_.Trim()} | Where-Object {$_.Length -gt 0}) if ((Invoke-FluxerDocker @('compose', 'stop')) -ne 0) { Stop-Fluxer 'docker compose stop failed.' $FluxerExitBackup } @@ -1505,13 +1522,13 @@ function Copy-FluxerVolumes([string]$Record, [string]$Project, [string]$TargetDi Write-FluxerLine "Copying $full." $code = Invoke-FluxerDocker @('run', '--rm', '-v', "${full}:/data:ro", '-v', "${Record}:/backup", $FluxerHelperImage, 'tar', $tarFlags, "/backup/$volume.$tarExtension", '-C', '/data', '.') if ($code -ne 0) { - [void](Invoke-FluxerDocker @('compose', 'up', '-d', '--remove-orphans')) + [void](Start-FluxerStopped $stopped) Stop-Fluxer "Copying $full failed. The stack is started again on the images it was running." $FluxerExitBackup } } Write-FluxerLine 'Starting the stack again before the upgrade continues.' - if ((Invoke-FluxerDocker @('compose', 'up', '-d', '--remove-orphans')) -ne 0) { - Stop-Fluxer 'docker compose up -d failed after the copy. Read docker compose logs.' $FluxerExitBackup + if ((Start-FluxerStopped $stopped) -ne 0) { + Stop-Fluxer 'docker start failed after the copy. Read docker compose logs.' $FluxerExitBackup } } diff --git a/fluxer_docs/src/installer/install.sh b/fluxer_docs/src/installer/install.sh index 3d0dddcc5..b0b404823 100644 --- a/fluxer_docs/src/installer/install.sh +++ b/fluxer_docs/src/installer/install.sh @@ -1686,6 +1686,11 @@ fluxer_free_kb() { df -Pk "$1" 2>/dev/null | awk 'NR==2 {print $4}' } +fluxer_start_stopped() { + [ -s "$fluxer_scratch/stopped" ] || return 0 + xargs $fluxer_engine start < "$fluxer_scratch/stopped" > /dev/null +} + # Step 2 continued: the volume copy. # # A live copy can catch a file mid-write, so the stack stops for it. The stack @@ -1693,9 +1698,10 @@ fluxer_free_kb() { # happens, so a failure here leaves a working instance rather than a stopped one. # # By hand: +# docker compose ps -q > running # docker compose stop # docker run --rm -v fluxer_seaweedfs-data:/data -v "$PWD/backups:/backup" alpine tar czf /backup/seaweedfs-data.tgz -C /data . -# docker compose up -d +# xargs docker start < running fluxer_copy_volumes() { if ! fluxer_stack_defines_service seaweedfs; then fluxer_say 'Skipping the uploads copy. This stack defines no seaweedfs service, so its objects are outside the stack and only the operator of that store can copy them.' @@ -1738,6 +1744,9 @@ $(fluxer_volume_error ' ')" ;; fluxer_tar_ext='tgz' fi fluxer_say 'Stopping the stack for a consistent copy of the uploads.' + if ! $fluxer_engine compose ps -q > "$fluxer_scratch/stopped"; then + fluxer_fail 7 "$fluxer_engine compose ps failed in $opt_dir, so the containers to start again after the copy cannot be read." + fi if ! $fluxer_engine compose stop; then fluxer_fail 7 "$fluxer_engine compose stop failed in $opt_dir." fi @@ -1746,13 +1755,13 @@ $(fluxer_volume_error ' ')" ;; fluxer_full="${fluxer_project}_${fluxer_volume}" fluxer_say "Copying $fluxer_full." if ! $fluxer_engine run --rm -v "$fluxer_full:/data:ro" -v "$fluxer_record:/backup" "$FLUXER_HELPER_IMAGE" tar "$fluxer_tar_flags" "/backup/$fluxer_volume.$fluxer_tar_ext" -C /data .; then - $fluxer_engine compose up -d --remove-orphans || true + fluxer_start_stopped || true fluxer_fail 7 "Copying $fluxer_full failed. The stack is started again on the images it was running." fi done < "$fluxer_scratch/copy-volumes" fluxer_say 'Starting the stack again before the upgrade continues.' - if ! $fluxer_engine compose up -d --remove-orphans; then - fluxer_fail 7 "$fluxer_engine compose up -d failed in $opt_dir after the copy. Read $fluxer_engine compose logs there." + if ! fluxer_start_stopped; then + fluxer_fail 7 "$fluxer_engine start failed in $opt_dir after the copy. Read $fluxer_engine compose logs there." fi }