fix(self-host): keep seaweedfs inside its memory ceiling (#2579)

deploy/self-hosting/.env.example

@@ -188,7 +188,7 @@ FLUXER_CAPTCHA_TURNSTILE_SITE_KEY=
 FLUXER_CAPTCHA_TURNSTILE_SECRET_KEY=
 FLUXER_DISCOVERY_ENABLED=true
 
-# Container memory. The 25 limits sum to 16.75 GiB, which is a sum of ceilings and
+# Container memory. The 25 limits sum to 18.25 GiB, which is a sum of ceilings and
 # not an allocation, so the defaults fit a host with 8 GB and are sized for 16 GB.
 # The four reservations are cgroup memory.low, which biases the kernel away from
 # reclaiming from the services whose death takes the whole instance down. They do
@@ -199,7 +199,7 @@ FLUXER_DISCOVERY_ENABLED=true
 #FLUXER_VALKEY_MEMORY_LIMIT=256mb
 #FLUXER_NATS_MEMORY_LIMIT=256mb
 #FLUXER_MEILISEARCH_MEMORY_LIMIT=768mb
-#FLUXER_SEAWEEDFS_MEMORY_LIMIT=512mb
+#FLUXER_SEAWEEDFS_MEMORY_LIMIT=2gb
 #FLUXER_SEAWEEDFS_INIT_MEMORY_LIMIT=128mb
 #FLUXER_LIVEKIT_MEMORY_LIMIT=512mb
 #FLUXER_API_MEMORY_LIMIT=2560mb
@@ -227,6 +227,14 @@ FLUXER_DISCOVERY_ENABLED=true
 # which is the container ceiling the indexer shares with the search process.
 #FLUXER_MEILISEARCH_MAX_INDEXING_MEMORY=384mb
 
+# SeaweedFS heap ceiling. Go collects against this value instead of against the
+# container limit, which it cannot see, so without it an upload burst grows the
+# heap past FLUXER_SEAWEEDFS_MEMORY_LIMIT and the kernel OOM-kills the container
+# mid-upload (exit 137). Keep it near three quarters of that limit, and raise both
+# together: the peak is the parts of one upload in flight at once, which is 25 MB
+# times 20 for a 500 MB attachment.
+#FLUXER_SEAWEEDFS_GOMEMLIMIT=1536MiB
+
 # Node sizes its own heap from the container memory limit by default, at roughly
 # 55 percent of it, which always leaves room for the buffers and stacks that live
 # outside the heap. Leave these unset unless you have a reason to pin the value.

deploy/self-hosting/docker-compose.yml

@@ -258,9 +258,11 @@ services:
     deploy:
       resources:
         limits:
-          memory: ${FLUXER_SEAWEEDFS_MEMORY_LIMIT:-512mb}
+          memory: ${FLUXER_SEAWEEDFS_MEMORY_LIMIT:-2gb}
     restart: unless-stopped
     networks: [fluxer]
+    environment:
+      GOMEMLIMIT: ${FLUXER_SEAWEEDFS_GOMEMLIMIT:-1536MiB}
     command: ["server", "-s3", "-dir=/data"]
     volumes:
       - seaweedfs-data:/data

fluxer_docs/src/content/docs/operator/configuration.mdx

@@ -1769,7 +1769,11 @@ Default `384mb`. Becomes `MEILI_MAX_INDEXING_MEMORY`. The memory the indexer may
 
 #### `FLUXER_SEAWEEDFS_MEMORY_LIMIT`
 
-Default `512mb`. The ceiling for `seaweedfs`. One process runs the master, the volume server and the S3 gateway.
+Default `2gb`. The ceiling for `seaweedfs`. One process runs the master, the volume server and the S3 gateway. The peak is the parts of one upload in flight at once, which the client uploads in parallel: a 500 MB attachment is 20 parts of 25 MB.
+
+#### `FLUXER_SEAWEEDFS_GOMEMLIMIT`
+
+Default `1536MiB`. The heap ceiling the Go runtime collects against. Go cannot see the container limit, so without this value an upload burst grows the heap past `FLUXER_SEAWEEDFS_MEMORY_LIMIT` and the kernel OOM-kills the container mid-upload with exit 137. Raising `FLUXER_SEAWEEDFS_MEMORY_LIMIT` on its own does not fix that, because the runtime grows to fill whatever it is given. Keep this near three quarters of the container limit and move the two together.
 
 #### `FLUXER_SEAWEEDFS_INIT_MEMORY_LIMIT`
 
@@ -1930,10 +1934,12 @@ FLUXER_GATEWAY_MEMORY_LIMIT=512mb
 FLUXER_GATEWAY_MEMORY_RESERVATION=256mb
 FLUXER_MEILISEARCH_MEMORY_LIMIT=512mb
 FLUXER_MEILISEARCH_MAX_INDEXING_MEMORY=256mb
+FLUXER_SEAWEEDFS_MEMORY_LIMIT=1gb
+FLUXER_SEAWEEDFS_GOMEMLIMIT=768MiB
 FLUXER_GIFS_SHARD_CACHE_MAX_BYTES=134217728
 ```
 
-At 8 GB the api and worker heap ceilings fall to 664 MB each and Postgres caches less of the working set, which shows up as slower search and slower history scrolling under load. Nothing is turned off.
+At 8 GB the api and worker heap ceilings fall to 664 MB each and Postgres caches less of the working set, which shows up as slower search and slower history scrolling under load. Nothing is turned off, though the largest attachments need the 16 GB SeaweedFS ceiling to upload every part at once.
 
 The 4 GB profile trades more.
 
@@ -1954,6 +1960,7 @@ FLUXER_MEILISEARCH_MEMORY_LIMIT=384mb
 FLUXER_MEILISEARCH_MAX_INDEXING_MEMORY=128mb
 FLUXER_MEDIA_PROXY_MEMORY_LIMIT=320mb
 FLUXER_SEAWEEDFS_MEMORY_LIMIT=320mb
+FLUXER_SEAWEEDFS_GOMEMLIMIT=240MiB
 FLUXER_LIVEKIT_MEMORY_LIMIT=320mb
 FLUXER_VALKEY_MEMORY_LIMIT=192mb
 FLUXER_VALKEY_MAXMEMORY=128mb