fix(self-hosting): grow seaweedfs one volume at a time (#3076)

deploy/self-hosting/.env.example

@@ -448,6 +448,11 @@ FLUXER_DISCOVERY_ENABLED=true
 #FLUXER_SEAWEEDFS_GOMEMLIMIT=1536MiB
 #FLUXER_SEAWEEDFS_TELEMETRY=false
 
+# Volumes SeaweedFS creates at once when a bucket needs space. Each reserves 1 GB
+# of free disk from the start, and SeaweedFS's own default of 7 fills a small
+# disk before every bucket has one, so uploads fail with no free volumes left.
+#FLUXER_SEAWEEDFS_VOLUME_GROWTH=1
+
 # Node sizes its heap from the container limit by default. Leave these unset
 # unless you need to pin it. A heap ceiling above the container limit gets the
 # container OOM-killed instead of reporting a heap error. The values below are

deploy/self-hosting/docker-compose.yml

@@ -353,6 +353,7 @@ services:
           memory: ${FLUXER_SEAWEEDFS_MEMORY_LIMIT:-2gb}
     environment:
       GOMEMLIMIT: ${FLUXER_SEAWEEDFS_GOMEMLIMIT:-1536MiB}
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: ${FLUXER_SEAWEEDFS_VOLUME_GROWTH:-1}
     command: ["server", "-s3", "-dir=/data", "-master.telemetry=${FLUXER_SEAWEEDFS_TELEMETRY:-false}"]
     volumes:
       - seaweedfs-data:/data

fluxer_api/src/api/worker/tasks/SendSystemDm.ts

@@ -6,7 +6,7 @@ import type {User} from '@app/api/models/User';
 import {UserChannelService} from '@app/api/user/services/UserChannelService';
 import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
 import {UserFlags} from '@fluxer/constants/src/UserConstants';
-import {JobCancelledError, type WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
+import {JobCancelledError, type WorkerTaskHelpers, type WorkerTaskResult} from '@pkgs/worker/src/contracts/WorkerTask';
 import {z} from 'zod';
 
 const SYSTEM_USER_ID: UserID = createUserID(0n);
@@ -56,7 +56,7 @@ async function* listedRecipients(userIds: Array<string>): AsyncGenerator<UserID>
 	}
 }
 
-export async function sendSystemDm(payload: unknown, helpers: WorkerTaskHelpers): Promise<void> {
+export async function sendSystemDm(payload: unknown, helpers: WorkerTaskHelpers): Promise<WorkerTaskResult> {
 	const parsed = PayloadSchema.parse(payload);
 	const {content} = parsed;
 	const total = 'user_ids' in parsed ? parsed.user_ids.length : null;
@@ -105,5 +105,7 @@ export async function sendSystemDm(payload: unknown, helpers: WorkerTaskHelpers)
 		}
 	}
 	requestCache.clear();
+	await helpers.reportProgress(sent + failed, sent + failed, `${sent} sent, ${failed} failed`);
 	helpers.logger.info({sent, failed, total: sent + failed}, 'System DM job complete');
+	return {sent_count: sent, failed_count: failed};
 }

fluxer_api/src/api/worker/tests/SendSystemDmCancel.test.ts

@@ -219,7 +219,7 @@ describe('System DM cancellation', () => {
 		await expect(runner.runJob(TASK_TYPE, createJobMessage({all_users: true}) as unknown as JsMsg)).resolves.toBe(true);
 
 		expect(recipientIds).toEqual([21n, 24n]);
-		expect(markSucceeded).toHaveBeenCalledTimes(1);
+		expect(markSucceeded).toHaveBeenCalledWith(LEDGER_JOB_ID, {sent_count: 2, failed_count: 0});
 		expect(kv.size).toBe(0);
 	});
 });

fluxer_docs/src/content/docs/operator/configuration.mdx

@@ -1834,6 +1834,10 @@ Default `1536MiB`. The heap ceiling the Go runtime collects against. Go cannot s
 
 Default `false`. Becomes the `-master.telemetry` flag of `seaweedfs`, which reports usage to the SeaweedFS project when on.
 
+#### `FLUXER_SEAWEEDFS_VOLUME_GROWTH`
+
+Default `1`. Becomes `WEED_MASTER_VOLUME_GROWTH_COPY_1`, the number of volumes SeaweedFS creates at once when a bucket runs out of room. Each volume reserves a 1 GB slot of free disk, and the stack writes to four buckets. SeaweedFS's own default of 7 needs about 28 GB free before every bucket has a volume, so on a smaller disk the first bucket takes every slot and uploads fail with `No writable volumes and no free volumes left`.
+
 #### `FLUXER_SEAWEEDFS_INIT_MEMORY_LIMIT`
 
 Default `128mb`. The ceiling for `seaweedfs-init`. A one-shot container that exits, so it never overlaps steady state.

packages/config/src/__tests__/SelfHostingComposeEnvironment.test.ts

@@ -49,6 +49,7 @@ const INPUT_NAMES: Record<string, string> = {
 	MEILI_NO_ANALYTICS: 'FLUXER_MEILISEARCH_NO_ANALYTICS',
 	MEILI_MAX_INDEXING_MEMORY: 'FLUXER_MEILISEARCH_MAX_INDEXING_MEMORY',
 	GOMEMLIMIT: 'FLUXER_SEAWEEDFS_GOMEMLIMIT',
+	WEED_MASTER_VOLUME_GROWTH_COPY_1: 'FLUXER_SEAWEEDFS_VOLUME_GROWTH',
 	LIVEKIT_KEYS: 'LIVEKIT_API_KEY',
 	NODE_EXTRA_CA_CERTS: 'FLUXER_NODE_EXTRA_CA_CERTS',
 };