feat(config): add an optional separate s3 provider for downloads (#1801)

packages/config/src/MasterConfig.ts

@@ -79,6 +79,14 @@ export interface MasterConfig {
 			static: string;
 		};
 	};
+	s3_downloads?: {
+		endpoint: string;
+		presigned_url_base?: string;
+		force_path_style?: boolean;
+		region?: string;
+		access_key_id?: string;
+		secret_access_key?: string;
+	};
 	services: {
 		api: {
 			port: number;

packages/config/src/S3DownloadsProvider.ts

@@ -0,0 +1,47 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import type {MasterConfig} from '@fluxer/config/src/MasterConfig';
+
+interface S3ProviderSettings {
+	endpoint: string;
+	presignedUrlBase?: string;
+	forcePathStyle: boolean;
+	region: string;
+	accessKeyId: string;
+	secretAccessKey: string;
+}
+
+interface ResolvedDownloadsProvider {
+	settings: S3ProviderSettings;
+	isOverridden: boolean;
+}
+
+export function resolveDownloadsProvider(master: Pick<MasterConfig, 's3' | 's3_downloads'>): ResolvedDownloadsProvider {
+	const base = master.s3;
+	if (!base) {
+		throw new Error('S3 configuration is required to resolve the downloads provider');
+	}
+	const baseSettings: S3ProviderSettings = {
+		endpoint: base.endpoint,
+		presignedUrlBase: base.presigned_url_base,
+		forcePathStyle: base.force_path_style,
+		region: base.region,
+		accessKeyId: base.access_key_id,
+		secretAccessKey: base.secret_access_key,
+	};
+	const override = master.s3_downloads;
+	if (!override?.endpoint) {
+		return {settings: baseSettings, isOverridden: false};
+	}
+	return {
+		settings: {
+			endpoint: override.endpoint,
+			presignedUrlBase: override.presigned_url_base ?? undefined,
+			forcePathStyle: override.force_path_style ?? baseSettings.forcePathStyle,
+			region: override.region ?? baseSettings.region,
+			accessKeyId: override.access_key_id ?? baseSettings.accessKeyId,
+			secretAccessKey: override.secret_access_key ?? baseSettings.secretAccessKey,
+		},
+		isOverridden: true,
+	};
+}

packages/config/src/__tests__/S3DownloadsProvider.test.ts

@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import type {MasterConfig} from '@fluxer/config/src/MasterConfig';
+import {resolveDownloadsProvider} from '@fluxer/config/src/S3DownloadsProvider';
+import {describe, expect, test} from 'vitest';
+
+const base: Pick<MasterConfig, 's3' | 's3_downloads'>['s3'] = {
+	endpoint: 'https://main.example.com',
+	presigned_url_base: 'https://public.example.com',
+	force_path_style: true,
+	region: 'us-east-1',
+	access_key_id: 'MAIN_KEY',
+	secret_access_key: 'MAIN_SECRET',
+	buckets: {cdn: 'cdn', uploads: 'uploads', downloads: 'downloads', reports: 'r', harvests: 'h', static: 's'},
+};
+
+describe('resolveDownloadsProvider', () => {
+	test('falls back to the main provider when no override is set', () => {
+		const resolved = resolveDownloadsProvider({s3: base});
+		expect(resolved.isOverridden).toBe(false);
+		expect(resolved.settings.endpoint).toBe('https://main.example.com');
+		expect(resolved.settings.accessKeyId).toBe('MAIN_KEY');
+		expect(resolved.settings.secretAccessKey).toBe('MAIN_SECRET');
+		expect(resolved.settings.region).toBe('us-east-1');
+		expect(resolved.settings.presignedUrlBase).toBe('https://public.example.com');
+	});
+
+	test('ignores a partial override that does not set an endpoint', () => {
+		const resolved = resolveDownloadsProvider({s3: base, s3_downloads: {endpoint: '', access_key_id: 'OTHER'}});
+		expect(resolved.isOverridden).toBe(false);
+		expect(resolved.settings.accessKeyId).toBe('MAIN_KEY');
+	});
+
+	test('uses the override provider when an endpoint is set', () => {
+		const resolved = resolveDownloadsProvider({
+			s3: base,
+			s3_downloads: {
+				endpoint: 'https://downloads.example.net',
+				region: 'eu-central-1',
+				access_key_id: 'DL_KEY',
+				secret_access_key: 'DL_SECRET',
+			},
+		});
+		expect(resolved.isOverridden).toBe(true);
+		expect(resolved.settings.endpoint).toBe('https://downloads.example.net');
+		expect(resolved.settings.region).toBe('eu-central-1');
+		expect(resolved.settings.accessKeyId).toBe('DL_KEY');
+	});
+
+	test('inherits unspecified fields from the main provider', () => {
+		const resolved = resolveDownloadsProvider({s3: base, s3_downloads: {endpoint: 'https://downloads.example.net'}});
+		expect(resolved.isOverridden).toBe(true);
+		expect(resolved.settings.endpoint).toBe('https://downloads.example.net');
+		expect(resolved.settings.region).toBe('us-east-1');
+		expect(resolved.settings.accessKeyId).toBe('MAIN_KEY');
+		expect(resolved.settings.forcePathStyle).toBe(true);
+	});
+
+	test('does not inherit the main presigned base for an override provider', () => {
+		const resolved = resolveDownloadsProvider({s3: base, s3_downloads: {endpoint: 'https://downloads.example.net'}});
+		expect(resolved.settings.presignedUrlBase).toBeUndefined();
+	});
+});

packages/config/src/config_loader/EnvironmentOverrides.ts

@@ -65,6 +65,12 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
 	FLUXER_S3_BUCKET_REPORTS: {path: ['s3', 'buckets', 'reports']},
 	FLUXER_S3_BUCKET_HARVESTS: {path: ['s3', 'buckets', 'harvests']},
 	FLUXER_S3_BUCKET_STATIC: {path: ['s3', 'buckets', 'static']},
+	FLUXER_S3_DOWNLOADS_ENDPOINT: {path: ['s3_downloads', 'endpoint']},
+	FLUXER_S3_DOWNLOADS_PUBLIC_ENDPOINT: {path: ['s3_downloads', 'presigned_url_base']},
+	FLUXER_S3_DOWNLOADS_FORCE_PATH_STYLE: {path: ['s3_downloads', 'force_path_style'], parse: parseEnvValue},
+	FLUXER_S3_DOWNLOADS_REGION: {path: ['s3_downloads', 'region']},
+	FLUXER_S3_DOWNLOADS_ACCESS_KEY_ID: {path: ['s3_downloads', 'access_key_id']},
+	FLUXER_S3_DOWNLOADS_SECRET_ACCESS_KEY: {path: ['s3_downloads', 'secret_access_key']},
 	FLUXER_NATS_URL: {path: ['services', 'nats', 'core_url']},
 	FLUXER_NATS_CORE_URL: {path: ['services', 'nats', 'core_url']},
 	FLUXER_NATS_JETSTREAM_URL: {path: ['services', 'nats', 'jetstream_url']},