From bdacaea4a836249dbef7891763713489b596066c Mon Sep 17 00:00:00 2001 From: Hampus Date: Thu, 20 Aug 2026 21:34:19 +0200 Subject: [PATCH] feat(config): add an optional separate s3 provider for downloads (#1801) --- packages/config/src/MasterConfig.ts | 8 +++ packages/config/src/S3DownloadsProvider.ts | 47 ++++++++++++++ .../src/__tests__/S3DownloadsProvider.test.ts | 63 +++++++++++++++++++ .../src/config_loader/EnvironmentOverrides.ts | 6 ++ 4 files changed, 124 insertions(+) create mode 100644 packages/config/src/S3DownloadsProvider.ts create mode 100644 packages/config/src/__tests__/S3DownloadsProvider.test.ts diff --git a/packages/config/src/MasterConfig.ts b/packages/config/src/MasterConfig.ts index e99118001..872fab115 100644 --- a/packages/config/src/MasterConfig.ts +++ b/packages/config/src/MasterConfig.ts @@ -79,6 +79,14 @@ export interface MasterConfig { static: string; }; }; + s3_downloads?: { + endpoint: string; + presigned_url_base?: string; + force_path_style?: boolean; + region?: string; + access_key_id?: string; + secret_access_key?: string; + }; services: { api: { port: number; diff --git a/packages/config/src/S3DownloadsProvider.ts b/packages/config/src/S3DownloadsProvider.ts new file mode 100644 index 000000000..a02d273d2 --- /dev/null +++ b/packages/config/src/S3DownloadsProvider.ts @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {MasterConfig} from '@fluxer/config/src/MasterConfig'; + +interface S3ProviderSettings { + endpoint: string; + presignedUrlBase?: string; + forcePathStyle: boolean; + region: string; + accessKeyId: string; + secretAccessKey: string; +} + +interface ResolvedDownloadsProvider { + settings: S3ProviderSettings; + isOverridden: boolean; +} + +export function resolveDownloadsProvider(master: Pick): ResolvedDownloadsProvider { + const base = master.s3; + if (!base) { + throw new Error('S3 configuration is required to resolve the downloads provider'); + } + const baseSettings: S3ProviderSettings = { + endpoint: base.endpoint, + presignedUrlBase: base.presigned_url_base, + forcePathStyle: base.force_path_style, + region: base.region, + accessKeyId: base.access_key_id, + secretAccessKey: base.secret_access_key, + }; + const override = master.s3_downloads; + if (!override?.endpoint) { + return {settings: baseSettings, isOverridden: false}; + } + return { + settings: { + endpoint: override.endpoint, + presignedUrlBase: override.presigned_url_base ?? undefined, + forcePathStyle: override.force_path_style ?? baseSettings.forcePathStyle, + region: override.region ?? baseSettings.region, + accessKeyId: override.access_key_id ?? baseSettings.accessKeyId, + secretAccessKey: override.secret_access_key ?? baseSettings.secretAccessKey, + }, + isOverridden: true, + }; +} diff --git a/packages/config/src/__tests__/S3DownloadsProvider.test.ts b/packages/config/src/__tests__/S3DownloadsProvider.test.ts new file mode 100644 index 000000000..a8a298404 --- /dev/null +++ b/packages/config/src/__tests__/S3DownloadsProvider.test.ts @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {MasterConfig} from '@fluxer/config/src/MasterConfig'; +import {resolveDownloadsProvider} from '@fluxer/config/src/S3DownloadsProvider'; +import {describe, expect, test} from 'vitest'; + +const base: Pick['s3'] = { + endpoint: 'https://main.example.com', + presigned_url_base: 'https://public.example.com', + force_path_style: true, + region: 'us-east-1', + access_key_id: 'MAIN_KEY', + secret_access_key: 'MAIN_SECRET', + buckets: {cdn: 'cdn', uploads: 'uploads', downloads: 'downloads', reports: 'r', harvests: 'h', static: 's'}, +}; + +describe('resolveDownloadsProvider', () => { + test('falls back to the main provider when no override is set', () => { + const resolved = resolveDownloadsProvider({s3: base}); + expect(resolved.isOverridden).toBe(false); + expect(resolved.settings.endpoint).toBe('https://main.example.com'); + expect(resolved.settings.accessKeyId).toBe('MAIN_KEY'); + expect(resolved.settings.secretAccessKey).toBe('MAIN_SECRET'); + expect(resolved.settings.region).toBe('us-east-1'); + expect(resolved.settings.presignedUrlBase).toBe('https://public.example.com'); + }); + + test('ignores a partial override that does not set an endpoint', () => { + const resolved = resolveDownloadsProvider({s3: base, s3_downloads: {endpoint: '', access_key_id: 'OTHER'}}); + expect(resolved.isOverridden).toBe(false); + expect(resolved.settings.accessKeyId).toBe('MAIN_KEY'); + }); + + test('uses the override provider when an endpoint is set', () => { + const resolved = resolveDownloadsProvider({ + s3: base, + s3_downloads: { + endpoint: 'https://downloads.example.net', + region: 'eu-central-1', + access_key_id: 'DL_KEY', + secret_access_key: 'DL_SECRET', + }, + }); + expect(resolved.isOverridden).toBe(true); + expect(resolved.settings.endpoint).toBe('https://downloads.example.net'); + expect(resolved.settings.region).toBe('eu-central-1'); + expect(resolved.settings.accessKeyId).toBe('DL_KEY'); + }); + + test('inherits unspecified fields from the main provider', () => { + const resolved = resolveDownloadsProvider({s3: base, s3_downloads: {endpoint: 'https://downloads.example.net'}}); + expect(resolved.isOverridden).toBe(true); + expect(resolved.settings.endpoint).toBe('https://downloads.example.net'); + expect(resolved.settings.region).toBe('us-east-1'); + expect(resolved.settings.accessKeyId).toBe('MAIN_KEY'); + expect(resolved.settings.forcePathStyle).toBe(true); + }); + + test('does not inherit the main presigned base for an override provider', () => { + const resolved = resolveDownloadsProvider({s3: base, s3_downloads: {endpoint: 'https://downloads.example.net'}}); + expect(resolved.settings.presignedUrlBase).toBeUndefined(); + }); +}); diff --git a/packages/config/src/config_loader/EnvironmentOverrides.ts b/packages/config/src/config_loader/EnvironmentOverrides.ts index ce3cc1cc3..eadb90753 100644 --- a/packages/config/src/config_loader/EnvironmentOverrides.ts +++ b/packages/config/src/config_loader/EnvironmentOverrides.ts @@ -65,6 +65,12 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record = { FLUXER_S3_BUCKET_REPORTS: {path: ['s3', 'buckets', 'reports']}, FLUXER_S3_BUCKET_HARVESTS: {path: ['s3', 'buckets', 'harvests']}, FLUXER_S3_BUCKET_STATIC: {path: ['s3', 'buckets', 'static']}, + FLUXER_S3_DOWNLOADS_ENDPOINT: {path: ['s3_downloads', 'endpoint']}, + FLUXER_S3_DOWNLOADS_PUBLIC_ENDPOINT: {path: ['s3_downloads', 'presigned_url_base']}, + FLUXER_S3_DOWNLOADS_FORCE_PATH_STYLE: {path: ['s3_downloads', 'force_path_style'], parse: parseEnvValue}, + FLUXER_S3_DOWNLOADS_REGION: {path: ['s3_downloads', 'region']}, + FLUXER_S3_DOWNLOADS_ACCESS_KEY_ID: {path: ['s3_downloads', 'access_key_id']}, + FLUXER_S3_DOWNLOADS_SECRET_ACCESS_KEY: {path: ['s3_downloads', 'secret_access_key']}, FLUXER_NATS_URL: {path: ['services', 'nats', 'core_url']}, FLUXER_NATS_CORE_URL: {path: ['services', 'nats', 'core_url']}, FLUXER_NATS_JETSTREAM_URL: {path: ['services', 'nats', 'jetstream_url']},