feat(api): route downloads through the configured downloads provider (#1802)

fluxer_api/src/api/Config.ts

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+import {resolveDownloadsProvider} from '@fluxer/config/src/S3DownloadsProvider';
 import type {MasterConfig} from '@fluxer/config/src/MasterConfig';
 import {parseIpAddress} from '@fluxer/ip_utils/src/IpAddress';
 import {parseGeoipSourceConfig, resolveGeoipRuntimeSourceConfig} from '@pkgs/geoip/src/GeoipStartup';
@@ -237,6 +238,7 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
 			cacheMinTtlSeconds: master.services.api.embeds.cache_min_ttl_seconds,
 			cacheRespectRemoteTtl: master.services.api.embeds.cache_respect_remote_ttl,
 		},
+		s3Downloads: resolveDownloadsProvider(master),
 		s3: {
 			endpoint: s3Config.endpoint,
 			presignedUrlBase: s3Config.presigned_url_base,

fluxer_api/src/api/config/APIConfig.ts

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+import type {ResolvedDownloadsProvider} from '@fluxer/config/src/S3DownloadsProvider';
 import type {WorkerTaskName} from '../worker/WorkerLaneConfig';
 
 export type APIWorkerMode = 'all_lanes' | 'single_lane' | 'single_task';
@@ -150,6 +151,7 @@ export interface APIConfig {
 			static: string;
 		};
 	};
+	s3Downloads: ResolvedDownloadsProvider;
 	email: {
 		enabled: boolean;
 		provider: 'smtp' | 'none';

fluxer_api/src/api/infrastructure/StorageService.test.ts

@@ -187,3 +187,26 @@ describe('StorageService.copyObjectWithMetadataStripping', () => {
 		]);
 	});
 });
+
+describe('provider selection', () => {
+	interface ClientProbe {
+		client: {config: {region: () => Promise<string>; endpoint?: () => Promise<{hostname: string}>}};
+	}
+
+	it('defaults to the shared S3 configuration', async () => {
+		const service = new StorageService() as unknown as ClientProbe;
+		expect(await service.client.config.region()).toBe(Config.s3.region);
+	});
+
+	it('uses an explicitly supplied provider instead of the shared one', async () => {
+		const service = new StorageService({
+			endpoint: 'https://downloads.example.net',
+			forcePathStyle: false,
+			region: 'eu-central-9',
+			accessKeyId: 'DL_KEY',
+			secretAccessKey: 'DL_SECRET',
+		}) as unknown as ClientProbe;
+		expect(await service.client.config.region()).toBe('eu-central-9');
+		expect(await service.client.config.region()).not.toBe(Config.s3.region);
+	});
+});

fluxer_api/src/api/infrastructure/StorageService.ts

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+import type {S3ProviderSettings} from '@fluxer/config/src/S3DownloadsProvider';
 import assert from 'node:assert/strict';
 import {createHash} from 'node:crypto';
 import fs from 'node:fs';
@@ -105,27 +106,36 @@ function extractStreamFromGet(out: GetObjectCommandOutput): Readable {
 export class StorageService implements IStorageService {
 	private readonly client: S3Client;
 	private readonly presignClient: S3Client;
+	private readonly provider: S3ProviderSettings;
 
-	constructor() {
+	constructor(provider?: S3ProviderSettings) {
-		this.client = buildPooledS3Client({
+		this.provider = provider ?? {
 			endpoint: Config.s3.endpoint,
+			presignedUrlBase: Config.s3.presignedUrlBase,
+			forcePathStyle: Config.s3.forcePathStyle,
 			region: Config.s3.region,
 			accessKeyId: Config.s3.accessKeyId,
 			secretAccessKey: Config.s3.secretAccessKey,
+		};
+		this.client = buildPooledS3Client({
+			endpoint: this.provider.endpoint,
+			region: this.provider.region,
+			accessKeyId: this.provider.accessKeyId,
+			secretAccessKey: this.provider.secretAccessKey,
 			forcePathStyle: true,
 		});
 		this.presignClient = buildPooledS3Client({
 			endpoint: this.resolvePresignEndpoint(),
-			region: Config.s3.region,
+			region: this.provider.region,
-			accessKeyId: Config.s3.accessKeyId,
+			accessKeyId: this.provider.accessKeyId,
-			secretAccessKey: Config.s3.secretAccessKey,
+			secretAccessKey: this.provider.secretAccessKey,
-			forcePathStyle: Config.s3.forcePathStyle,
+			forcePathStyle: this.provider.forcePathStyle,
 		});
 	}
 
 	private resolvePresignEndpoint(): string {
-		const fallbackEndpoint = Config.s3.endpoint;
+		const fallbackEndpoint = this.provider.endpoint;
-		const configuredEndpoint = Config.s3.presignedUrlBase;
+		const configuredEndpoint = this.provider.presignedUrlBase;
 		if (!configuredEndpoint) {
 			return fallbackEndpoint;
 		}

fluxer_api/src/api/infrastructure/StorageServiceFactory.test.ts

@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {describe, expect, it} from 'vitest';
+import {Config} from '../Config';
+import {createDownloadsStorageService} from './StorageServiceFactory';
+
+describe('createDownloadsStorageService', () => {
+	it('returns null when no downloads override is configured', () => {
+		expect(Config.s3Downloads.isOverridden).toBe(false);
+		expect(createDownloadsStorageService()).toBeNull();
+	});
+
+	it('resolves the downloads provider to the shared provider by default', () => {
+		expect(Config.s3Downloads.settings.endpoint).toBe(Config.s3.endpoint);
+		expect(Config.s3Downloads.settings.region).toBe(Config.s3.region);
+		expect(Config.s3Downloads.settings.accessKeyId).toBe(Config.s3.accessKeyId);
+	});
+});

fluxer_api/src/api/infrastructure/StorageServiceFactory.ts

@@ -1,8 +1,16 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+import {Config} from '../Config';
 import type {IStorageService} from './IStorageService';
 import {StorageService} from './StorageService';
 
 export function createStorageService(): IStorageService {
 	return new StorageService();
 }
+
+export function createDownloadsStorageService(): IStorageService | null {
+	if (!Config.s3Downloads.isOverridden) {
+		return null;
+	}
+	return new StorageService(Config.s3Downloads.settings);
+}

fluxer_api/src/api/middleware/ServiceSingletons.ts

@@ -60,7 +60,7 @@ import {KVActivityTracker} from '../infrastructure/KVActivityTracker';
 import {KVBulkMessageDeletionQueueService} from '../infrastructure/KVBulkMessageDeletionQueueService';
 import {NatsUnfurlerService} from '../infrastructure/NatsUnfurlerService';
 import {PremiumStateReconciliationQueueService} from '../infrastructure/PremiumStateReconciliationQueueService';
-import {createStorageService} from '../infrastructure/StorageServiceFactory';
+import {createDownloadsStorageService, createStorageService} from '../infrastructure/StorageServiceFactory';
 import {UserCacheService} from '../infrastructure/UserCacheService';
 import {createUsersServiceClient} from '../infrastructure/UsersServiceClient';
 import {VirusScanService} from '../infrastructure/VirusScanService';
@@ -193,6 +193,10 @@ export const getStorageService: () => IStorageService = (() => {
 	const fallback = singleton(() => createStorageService());
 	return () => _injectedStorageService ?? fallback();
 })();
+const getDownloadsStorageService: () => IStorageService = (() => {
+	const override = singleton(() => createDownloadsStorageService());
+	return () => override() ?? getStorageService();
+})();
 export const getErrorI18nService = singleton(() => new ErrorI18nService());
 export const getLimitConfigService = singleton(
 	() => new LimitConfigService(getInstanceConfigRepository(), getCacheService(), getKVClient()),
@@ -262,7 +266,7 @@ export function getKVAccountDeletionQueue(): KVAccountDeletionQueueService {
 	return accountDeletionQueue;
 }
 
-export const getDownloadService = singleton(() => new DownloadService(getStorageService()));
+export const getDownloadService = singleton(() => new DownloadService(getDownloadsStorageService()));
 export const getThemeService = singleton(() => new ThemeService(getStorageService()));
 const getNcmecReporter = singleton(() => new NcmecReporter({config: createNcmecApiConfig(), fetch}));
 const getNcmecRepository = singleton(() => new NcmecRepository());

packages/config/src/S3DownloadsProvider.ts

@@ -2,7 +2,7 @@
 
 import type {MasterConfig} from '@fluxer/config/src/MasterConfig';
 
-interface S3ProviderSettings {
+export interface S3ProviderSettings {
 	endpoint: string;
 	presignedUrlBase?: string;
 	forcePathStyle: boolean;
@@ -11,7 +11,7 @@ interface S3ProviderSettings {
 	secretAccessKey: string;
 }
 
-interface ResolvedDownloadsProvider {
+export interface ResolvedDownloadsProvider {
 	settings: S3ProviderSettings;
 	isOverridden: boolean;
 }