fix(api): make the http header and request timeouts tunable (#2259)

deploy/self-hosting/.env.example

@@ -121,3 +121,12 @@ FLUXER_DISCOVERY_ENABLED=true
 # every service. The bundled compose talks to Postgres directly, where naming is
 # a win and the default is correct.
 #FLUXER_POSTGRES_PREPARED_STATEMENTS=true
+
+# The api bounds how long a client may take to send a request. The header timeout
+# covers the request line and headers only, while the request timeout covers the
+# whole exchange, so a slow uploader is bounded by the second value and not by
+# the first. Raise both if you front large uploads or serve clients on high
+# latency links. The header timeout is clamped down to the request timeout, so
+# raising it alone does nothing. Both are milliseconds, between 1000 and 3600000.
+#FLUXER_API_HEADERS_TIMEOUT_MS=30000
+#FLUXER_API_REQUEST_TIMEOUT_MS=120000

deploy/self-hosting/docker-compose.yml

@@ -9,6 +9,8 @@ x-fluxer-env: &fluxer-env
   FLUXER_PUBLIC_PORT: ${FLUXER_PUBLIC_PORT:-443}
   FLUXER_TRUST_CLIENT_IP_HEADER: "true"
   FLUXER_CLIENT_IP_HEADER_NAME: x-forwarded-for
+  FLUXER_API_HEADERS_TIMEOUT_MS: ${FLUXER_API_HEADERS_TIMEOUT_MS:-30000}
+  FLUXER_API_REQUEST_TIMEOUT_MS: ${FLUXER_API_REQUEST_TIMEOUT_MS:-120000}
 
   FLUXER_DATABASE_BACKEND: postgres
   FLUXER_POSTGRES_HOST: postgres

fluxer_api/src/App.ts

@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import {createAPIApp} from '@app/api/App';
-import {initializeConfig} from '@app/api/Config';
+import {buildAPIServerOptions, initializeConfig} from '@app/api/Config';
 import {initializeLogger} from '@app/api/Logger';
 import {Config} from '@app/Config';
 import {shutdownInstrumentation} from '@app/Instrument';
@@ -34,7 +34,7 @@ async function main(): Promise<void> {
 	process.on('unhandledRejection', (reason) => {
 		Logger.error({reason}, 'Unhandled rejection (suppressed)');
 	});
-	const server = createServer(app, {port: Config.port});
+	const server = createServer(app, buildAPIServerOptions(Config));
 	Logger.info({port: Config.port}, `Starting Fluxer API on port ${Config.port}`);
 	setupGracefulShutdown(
 		async () => {

fluxer_api/src/api/Config.test.ts

@@ -0,0 +1,65 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {loadConfig, resetConfig} from '@fluxer/config/src/ConfigLoader';
+import {createServer} from '@fluxer/hono/src/Server';
+import {Hono} from 'hono';
+import {afterAll, afterEach, describe, expect, test, vi} from 'vitest';
+import {buildAPIConfigFromMaster, buildAPIServerOptions} from './Config';
+
+interface ListeningServer {
+	close: (callback: () => void) => void;
+	headersTimeout: number;
+	requestTimeout: number;
+}
+
+const servers: Array<ListeningServer> = [];
+
+async function listenWithEnv(env: Record<string, string> = {}): Promise<ListeningServer> {
+	for (const [key, value] of Object.entries({FLUXER_API_PORT: '0', ...env})) {
+		vi.stubEnv(key, value);
+	}
+	resetConfig();
+	const config = buildAPIConfigFromMaster(await loadConfig());
+	const server = createServer(new Hono(), buildAPIServerOptions(config)) as unknown as ListeningServer;
+	servers.push(server);
+	return server;
+}
+
+afterEach(async () => {
+	await Promise.all(servers.splice(0).map((server) => new Promise<void>((resolve) => server.close(() => resolve()))));
+	vi.unstubAllEnvs();
+	resetConfig();
+});
+
+afterAll(async () => {
+	await loadConfig();
+});
+
+describe('buildAPIServerOptions', () => {
+	test('starts the api on the shipped header and request timeouts', async () => {
+		const server = await listenWithEnv();
+		expect(server.headersTimeout).toBe(30_000);
+		expect(server.requestTimeout).toBe(120_000);
+	});
+
+	test('carries the operator header timeout from the environment into the server', async () => {
+		const server = await listenWithEnv({FLUXER_API_HEADERS_TIMEOUT_MS: '45000'});
+		expect(server.headersTimeout).toBe(45_000);
+		expect(server.requestTimeout).toBe(120_000);
+	});
+
+	test('carries the operator request timeout from the environment into the server', async () => {
+		const server = await listenWithEnv({FLUXER_API_REQUEST_TIMEOUT_MS: '600000'});
+		expect(server.headersTimeout).toBe(30_000);
+		expect(server.requestTimeout).toBe(600_000);
+	});
+
+	test('clamps a header timeout set above the request timeout', async () => {
+		const server = await listenWithEnv({
+			FLUXER_API_HEADERS_TIMEOUT_MS: '90000',
+			FLUXER_API_REQUEST_TIMEOUT_MS: '45000',
+		});
+		expect(server.requestTimeout).toBe(45_000);
+		expect(server.headersTimeout).toBe(45_000);
+	});
+});

fluxer_api/src/api/Config.ts

@@ -160,6 +160,8 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
 	return {
 		nodeEnv: master.env === 'test' ? 'development' : master.env,
 		port: master.services.api.port,
+		headersTimeoutMs: master.services.api.headers_timeout_ms,
+		requestTimeoutMs: master.services.api.request_timeout_ms,
 		maxInflightRequests: master.services.api.max_inflight_requests,
 		ipBanExemptIps: normalizeIpBanExemptIps(master.services.api.ip_ban_exempt_ips),
 		desktopGitHubRedirectCountries: normalizeCountryCodes(
@@ -530,6 +532,20 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
 	};
 }
 
+interface APIServerOptions {
+	port: number;
+	headersTimeoutMs: number;
+	requestTimeoutMs: number;
+}
+
+export function buildAPIServerOptions(config: APIConfig): APIServerOptions {
+	return {
+		port: config.port,
+		headersTimeoutMs: config.headersTimeoutMs,
+		requestTimeoutMs: config.requestTimeoutMs,
+	};
+}
+
 let _config: APIConfig | null = null;
 
 export function initializeConfig(config: APIConfig): void {

fluxer_api/src/api/config/APIConfig.ts

@@ -34,6 +34,8 @@ export type APIGeoipConfig = APIGeoipFilesystemConfig | APIGeoipS3Config;
 export interface APIConfig {
 	nodeEnv: 'development' | 'production';
 	port: number;
+	headersTimeoutMs: number;
+	requestTimeoutMs: number;
 	maxInflightRequests: number;
 	ipBanExemptIps: Array<string>;
 	desktopGitHubRedirectCountries: ReadonlySet<string>;

packages/config/src/ConfigLoader.ts

@@ -92,6 +92,8 @@ function defaultConfig(): MasterConfig {
 		services: {
 			api: {
 				port: 8080,
+				headers_timeout_ms: 30_000,
+				request_timeout_ms: 120_000,
 				max_inflight_requests: 512,
 				ip_ban_exempt_ips: [],
 				desktop_github_redirect_countries: [],
@@ -412,6 +414,8 @@ function normalizeConfig(config: MasterConfig): MasterConfig {
 	validatePostgresConfig(config);
 	validateApiWorkerConfig(config);
 	assertIntegerInRange(config.services.api.max_inflight_requests, 'FLUXER_API_MAX_INFLIGHT_REQUESTS', 1, 100_000);
+	assertIntegerInRange(config.services.api.headers_timeout_ms, 'FLUXER_API_HEADERS_TIMEOUT_MS', 1_000, 3_600_000);
+	assertIntegerInRange(config.services.api.request_timeout_ms, 'FLUXER_API_REQUEST_TIMEOUT_MS', 1_000, 3_600_000);
 	requireString(config.domain.base_domain, 'FLUXER_BASE_DOMAIN');
 	requireString(config.auth.sudo_mode_secret, 'FLUXER_SUDO_MODE_SECRET');
 	requireString(config.auth.connection_initiation_secret, 'FLUXER_CONNECTION_INITIATION_SECRET');

packages/config/src/MasterConfig.ts

@@ -91,6 +91,8 @@ export interface MasterConfig {
 	services: {
 		api: {
 			port: number;
+			headers_timeout_ms: number;
+			request_timeout_ms: number;
 			max_inflight_requests: number;
 			ip_ban_exempt_ips: Array<string>;
 			desktop_github_redirect_countries: Array<string>;

packages/config/src/__tests__/ConfigLoader.test.ts

@@ -160,6 +160,30 @@ describe('ConfigLoader', () => {
 		await expect(loadConfig()).rejects.toThrow('FLUXER_POSTGRES_PREPARED_STATEMENTS');
 	});
 
+	test('keeps the api http timeouts at their defaults', async () => {
+		stubMinimalEnv();
+		const config = await loadConfig();
+		expect(config.services.api.headers_timeout_ms).toBe(30_000);
+		expect(config.services.api.request_timeout_ms).toBe(120_000);
+	});
+
+	test('reads the api http timeouts from the environment', async () => {
+		stubMinimalEnv({FLUXER_API_HEADERS_TIMEOUT_MS: '45000', FLUXER_API_REQUEST_TIMEOUT_MS: '600000'});
+		const config = await loadConfig();
+		expect(config.services.api.headers_timeout_ms).toBe(45_000);
+		expect(config.services.api.request_timeout_ms).toBe(600_000);
+	});
+
+	test('rejects a non-numeric api header timeout', async () => {
+		stubMinimalEnv({FLUXER_API_HEADERS_TIMEOUT_MS: 'soon'});
+		await expect(loadConfig()).rejects.toThrow('FLUXER_API_HEADERS_TIMEOUT_MS');
+	});
+
+	test('rejects a non-numeric api request timeout', async () => {
+		stubMinimalEnv({FLUXER_API_REQUEST_TIMEOUT_MS: 'soon'});
+		await expect(loadConfig()).rejects.toThrow('FLUXER_API_REQUEST_TIMEOUT_MS');
+	});
+
 	test('rejects unsafe production Postgres defaults', async () => {
 		stubMinimalEnv({FLUXER_ENV: 'production'});
 		await expect(loadConfig()).rejects.toThrow('FLUXER_POSTGRES_HOST');

packages/config/src/config_loader/EnvironmentOverrides.ts

@@ -77,6 +77,8 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
 	FLUXER_NATS_JETSTREAM_URL: {path: ['services', 'nats', 'jetstream_url']},
 	FLUXER_NATS_AUTH_TOKEN: {path: ['services', 'nats', 'auth_token']},
 	FLUXER_API_PORT: {path: ['services', 'api', 'port'], parse: parseEnvValue},
+	FLUXER_API_HEADERS_TIMEOUT_MS: {path: ['services', 'api', 'headers_timeout_ms'], parse: parseEnvValue},
+	FLUXER_API_REQUEST_TIMEOUT_MS: {path: ['services', 'api', 'request_timeout_ms'], parse: parseEnvValue},
 	FLUXER_API_MAX_INFLIGHT_REQUESTS: {path: ['services', 'api', 'max_inflight_requests'], parse: parseEnvValue},
 	FLUXER_API_IP_BAN_EXEMPT_IPS: {path: ['services', 'api', 'ip_ban_exempt_ips'], parse: parseCsv},
 	FLUXER_API_DESKTOP_GITHUB_REDIRECT_COUNTRIES: {