fix(api): bound http server limits and shed on overload (#2170)

fluxer_api/src/api/App.ts

@@ -48,6 +48,7 @@ export async function createAPIApp(options: CreateAPIAppOptions): Promise<APIApp
 		corsOrigins: [config.endpoints.webApp, config.endpoints.marketing],
 		trustClientIpHeader: config.proxy.trust_client_ip_header,
 		clientIpHeaderName: config.proxy.client_ip_header,
+		maxInflightRequests: config.maxInflightRequests,
 	});
 	routes.onError(AbuseAwareAppErrorHandler);
 	routes.notFound(AppNotFoundHandler);

fluxer_api/src/api/Config.ts

@@ -160,6 +160,7 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
 	return {
 		nodeEnv: master.env === 'test' ? 'development' : master.env,
 		port: master.services.api.port,
+		maxInflightRequests: master.services.api.max_inflight_requests,
 		ipBanExemptIps: normalizeIpBanExemptIps(master.services.api.ip_ban_exempt_ips),
 		desktopGitHubRedirectCountries: normalizeCountryCodes(
 			master.services.api.desktop_github_redirect_countries,

fluxer_api/src/api/app/MiddlewarePipeline.ts

@@ -9,6 +9,7 @@ import {resolveClientIpHeaderName} from '@fluxer/ip_utils/src/ClientIp';
 import type {ILogger} from '../ILogger';
 import {ClientErrorAbuseSignalMiddleware} from '../middleware/AbusiveIpAutoBanner';
 import {AuditLogMiddleware} from '../middleware/AuditLogMiddleware';
+import {ConcurrencyLimitMiddleware} from '../middleware/ConcurrencyLimitMiddleware';
 import ContentFilterMiddleware from '../middleware/ContentFilterMiddleware';
 import {GuildAvailabilityMiddleware} from '../middleware/GuildAvailabilityMiddleware';
 import {IpBanMiddleware} from '../middleware/IpBanMiddleware';
@@ -27,10 +28,11 @@ interface MiddlewarePipelineOptions {
 	corsOrigins: Array<string>;
 	trustClientIpHeader: boolean;
 	clientIpHeaderName?: string;
+	maxInflightRequests: number;
 }
 
 export function configureMiddleware(routes: HonoApp, options: MiddlewarePipelineOptions): void {
-	const {logger, nodeEnv, corsOrigins, trustClientIpHeader, clientIpHeaderName} = options;
+	const {logger, nodeEnv, corsOrigins, trustClientIpHeader, clientIpHeaderName, maxInflightRequests} = options;
 	const resolvedHeader = resolveClientIpHeaderName(clientIpHeaderName);
 	routes.use('/webhooks/:webhook_id/:token', cors({origins: '*'}));
 	routes.use('/webhooks/:webhook_id/:token/messages/:message_id', cors({origins: '*'}));
@@ -40,6 +42,7 @@ export function configureMiddleware(routes: HonoApp, options: MiddlewarePipeline
 		skipLogger: true,
 		skipErrorHandler: true,
 	});
+	routes.use(ConcurrencyLimitMiddleware({maxInflightRequests}));
 	routes.get('/_health', async (ctx) => ctx.text('OK'));
 	routes.use(IpBanMiddleware);
 	routes.use(

fluxer_api/src/api/config/APIConfig.ts

@@ -34,6 +34,7 @@ export type APIGeoipConfig = APIGeoipFilesystemConfig | APIGeoipS3Config;
 export interface APIConfig {
 	nodeEnv: 'development' | 'production';
 	port: number;
+	maxInflightRequests: number;
 	ipBanExemptIps: Array<string>;
 	desktopGitHubRedirectCountries: ReadonlySet<string>;
 	cassandra: {

fluxer_api/src/api/middleware/ConcurrencyLimitMiddleware.ts

@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {ServiceUnavailableError} from '@fluxer/errors/src/domains/core/ServiceUnavailableError';
+import {createMiddleware} from 'hono/factory';
+import type {HonoEnv} from '../types/HonoEnv';
+import {normalizeRequestPath} from '../utils/RequestPathUtils';
+
+const PROBE_PATHS = new Set(['/_health', '/_healthz', '/_metrics']);
+const OVERLOAD_RETRY_AFTER_SECONDS = 1;
+
+interface ConcurrencyLimitOptions {
+	maxInflightRequests: number;
+}
+
+export function ConcurrencyLimitMiddleware({maxInflightRequests}: ConcurrencyLimitOptions) {
+	let inflight = 0;
+	return createMiddleware<HonoEnv>(async (ctx, next) => {
+		if (PROBE_PATHS.has(normalizeRequestPath(ctx.req.path))) {
+			await next();
+			return;
+		}
+		inflight += 1;
+		try {
+			if (inflight > maxInflightRequests) {
+				throw new ServiceUnavailableError({
+					headers: {'Retry-After': String(OVERLOAD_RETRY_AFTER_SECONDS)},
+				});
+			}
+			await next();
+		} finally {
+			inflight -= 1;
+		}
+	});
+}

fluxer_api/src/api/middleware/tests/ConcurrencyLimitMiddleware.test.ts

@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {AppErrorHandler} from '@fluxer/errors/src/domains/core/ErrorHandlers';
+import {Hono} from 'hono';
+import {describe, expect, test} from 'vitest';
+import type {HonoEnv} from '../../types/HonoEnv';
+import {ConcurrencyLimitMiddleware} from '../ConcurrencyLimitMiddleware';
+
+interface Deferred {
+	promise: Promise<void>;
+	resolve: () => void;
+}
+
+function createDeferred(): Deferred {
+	let resolve: () => void = () => undefined;
+	const promise = new Promise<void>((resolveFn) => {
+		resolve = resolveFn;
+	});
+	return {promise, resolve};
+}
+
+function buildApp(maxInflightRequests: number) {
+	const release = createDeferred();
+	const entered = createDeferred();
+	const routes = new Hono<HonoEnv>({strict: true});
+	routes.use(ConcurrencyLimitMiddleware({maxInflightRequests}));
+	routes.get('/_health', (ctx) => ctx.text('OK'));
+	routes.get('/_metrics', (ctx) => ctx.text('metrics'));
+	routes.get('/fast', (ctx) => ctx.text('fast'));
+	routes.get('/slow', async (ctx) => {
+		entered.resolve();
+		await release.promise;
+		return ctx.text('slow');
+	});
+	routes.get('/boom', () => {
+		throw new Error('boom');
+	});
+	routes.onError(AppErrorHandler);
+	const app = new Hono<HonoEnv>({strict: true});
+	app.route('/v1', routes);
+	app.route('/', routes);
+	app.onError(AppErrorHandler);
+	return {app, release, entered};
+}
+
+describe('ConcurrencyLimitMiddleware', () => {
+	test('sheds with 503 once the in-flight ceiling is reached', async () => {
+		const {app, release, entered} = buildApp(1);
+		const inflight = app.request('/slow');
+		await entered.promise;
+
+		const shed = await app.request('/fast');
+
+		expect(shed.status).toBe(503);
+		expect(shed.headers.get('Retry-After')).toBe('1');
+		expect(await shed.json()).toMatchObject({code: 'SERVICE_UNAVAILABLE'});
+
+		release.resolve();
+		expect((await inflight).status).toBe(200);
+	});
+
+	test('admits probe paths while shedding everything else', async () => {
+		const {app, release, entered} = buildApp(1);
+		const inflight = app.request('/slow');
+		await entered.promise;
+
+		expect((await app.request('/_health')).status).toBe(200);
+		expect((await app.request('/_metrics')).status).toBe(200);
+		expect((await app.request('/v1/_health')).status).toBe(200);
+		expect((await app.request('/fast')).status).toBe(503);
+
+		release.resolve();
+		await inflight;
+	});
+
+	test('releases the slot when the handler throws', async () => {
+		const {app} = buildApp(1);
+		for (let attempt = 0; attempt < 3; attempt += 1) {
+			expect((await app.request('/boom')).status).toBe(500);
+		}
+		expect((await app.request('/fast')).status).toBe(200);
+	});
+
+	test('releases the slot taken by a shed request', async () => {
+		const {app, release, entered} = buildApp(1);
+		const inflight = app.request('/slow');
+		await entered.promise;
+
+		expect((await app.request('/fast')).status).toBe(503);
+		expect((await app.request('/fast')).status).toBe(503);
+
+		release.resolve();
+		await inflight;
+
+		expect((await app.request('/fast')).status).toBe(200);
+	});
+});