feat(api): make tor and breached password lookups opt-in (#2834)

deploy/self-hosting/.env.example

@@ -104,6 +104,12 @@ MEILI_MASTER_KEY=CHANGE_ME
 #FLUXER_NCMEC_ENABLED=false
 #FLUXER_CLAMAV_ENABLED=false
 
+# Outside lookups, off unless turned on. The Tor exit list comes from
+# onionoo.torproject.org and the breached password check asks
+# api.pwnedpasswords.com.
+#FLUXER_TOR_EXIT_LIST_ENABLED=true
+#FLUXER_BREACHED_PASSWORD_CHECK_ENABLED=true
+
 # The client address. Name the header your proxy actually writes, and turn the
 # trust off when nothing sits in front.
 #FLUXER_CLIENT_IP_HEADER_NAME=cf-connecting-ip

deploy/self-hosting/docker-compose.yml

@@ -24,6 +24,8 @@ x-fluxer-env: &fluxer-env
   FLUXER_CLIENT_IP_HEADER_NAME: ${FLUXER_CLIENT_IP_HEADER_NAME:-x-forwarded-for}
   FLUXER_API_HEADERS_TIMEOUT_MS: ${FLUXER_API_HEADERS_TIMEOUT_MS:-30000}
   FLUXER_API_REQUEST_TIMEOUT_MS: ${FLUXER_API_REQUEST_TIMEOUT_MS:-120000}
+  FLUXER_TOR_EXIT_LIST_ENABLED: "${FLUXER_TOR_EXIT_LIST_ENABLED:-false}"
+  FLUXER_BREACHED_PASSWORD_CHECK_ENABLED: "${FLUXER_BREACHED_PASSWORD_CHECK_ENABLED:-false}"
 
   FLUXER_KV_URL: ${FLUXER_KV_URL:-redis://valkey:6379/0}
   FLUXER_NATS_URL: ${FLUXER_NATS_URL:-nats://nats:4222}

fluxer_api/src/api/App.ts

@@ -49,6 +49,7 @@ export async function createAPIApp(options: CreateAPIAppOptions): Promise<APIApp
 		trustClientIpHeader: config.proxy.trust_client_ip_header,
 		clientIpHeaderName: config.proxy.client_ip_header,
 		maxInflightRequests: config.maxInflightRequests,
+		torExitBlockingEnabled: config.torExitList.enabled,
 	});
 	routes.onError(AbuseAwareAppErrorHandler);
 	routes.notFound(AppNotFoundHandler);

fluxer_api/src/api/Config.test.ts

@@ -166,3 +166,57 @@ describe('buildAPIConfigFromMaster stripe legacy prices', () => {
 		expect(buildAPIConfigFromMaster(withStripeLegacyPrices(master, undefined)).stripe.legacyPrices).toBeUndefined();
 	});
 });
+
+function withOptionalOutboundLookups(
+	master: MasterConfig,
+	selfHosted: boolean,
+	overrides: {torExitList?: boolean; breachedPasswordCheck?: boolean} = {},
+): MasterConfig {
+	return {
+		...master,
+		integrations: {
+			...master.integrations,
+			tor_exit_list: {enabled: overrides.torExitList},
+			breached_password_check: {enabled: overrides.breachedPasswordCheck},
+		},
+		instance: {
+			...master.instance,
+			self_hosted: selfHosted,
+		},
+	};
+}
+
+describe('buildAPIConfigFromMaster optional outbound lookups', () => {
+	let master: MasterConfig;
+	beforeAll(async () => {
+		master = await loadConfig();
+	});
+
+	it('keeps both lookups on when the instance is not self-hosted', () => {
+		const config = buildAPIConfigFromMaster(withOptionalOutboundLookups(master, false));
+		expect(config.torExitList.enabled).toBe(true);
+		expect(config.breachedPasswordCheck.enabled).toBe(true);
+	});
+
+	it('leaves both lookups off on a self-hosted instance', () => {
+		const config = buildAPIConfigFromMaster(withOptionalOutboundLookups(master, true));
+		expect(config.torExitList.enabled).toBe(false);
+		expect(config.breachedPasswordCheck.enabled).toBe(false);
+	});
+
+	it('lets a self-hosted operator switch each lookup on', () => {
+		const config = buildAPIConfigFromMaster(
+			withOptionalOutboundLookups(master, true, {torExitList: true, breachedPasswordCheck: true}),
+		);
+		expect(config.torExitList.enabled).toBe(true);
+		expect(config.breachedPasswordCheck.enabled).toBe(true);
+	});
+
+	it('lets an operator switch each lookup off when the instance is not self-hosted', () => {
+		const config = buildAPIConfigFromMaster(
+			withOptionalOutboundLookups(master, false, {torExitList: false, breachedPasswordCheck: false}),
+		);
+		expect(config.torExitList.enabled).toBe(false);
+		expect(config.breachedPasswordCheck.enabled).toBe(false);
+	});
+});

fluxer_api/src/api/Config.ts

@@ -348,6 +348,12 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
 		blocklistFeeds: {
 			enabled: master.integrations.blocklist_feeds.enabled ?? !master.instance.self_hosted,
 		},
+		torExitList: {
+			enabled: master.integrations.tor_exit_list.enabled ?? !master.instance.self_hosted,
+		},
+		breachedPasswordCheck: {
+			enabled: master.integrations.breached_password_check.enabled ?? !master.instance.self_hosted,
+		},
 		captcha: {
 			enabled: master.integrations.captcha.enabled,
 			provider: master.integrations.captcha.provider,

fluxer_api/src/api/app/APILifecycle.ts

@@ -137,9 +137,11 @@ export function createInitializer(config: APIConfig, logger: ILogger): () => Pro
 			await initializeRefreshCache(ipBanCache, 'IP ban cache', logger);
 			await startAbuseReplicationSubscriber(kvClient);
 			logger.info('Abusive-IP auto-banner replication started');
-			torExitListCache.setKvClient(kvClient);
-			await torExitListCache.initialize();
-			logger.info('Tor exit list cache initialized');
+			if (config.torExitList.enabled) {
+				torExitListCache.setKvClient(kvClient);
+				await torExitListCache.initialize();
+				logger.info('Tor exit list cache initialized');
+			}
 			const {urlBlocklistCache} = await import('@app/api/middleware/UrlBlocklistCache');
 			urlBlocklistCache.setRefreshSubscriber(kvClient);
 			const {getStorageService} = await import('@app/api/middleware/ServiceSingletons');

fluxer_api/src/api/app/MiddlewarePipeline.ts

@@ -29,10 +29,19 @@ interface MiddlewarePipelineOptions {
 	trustClientIpHeader: boolean;
 	clientIpHeaderName?: string;
 	maxInflightRequests: number;
+	torExitBlockingEnabled: boolean;
 }
 
 export function configureMiddleware(routes: HonoApp, options: MiddlewarePipelineOptions): void {
-	const {logger, nodeEnv, corsOrigins, trustClientIpHeader, clientIpHeaderName, maxInflightRequests} = options;
+	const {
+		logger,
+		nodeEnv,
+		corsOrigins,
+		trustClientIpHeader,
+		clientIpHeaderName,
+		maxInflightRequests,
+		torExitBlockingEnabled,
+	} = options;
 	const resolvedHeader = resolveClientIpHeaderName(clientIpHeaderName);
 	routes.use('/webhooks/:webhook_id/:token', cors({origins: '*'}));
 	routes.use('/webhooks/:webhook_id/:token/messages/:message_id', cors({origins: '*'}));
@@ -100,7 +109,9 @@ export function configureMiddleware(routes: HonoApp, options: MiddlewarePipeline
 			}),
 		);
 	}
-	routes.use(TorExitMiddleware);
+	if (torExitBlockingEnabled) {
+		routes.use(TorExitMiddleware);
+	}
 	routes.use(AuditLogMiddleware);
 	routes.use(RequireClientIpMiddleware());
 	routes.use(ServiceMiddleware);

fluxer_api/src/api/app/tests/MiddlewarePipeline.test.ts

@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {configureMiddleware} from '@app/api/app/MiddlewarePipeline';
+import {TorExitMiddleware} from '@app/api/middleware/TorExitMiddleware';
+import {NoopLogger} from '@app/api/test/mocks/NoopLogger';
+import type {HonoEnv} from '@app/api/types/HonoEnv';
+import {Hono} from 'hono';
+import {describe, expect, it} from 'vitest';
+
+function registeredHandlers(torExitBlockingEnabled: boolean): Array<unknown> {
+	const routes = new Hono<HonoEnv>({strict: true});
+	configureMiddleware(routes, {
+		logger: new NoopLogger(),
+		nodeEnv: 'test',
+		corsOrigins: ['http://localhost:3000'],
+		trustClientIpHeader: true,
+		clientIpHeaderName: 'x-forwarded-for',
+		maxInflightRequests: 100,
+		torExitBlockingEnabled,
+	});
+	return routes.routes.map((route) => route.handler);
+}
+
+describe('tor exit blocking in the middleware pipeline', () => {
+	it('registers the tor exit middleware when the switch is on', () => {
+		expect(registeredHandlers(true)).toContain(TorExitMiddleware);
+	});
+
+	it('leaves the tor exit middleware unregistered when the switch is off', () => {
+		expect(registeredHandlers(false)).not.toContain(TorExitMiddleware);
+	});
+});

fluxer_api/src/api/auth/AuthPassword.ts

@@ -5,6 +5,7 @@ import type {ApiContext} from '@app/api/ApiContext';
 import * as AuthSession from '@app/api/auth/AuthSession';
 import * as AuthUtility from '@app/api/auth/AuthUtility';
 import {createMfaTicket, createPasswordResetToken} from '@app/api/BrandedTypes';
+import {Config} from '@app/api/Config';
 import {Logger} from '@app/api/Logger';
 import type {User} from '@app/api/models/User';
 import {EXTERNAL_RESPONSE_LIMITS} from '@app/api/utils/ExternalResponseLimits';
@@ -116,6 +117,9 @@ export async function verifyPassword(
 }
 
 export async function isPasswordPwned(_ctx: ApiContext, password: string): Promise<boolean> {
+	if (!Config.breachedPasswordCheck.enabled) {
+		return false;
+	}
 	const hashed = crypto.createHash('sha1').update(password).digest('hex').toUpperCase();
 	const hashPrefix = hashed.slice(0, 5);
 	const hashSuffix = hashed.slice(5);

fluxer_api/src/api/auth/tests/PwnedPasswordCheck.test.ts

@@ -3,6 +3,7 @@
 import crypto from 'node:crypto';
 import type {ApiContext} from '@app/api/ApiContext';
 import {isPasswordPwned, resetPwnedPasswordCacheForTesting} from '@app/api/auth/AuthPassword';
+import {getConfig} from '@app/api/Config';
 import {server} from '@app/api/test/msw/server';
 import {delay, HttpResponse, http} from 'msw';
 import {beforeEach, describe, expect, test} from 'vitest';
@@ -68,6 +69,19 @@ describe('isPasswordPwned', () => {
 		await expect(isPasswordPwned(ctx, SAFE_PASSWORD_SAME_PREFIX)).resolves.toBe(false);
 		expect(requestedPrefixes).toHaveLength(1);
 	});
+	test('makes no upstream call when the check is switched off', async () => {
+		const config = getConfig();
+		const originalEnabled = config.breachedPasswordCheck.enabled;
+		const requestedPrefixes: Array<string> = [];
+		server.use(rangeHandler(requestedPrefixes, [suffixOf(PWNED_PASSWORD)]));
+		try {
+			config.breachedPasswordCheck.enabled = false;
+			await expect(isPasswordPwned(ctx, PWNED_PASSWORD)).resolves.toBe(false);
+			expect(requestedPrefixes).toHaveLength(0);
+		} finally {
+			config.breachedPasswordCheck.enabled = originalEnabled;
+		}
+	});
 	test('fails open on a non-OK response', async () => {
 		server.use(http.get('https://api.pwnedpasswords.com/range/:prefix', () => HttpResponse.text('', {status: 503})));
 		await expect(isPasswordPwned(ctx, PWNED_PASSWORD)).resolves.toBe(false);

fluxer_api/src/api/config/APIConfig.ts

@@ -207,6 +207,12 @@ export interface APIConfig {
 	blocklistFeeds: {
 		enabled: boolean;
 	};
+	torExitList: {
+		enabled: boolean;
+	};
+	breachedPasswordCheck: {
+		enabled: boolean;
+	};
 	captcha: {
 		enabled: boolean;
 		provider: 'hcaptcha' | 'turnstile' | 'none';

fluxer_docs/src/content/docs/http-api/users/current-user.mdx

@@ -97,7 +97,7 @@ Modifies the current account and returns the resulting [user](/http-api/users/#u
 
 <sup>4</sup> Any string value returns [`EMAIL_MUST_BE_CHANGED_VIA_TOKEN`](/http-api/errors/) on the `email` path, joined by `INVALID_EMAIL_FORMAT` when the string is not an address
 
-<sup>5</sup> A claimed account also supplies `password`, and a body that omits it is rejected with `PASSWORD_NOT_SET`. A breached password is rejected with [`PASSWORD_IS_TOO_COMMON`](/http-api/errors/)
+<sup>5</sup> A claimed account also supplies `password`, and a body that omits it is rejected with `PASSWORD_NOT_SET`. When the instance checks for breached passwords, a breached one is rejected with [`PASSWORD_IS_TOO_COMMON`](/http-api/errors/)
 
 <sup>6</sup> The value may have a `data:...;base64,` prefix. The remaining payload runs 1 to 13981016 characters
 

fluxer_docs/src/content/docs/operator/configuration.mdx

@@ -965,18 +965,6 @@ Default empty. The ipinfo key. Paired with `FLUXER_RISK_INTEGRATION_ENABLED`.
 
 No default. The account risk policy. JSON. Malformed JSON fails startup, and an unknown key in a well-formed policy surfaces when the policy runs.
 
-#### `FLUXER_RISK_TOR_BLOCK_ALL_RELAYS`
-
-Default `false`. Whether every Tor relay is blocked. Covers entry and middle relays as well as exit nodes.
-
-#### `FLUXER_RISK_TOR_REVERSE_DNS_HEURISTIC`
-
-Default `false`. Reverse DNS Tor detection. Adds a lookup to the request path.
-
-#### `FLUXER_RISK_TOR_REVERSE_DNS_TIMEOUT_MS`
-
-Default `750`. The lookup timeout. Milliseconds.
-
 #### `FLUXER_ABUSE_INBOUND_PHONE_COUNTRY_CODES`
 
 Default empty. Allowed inbound phone countries. Comma separated, passed through unvalidated.
@@ -1009,6 +997,14 @@ Default `flag_spammer`. What happens when it fires. `flag_spammer` or `suppress_
 
 Defaults to the inverse of `FLUXER_SELF_HOSTED`. External blocklist feeds. Off by default on a self-hosted instance.
 
+#### `FLUXER_TOR_EXIT_LIST_ENABLED`
+
+Defaults to the inverse of `FLUXER_SELF_HOSTED`. Tor exit blocking. The API fetches the exit relay list from `onionoo.torproject.org` at startup and every 30 minutes. Off by default on a self-hosted instance.
+
+#### `FLUXER_BREACHED_PASSWORD_CHECK_ENABLED`
+
+Defaults to the inverse of `FLUXER_SELF_HOSTED`. Breached password rejection. Sends the first five characters of the password's SHA-1 hash to `api.pwnedpasswords.com`. Off by default on a self-hosted instance.
+
 A second family, unrelated to the rules above, tunes the IP auto-banner: `FLUXER_ABUSE_WINDOW_MS`, the `FLUXER_ABUSE_THRESHOLD_` names, the `FLUXER_ABUSE_TOKEN_DIVERSITY_` names, `FLUXER_ABUSE_BAN_TTL_SEC`, `FLUXER_ABUSE_BATCH_FLUSH_MS`, `FLUXER_ABUSE_MAX_BATCH_TICKS`, `FLUXER_ABUSE_MAX_NEW_TOKENS_PER_TICK`, `FLUXER_ABUSE_MAX_TRACKED_IPS`, `FLUXER_ABUSE_MAX_TOKEN_HASHES_PER_IP`, `FLUXER_ABUSE_MIN_SCORE_FOR_LOOKUP`, `FLUXER_ABUSE_MIN_TOKENS_FOR_LOOKUP`, and `FLUXER_ABUSE_REQUIRED_SCORE_WINDOWS_FOR_AUTO_BAN`. All are read directly from the environment, none are in `.env.example` or the Compose file, and a non-finite value or one at or below zero falls back to the default.
 
 `FLUXER_ABUSE_EXEMPT_ASNS` takes comma-separated ASN numbers that the auto-banner never bans. Non-numeric entries are dropped. Before it buys a classification the auto-banner resolves the IP against the local MaxMind ASN database, and an IP on a listed ASN is neither classified nor banned. With no MaxMind ASN database the list has no effect. It does not change admin or guild IP bans, and `FLUXER_API_IP_BAN_EXEMPT_IPS` remains the way to exempt single addresses and ranges from every kind of IP ban.

packages/config/src/ConfigLoader.ts

@@ -252,15 +252,12 @@ function defaultConfig(): MasterConfig {
 				},
 			},
 			blocklist_feeds: {},
+			tor_exit_list: {},
+			breached_password_check: {},
 			risk_integration: {
 				enabled: false,
 				ipinfo_api_key: '',
 				account_policy_dsl: undefined,
-				tor: {
-					block_all_relays: false,
-					reverse_dns_heuristic: false,
-					reverse_dns_timeout_ms: 750,
-				},
 			},
 			push: {
 				apns: {

packages/config/src/MasterConfig.ts

@@ -298,15 +298,16 @@ export interface MasterConfig {
 		blocklist_feeds: {
 			enabled?: boolean;
 		};
+		tor_exit_list: {
+			enabled?: boolean;
+		};
+		breached_password_check: {
+			enabled?: boolean;
+		};
 		risk_integration: {
 			enabled: boolean;
 			ipinfo_api_key: string;
 			account_policy_dsl?: unknown;
-			tor: {
-				block_all_relays: boolean;
-				reverse_dns_heuristic: boolean;
-				reverse_dns_timeout_ms: number;
-			};
 		};
 		push: {
 			apns: {

packages/config/src/__tests__/ConfigLoader.test.ts

@@ -638,6 +638,27 @@ describe('ConfigLoader', () => {
 		expect((await loadConfig()).integrations.cache_purge.adapter).toBe('none');
 	});
 
+	test('leaves the optional outbound lookups unset by default', async () => {
+		stubMinimalEnv();
+
+		const config = await loadConfig();
+
+		expect(config.integrations.tor_exit_list.enabled).toBeUndefined();
+		expect(config.integrations.breached_password_check.enabled).toBeUndefined();
+	});
+
+	test('reads the optional outbound lookup switches from the environment', async () => {
+		stubMinimalEnv({
+			FLUXER_TOR_EXIT_LIST_ENABLED: 'true',
+			FLUXER_BREACHED_PASSWORD_CHECK_ENABLED: 'false',
+		});
+
+		const config = await loadConfig();
+
+		expect(config.integrations.tor_exit_list.enabled).toBe(true);
+		expect(config.integrations.breached_password_check.enabled).toBe(false);
+	});
+
 	test('leaves Bluesky login off with no legal URLs by default', async () => {
 		stubMinimalEnv();
 

packages/config/src/config_loader/EnvironmentOverrides.ts

@@ -340,24 +340,17 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
 		parse: parseInteger,
 	},
 	FLUXER_BLOCKLIST_FEEDS_ENABLED: {path: ['integrations', 'blocklist_feeds', 'enabled'], parse: parseBoolean},
+	FLUXER_TOR_EXIT_LIST_ENABLED: {path: ['integrations', 'tor_exit_list', 'enabled'], parse: parseBoolean},
+	FLUXER_BREACHED_PASSWORD_CHECK_ENABLED: {
+		path: ['integrations', 'breached_password_check', 'enabled'],
+		parse: parseBoolean,
+	},
 	FLUXER_RISK_INTEGRATION_ENABLED: {path: ['integrations', 'risk_integration', 'enabled'], parse: parseBoolean},
 	FLUXER_RISK_IPINFO_API_KEY: {path: ['integrations', 'risk_integration', 'ipinfo_api_key']},
 	FLUXER_ACCOUNT_POLICY_DSL: {
 		path: ['integrations', 'risk_integration', 'account_policy_dsl'],
 		parse: parseEnvValue,
 	},
-	FLUXER_RISK_TOR_BLOCK_ALL_RELAYS: {
-		path: ['integrations', 'risk_integration', 'tor', 'block_all_relays'],
-		parse: parseBoolean,
-	},
-	FLUXER_RISK_TOR_REVERSE_DNS_HEURISTIC: {
-		path: ['integrations', 'risk_integration', 'tor', 'reverse_dns_heuristic'],
-		parse: parseBoolean,
-	},
-	FLUXER_RISK_TOR_REVERSE_DNS_TIMEOUT_MS: {
-		path: ['integrations', 'risk_integration', 'tor', 'reverse_dns_timeout_ms'],
-		parse: parseInteger,
-	},
 	FLUXER_PUSH_APNS_ENABLED: {path: ['integrations', 'push', 'apns', 'enabled'], parse: parseBoolean},
 	FLUXER_PUSH_APNS_TEAM_ID: {path: ['integrations', 'push', 'apns', 'team_id']},
 	FLUXER_PUSH_APNS_KEY_ID: {path: ['integrations', 'push', 'apns', 'key_id']},