From f32356801d569afd570c5a8357d38fa932a44c69 Mon Sep 17 00:00:00 2001 From: Hampus Date: Sat, 19 Sep 2026 01:22:17 +0200 Subject: [PATCH] feat(api): make tor and breached password lookups opt-in (#2834) --- deploy/self-hosting/.env.example | 6 +++ deploy/self-hosting/docker-compose.yml | 2 + fluxer_api/src/api/App.ts | 1 + fluxer_api/src/api/Config.test.ts | 54 +++++++++++++++++++ fluxer_api/src/api/Config.ts | 6 +++ fluxer_api/src/api/app/APILifecycle.ts | 8 +-- fluxer_api/src/api/app/MiddlewarePipeline.ts | 15 +++++- .../api/app/tests/MiddlewarePipeline.test.ts | 32 +++++++++++ fluxer_api/src/api/auth/AuthPassword.ts | 4 ++ .../api/auth/tests/PwnedPasswordCheck.test.ts | 14 +++++ fluxer_api/src/api/config/APIConfig.ts | 6 +++ .../docs/http-api/users/current-user.mdx | 2 +- .../content/docs/operator/configuration.mdx | 20 +++---- packages/config/src/ConfigLoader.ts | 7 +-- packages/config/src/MasterConfig.ts | 11 ++-- .../config/src/__tests__/ConfigLoader.test.ts | 21 ++++++++ .../src/config_loader/EnvironmentOverrides.ts | 17 ++---- 17 files changed, 186 insertions(+), 40 deletions(-) create mode 100644 fluxer_api/src/api/app/tests/MiddlewarePipeline.test.ts diff --git a/deploy/self-hosting/.env.example b/deploy/self-hosting/.env.example index 89dd904ab..d8fa6ebd1 100644 --- a/deploy/self-hosting/.env.example +++ b/deploy/self-hosting/.env.example @@ -104,6 +104,12 @@ MEILI_MASTER_KEY=CHANGE_ME #FLUXER_NCMEC_ENABLED=false #FLUXER_CLAMAV_ENABLED=false +# Outside lookups, off unless turned on. The Tor exit list comes from +# onionoo.torproject.org and the breached password check asks +# api.pwnedpasswords.com. +#FLUXER_TOR_EXIT_LIST_ENABLED=true +#FLUXER_BREACHED_PASSWORD_CHECK_ENABLED=true + # The client address. Name the header your proxy actually writes, and turn the # trust off when nothing sits in front. #FLUXER_CLIENT_IP_HEADER_NAME=cf-connecting-ip diff --git a/deploy/self-hosting/docker-compose.yml b/deploy/self-hosting/docker-compose.yml index b52046b31..33ba7c374 100644 --- a/deploy/self-hosting/docker-compose.yml +++ b/deploy/self-hosting/docker-compose.yml @@ -24,6 +24,8 @@ x-fluxer-env: &fluxer-env FLUXER_CLIENT_IP_HEADER_NAME: ${FLUXER_CLIENT_IP_HEADER_NAME:-x-forwarded-for} FLUXER_API_HEADERS_TIMEOUT_MS: ${FLUXER_API_HEADERS_TIMEOUT_MS:-30000} FLUXER_API_REQUEST_TIMEOUT_MS: ${FLUXER_API_REQUEST_TIMEOUT_MS:-120000} + FLUXER_TOR_EXIT_LIST_ENABLED: "${FLUXER_TOR_EXIT_LIST_ENABLED:-false}" + FLUXER_BREACHED_PASSWORD_CHECK_ENABLED: "${FLUXER_BREACHED_PASSWORD_CHECK_ENABLED:-false}" FLUXER_KV_URL: ${FLUXER_KV_URL:-redis://valkey:6379/0} FLUXER_NATS_URL: ${FLUXER_NATS_URL:-nats://nats:4222} diff --git a/fluxer_api/src/api/App.ts b/fluxer_api/src/api/App.ts index 502b90304..a35bef59e 100644 --- a/fluxer_api/src/api/App.ts +++ b/fluxer_api/src/api/App.ts @@ -49,6 +49,7 @@ export async function createAPIApp(options: CreateAPIAppOptions): Promise { expect(buildAPIConfigFromMaster(withStripeLegacyPrices(master, undefined)).stripe.legacyPrices).toBeUndefined(); }); }); + +function withOptionalOutboundLookups( + master: MasterConfig, + selfHosted: boolean, + overrides: {torExitList?: boolean; breachedPasswordCheck?: boolean} = {}, +): MasterConfig { + return { + ...master, + integrations: { + ...master.integrations, + tor_exit_list: {enabled: overrides.torExitList}, + breached_password_check: {enabled: overrides.breachedPasswordCheck}, + }, + instance: { + ...master.instance, + self_hosted: selfHosted, + }, + }; +} + +describe('buildAPIConfigFromMaster optional outbound lookups', () => { + let master: MasterConfig; + beforeAll(async () => { + master = await loadConfig(); + }); + + it('keeps both lookups on when the instance is not self-hosted', () => { + const config = buildAPIConfigFromMaster(withOptionalOutboundLookups(master, false)); + expect(config.torExitList.enabled).toBe(true); + expect(config.breachedPasswordCheck.enabled).toBe(true); + }); + + it('leaves both lookups off on a self-hosted instance', () => { + const config = buildAPIConfigFromMaster(withOptionalOutboundLookups(master, true)); + expect(config.torExitList.enabled).toBe(false); + expect(config.breachedPasswordCheck.enabled).toBe(false); + }); + + it('lets a self-hosted operator switch each lookup on', () => { + const config = buildAPIConfigFromMaster( + withOptionalOutboundLookups(master, true, {torExitList: true, breachedPasswordCheck: true}), + ); + expect(config.torExitList.enabled).toBe(true); + expect(config.breachedPasswordCheck.enabled).toBe(true); + }); + + it('lets an operator switch each lookup off when the instance is not self-hosted', () => { + const config = buildAPIConfigFromMaster( + withOptionalOutboundLookups(master, false, {torExitList: false, breachedPasswordCheck: false}), + ); + expect(config.torExitList.enabled).toBe(false); + expect(config.breachedPasswordCheck.enabled).toBe(false); + }); +}); diff --git a/fluxer_api/src/api/Config.ts b/fluxer_api/src/api/Config.ts index 2694cc1d2..b9d580358 100644 --- a/fluxer_api/src/api/Config.ts +++ b/fluxer_api/src/api/Config.ts @@ -348,6 +348,12 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig { blocklistFeeds: { enabled: master.integrations.blocklist_feeds.enabled ?? !master.instance.self_hosted, }, + torExitList: { + enabled: master.integrations.tor_exit_list.enabled ?? !master.instance.self_hosted, + }, + breachedPasswordCheck: { + enabled: master.integrations.breached_password_check.enabled ?? !master.instance.self_hosted, + }, captcha: { enabled: master.integrations.captcha.enabled, provider: master.integrations.captcha.provider, diff --git a/fluxer_api/src/api/app/APILifecycle.ts b/fluxer_api/src/api/app/APILifecycle.ts index f99b1a6e5..0a978f3e9 100644 --- a/fluxer_api/src/api/app/APILifecycle.ts +++ b/fluxer_api/src/api/app/APILifecycle.ts @@ -137,9 +137,11 @@ export function createInitializer(config: APIConfig, logger: ILogger): () => Pro await initializeRefreshCache(ipBanCache, 'IP ban cache', logger); await startAbuseReplicationSubscriber(kvClient); logger.info('Abusive-IP auto-banner replication started'); - torExitListCache.setKvClient(kvClient); - await torExitListCache.initialize(); - logger.info('Tor exit list cache initialized'); + if (config.torExitList.enabled) { + torExitListCache.setKvClient(kvClient); + await torExitListCache.initialize(); + logger.info('Tor exit list cache initialized'); + } const {urlBlocklistCache} = await import('@app/api/middleware/UrlBlocklistCache'); urlBlocklistCache.setRefreshSubscriber(kvClient); const {getStorageService} = await import('@app/api/middleware/ServiceSingletons'); diff --git a/fluxer_api/src/api/app/MiddlewarePipeline.ts b/fluxer_api/src/api/app/MiddlewarePipeline.ts index 03bfba0fa..4021b0a31 100644 --- a/fluxer_api/src/api/app/MiddlewarePipeline.ts +++ b/fluxer_api/src/api/app/MiddlewarePipeline.ts @@ -29,10 +29,19 @@ interface MiddlewarePipelineOptions { trustClientIpHeader: boolean; clientIpHeaderName?: string; maxInflightRequests: number; + torExitBlockingEnabled: boolean; } export function configureMiddleware(routes: HonoApp, options: MiddlewarePipelineOptions): void { - const {logger, nodeEnv, corsOrigins, trustClientIpHeader, clientIpHeaderName, maxInflightRequests} = options; + const { + logger, + nodeEnv, + corsOrigins, + trustClientIpHeader, + clientIpHeaderName, + maxInflightRequests, + torExitBlockingEnabled, + } = options; const resolvedHeader = resolveClientIpHeaderName(clientIpHeaderName); routes.use('/webhooks/:webhook_id/:token', cors({origins: '*'})); routes.use('/webhooks/:webhook_id/:token/messages/:message_id', cors({origins: '*'})); @@ -100,7 +109,9 @@ export function configureMiddleware(routes: HonoApp, options: MiddlewarePipeline }), ); } - routes.use(TorExitMiddleware); + if (torExitBlockingEnabled) { + routes.use(TorExitMiddleware); + } routes.use(AuditLogMiddleware); routes.use(RequireClientIpMiddleware()); routes.use(ServiceMiddleware); diff --git a/fluxer_api/src/api/app/tests/MiddlewarePipeline.test.ts b/fluxer_api/src/api/app/tests/MiddlewarePipeline.test.ts new file mode 100644 index 000000000..0ef8e1140 --- /dev/null +++ b/fluxer_api/src/api/app/tests/MiddlewarePipeline.test.ts @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {configureMiddleware} from '@app/api/app/MiddlewarePipeline'; +import {TorExitMiddleware} from '@app/api/middleware/TorExitMiddleware'; +import {NoopLogger} from '@app/api/test/mocks/NoopLogger'; +import type {HonoEnv} from '@app/api/types/HonoEnv'; +import {Hono} from 'hono'; +import {describe, expect, it} from 'vitest'; + +function registeredHandlers(torExitBlockingEnabled: boolean): Array { + const routes = new Hono({strict: true}); + configureMiddleware(routes, { + logger: new NoopLogger(), + nodeEnv: 'test', + corsOrigins: ['http://localhost:3000'], + trustClientIpHeader: true, + clientIpHeaderName: 'x-forwarded-for', + maxInflightRequests: 100, + torExitBlockingEnabled, + }); + return routes.routes.map((route) => route.handler); +} + +describe('tor exit blocking in the middleware pipeline', () => { + it('registers the tor exit middleware when the switch is on', () => { + expect(registeredHandlers(true)).toContain(TorExitMiddleware); + }); + + it('leaves the tor exit middleware unregistered when the switch is off', () => { + expect(registeredHandlers(false)).not.toContain(TorExitMiddleware); + }); +}); diff --git a/fluxer_api/src/api/auth/AuthPassword.ts b/fluxer_api/src/api/auth/AuthPassword.ts index d94a40c12..950a8c1f0 100644 --- a/fluxer_api/src/api/auth/AuthPassword.ts +++ b/fluxer_api/src/api/auth/AuthPassword.ts @@ -5,6 +5,7 @@ import type {ApiContext} from '@app/api/ApiContext'; import * as AuthSession from '@app/api/auth/AuthSession'; import * as AuthUtility from '@app/api/auth/AuthUtility'; import {createMfaTicket, createPasswordResetToken} from '@app/api/BrandedTypes'; +import {Config} from '@app/api/Config'; import {Logger} from '@app/api/Logger'; import type {User} from '@app/api/models/User'; import {EXTERNAL_RESPONSE_LIMITS} from '@app/api/utils/ExternalResponseLimits'; @@ -116,6 +117,9 @@ export async function verifyPassword( } export async function isPasswordPwned(_ctx: ApiContext, password: string): Promise { + if (!Config.breachedPasswordCheck.enabled) { + return false; + } const hashed = crypto.createHash('sha1').update(password).digest('hex').toUpperCase(); const hashPrefix = hashed.slice(0, 5); const hashSuffix = hashed.slice(5); diff --git a/fluxer_api/src/api/auth/tests/PwnedPasswordCheck.test.ts b/fluxer_api/src/api/auth/tests/PwnedPasswordCheck.test.ts index e25546ded..b5b5ac93a 100644 --- a/fluxer_api/src/api/auth/tests/PwnedPasswordCheck.test.ts +++ b/fluxer_api/src/api/auth/tests/PwnedPasswordCheck.test.ts @@ -3,6 +3,7 @@ import crypto from 'node:crypto'; import type {ApiContext} from '@app/api/ApiContext'; import {isPasswordPwned, resetPwnedPasswordCacheForTesting} from '@app/api/auth/AuthPassword'; +import {getConfig} from '@app/api/Config'; import {server} from '@app/api/test/msw/server'; import {delay, HttpResponse, http} from 'msw'; import {beforeEach, describe, expect, test} from 'vitest'; @@ -68,6 +69,19 @@ describe('isPasswordPwned', () => { await expect(isPasswordPwned(ctx, SAFE_PASSWORD_SAME_PREFIX)).resolves.toBe(false); expect(requestedPrefixes).toHaveLength(1); }); + test('makes no upstream call when the check is switched off', async () => { + const config = getConfig(); + const originalEnabled = config.breachedPasswordCheck.enabled; + const requestedPrefixes: Array = []; + server.use(rangeHandler(requestedPrefixes, [suffixOf(PWNED_PASSWORD)])); + try { + config.breachedPasswordCheck.enabled = false; + await expect(isPasswordPwned(ctx, PWNED_PASSWORD)).resolves.toBe(false); + expect(requestedPrefixes).toHaveLength(0); + } finally { + config.breachedPasswordCheck.enabled = originalEnabled; + } + }); test('fails open on a non-OK response', async () => { server.use(http.get('https://api.pwnedpasswords.com/range/:prefix', () => HttpResponse.text('', {status: 503}))); await expect(isPasswordPwned(ctx, PWNED_PASSWORD)).resolves.toBe(false); diff --git a/fluxer_api/src/api/config/APIConfig.ts b/fluxer_api/src/api/config/APIConfig.ts index 26b68a04e..00ceba00b 100644 --- a/fluxer_api/src/api/config/APIConfig.ts +++ b/fluxer_api/src/api/config/APIConfig.ts @@ -207,6 +207,12 @@ export interface APIConfig { blocklistFeeds: { enabled: boolean; }; + torExitList: { + enabled: boolean; + }; + breachedPasswordCheck: { + enabled: boolean; + }; captcha: { enabled: boolean; provider: 'hcaptcha' | 'turnstile' | 'none'; diff --git a/fluxer_docs/src/content/docs/http-api/users/current-user.mdx b/fluxer_docs/src/content/docs/http-api/users/current-user.mdx index 1258899d7..5679990be 100644 --- a/fluxer_docs/src/content/docs/http-api/users/current-user.mdx +++ b/fluxer_docs/src/content/docs/http-api/users/current-user.mdx @@ -97,7 +97,7 @@ Modifies the current account and returns the resulting [user](/http-api/users/#u 4 Any string value returns [`EMAIL_MUST_BE_CHANGED_VIA_TOKEN`](/http-api/errors/) on the `email` path, joined by `INVALID_EMAIL_FORMAT` when the string is not an address -5 A claimed account also supplies `password`, and a body that omits it is rejected with `PASSWORD_NOT_SET`. A breached password is rejected with [`PASSWORD_IS_TOO_COMMON`](/http-api/errors/) +5 A claimed account also supplies `password`, and a body that omits it is rejected with `PASSWORD_NOT_SET`. When the instance checks for breached passwords, a breached one is rejected with [`PASSWORD_IS_TOO_COMMON`](/http-api/errors/) 6 The value may have a `data:...;base64,` prefix. The remaining payload runs 1 to 13981016 characters diff --git a/fluxer_docs/src/content/docs/operator/configuration.mdx b/fluxer_docs/src/content/docs/operator/configuration.mdx index 06fc5971d..e4766c19b 100644 --- a/fluxer_docs/src/content/docs/operator/configuration.mdx +++ b/fluxer_docs/src/content/docs/operator/configuration.mdx @@ -965,18 +965,6 @@ Default empty. The ipinfo key. Paired with `FLUXER_RISK_INTEGRATION_ENABLED`. No default. The account risk policy. JSON. Malformed JSON fails startup, and an unknown key in a well-formed policy surfaces when the policy runs. -#### `FLUXER_RISK_TOR_BLOCK_ALL_RELAYS` - -Default `false`. Whether every Tor relay is blocked. Covers entry and middle relays as well as exit nodes. - -#### `FLUXER_RISK_TOR_REVERSE_DNS_HEURISTIC` - -Default `false`. Reverse DNS Tor detection. Adds a lookup to the request path. - -#### `FLUXER_RISK_TOR_REVERSE_DNS_TIMEOUT_MS` - -Default `750`. The lookup timeout. Milliseconds. - #### `FLUXER_ABUSE_INBOUND_PHONE_COUNTRY_CODES` Default empty. Allowed inbound phone countries. Comma separated, passed through unvalidated. @@ -1009,6 +997,14 @@ Default `flag_spammer`. What happens when it fires. `flag_spammer` or `suppress_ Defaults to the inverse of `FLUXER_SELF_HOSTED`. External blocklist feeds. Off by default on a self-hosted instance. +#### `FLUXER_TOR_EXIT_LIST_ENABLED` + +Defaults to the inverse of `FLUXER_SELF_HOSTED`. Tor exit blocking. The API fetches the exit relay list from `onionoo.torproject.org` at startup and every 30 minutes. Off by default on a self-hosted instance. + +#### `FLUXER_BREACHED_PASSWORD_CHECK_ENABLED` + +Defaults to the inverse of `FLUXER_SELF_HOSTED`. Breached password rejection. Sends the first five characters of the password's SHA-1 hash to `api.pwnedpasswords.com`. Off by default on a self-hosted instance. + A second family, unrelated to the rules above, tunes the IP auto-banner: `FLUXER_ABUSE_WINDOW_MS`, the `FLUXER_ABUSE_THRESHOLD_` names, the `FLUXER_ABUSE_TOKEN_DIVERSITY_` names, `FLUXER_ABUSE_BAN_TTL_SEC`, `FLUXER_ABUSE_BATCH_FLUSH_MS`, `FLUXER_ABUSE_MAX_BATCH_TICKS`, `FLUXER_ABUSE_MAX_NEW_TOKENS_PER_TICK`, `FLUXER_ABUSE_MAX_TRACKED_IPS`, `FLUXER_ABUSE_MAX_TOKEN_HASHES_PER_IP`, `FLUXER_ABUSE_MIN_SCORE_FOR_LOOKUP`, `FLUXER_ABUSE_MIN_TOKENS_FOR_LOOKUP`, and `FLUXER_ABUSE_REQUIRED_SCORE_WINDOWS_FOR_AUTO_BAN`. All are read directly from the environment, none are in `.env.example` or the Compose file, and a non-finite value or one at or below zero falls back to the default. `FLUXER_ABUSE_EXEMPT_ASNS` takes comma-separated ASN numbers that the auto-banner never bans. Non-numeric entries are dropped. Before it buys a classification the auto-banner resolves the IP against the local MaxMind ASN database, and an IP on a listed ASN is neither classified nor banned. With no MaxMind ASN database the list has no effect. It does not change admin or guild IP bans, and `FLUXER_API_IP_BAN_EXEMPT_IPS` remains the way to exempt single addresses and ranges from every kind of IP ban. diff --git a/packages/config/src/ConfigLoader.ts b/packages/config/src/ConfigLoader.ts index 2c56010af..de5828046 100644 --- a/packages/config/src/ConfigLoader.ts +++ b/packages/config/src/ConfigLoader.ts @@ -252,15 +252,12 @@ function defaultConfig(): MasterConfig { }, }, blocklist_feeds: {}, + tor_exit_list: {}, + breached_password_check: {}, risk_integration: { enabled: false, ipinfo_api_key: '', account_policy_dsl: undefined, - tor: { - block_all_relays: false, - reverse_dns_heuristic: false, - reverse_dns_timeout_ms: 750, - }, }, push: { apns: { diff --git a/packages/config/src/MasterConfig.ts b/packages/config/src/MasterConfig.ts index 8e6c04998..ed91f8340 100644 --- a/packages/config/src/MasterConfig.ts +++ b/packages/config/src/MasterConfig.ts @@ -298,15 +298,16 @@ export interface MasterConfig { blocklist_feeds: { enabled?: boolean; }; + tor_exit_list: { + enabled?: boolean; + }; + breached_password_check: { + enabled?: boolean; + }; risk_integration: { enabled: boolean; ipinfo_api_key: string; account_policy_dsl?: unknown; - tor: { - block_all_relays: boolean; - reverse_dns_heuristic: boolean; - reverse_dns_timeout_ms: number; - }; }; push: { apns: { diff --git a/packages/config/src/__tests__/ConfigLoader.test.ts b/packages/config/src/__tests__/ConfigLoader.test.ts index 7841d0696..17d8e188f 100644 --- a/packages/config/src/__tests__/ConfigLoader.test.ts +++ b/packages/config/src/__tests__/ConfigLoader.test.ts @@ -638,6 +638,27 @@ describe('ConfigLoader', () => { expect((await loadConfig()).integrations.cache_purge.adapter).toBe('none'); }); + test('leaves the optional outbound lookups unset by default', async () => { + stubMinimalEnv(); + + const config = await loadConfig(); + + expect(config.integrations.tor_exit_list.enabled).toBeUndefined(); + expect(config.integrations.breached_password_check.enabled).toBeUndefined(); + }); + + test('reads the optional outbound lookup switches from the environment', async () => { + stubMinimalEnv({ + FLUXER_TOR_EXIT_LIST_ENABLED: 'true', + FLUXER_BREACHED_PASSWORD_CHECK_ENABLED: 'false', + }); + + const config = await loadConfig(); + + expect(config.integrations.tor_exit_list.enabled).toBe(true); + expect(config.integrations.breached_password_check.enabled).toBe(false); + }); + test('leaves Bluesky login off with no legal URLs by default', async () => { stubMinimalEnv(); diff --git a/packages/config/src/config_loader/EnvironmentOverrides.ts b/packages/config/src/config_loader/EnvironmentOverrides.ts index 11fa34e24..96629e6ce 100644 --- a/packages/config/src/config_loader/EnvironmentOverrides.ts +++ b/packages/config/src/config_loader/EnvironmentOverrides.ts @@ -340,24 +340,17 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record = { parse: parseInteger, }, FLUXER_BLOCKLIST_FEEDS_ENABLED: {path: ['integrations', 'blocklist_feeds', 'enabled'], parse: parseBoolean}, + FLUXER_TOR_EXIT_LIST_ENABLED: {path: ['integrations', 'tor_exit_list', 'enabled'], parse: parseBoolean}, + FLUXER_BREACHED_PASSWORD_CHECK_ENABLED: { + path: ['integrations', 'breached_password_check', 'enabled'], + parse: parseBoolean, + }, FLUXER_RISK_INTEGRATION_ENABLED: {path: ['integrations', 'risk_integration', 'enabled'], parse: parseBoolean}, FLUXER_RISK_IPINFO_API_KEY: {path: ['integrations', 'risk_integration', 'ipinfo_api_key']}, FLUXER_ACCOUNT_POLICY_DSL: { path: ['integrations', 'risk_integration', 'account_policy_dsl'], parse: parseEnvValue, }, - FLUXER_RISK_TOR_BLOCK_ALL_RELAYS: { - path: ['integrations', 'risk_integration', 'tor', 'block_all_relays'], - parse: parseBoolean, - }, - FLUXER_RISK_TOR_REVERSE_DNS_HEURISTIC: { - path: ['integrations', 'risk_integration', 'tor', 'reverse_dns_heuristic'], - parse: parseBoolean, - }, - FLUXER_RISK_TOR_REVERSE_DNS_TIMEOUT_MS: { - path: ['integrations', 'risk_integration', 'tor', 'reverse_dns_timeout_ms'], - parse: parseInteger, - }, FLUXER_PUSH_APNS_ENABLED: {path: ['integrations', 'push', 'apns', 'enabled'], parse: parseBoolean}, FLUXER_PUSH_APNS_TEAM_ID: {path: ['integrations', 'push', 'apns', 'team_id']}, FLUXER_PUSH_APNS_KEY_ID: {path: ['integrations', 'push', 'apns', 'key_id']},