fix(api): dedupe and budget ipinfo lookups across api pods (#2584)

This commit is contained in:
Hampus authored and GitHub committed 2026-09-08 15:13:32 +02:00
1 parent a2a68847fd
commit 824b5c86c9
24 files changed
+2125 -66

No files matched your search

@@ -15,6 +15,7 @@ import type {HonoEnv} from '../types/HonoEnv';
import {parseJsonRecord} from '../utils/JsonBoundaryUtils';
import {ipBanCache} from './IpBanMiddleware';
import {getIpInfoService} from './ServiceMiddleware';
import {getKVClient} from './ServiceRegistry';
import {getCacheService} from './ServiceSingletons';
type IpClass = 'datacenter' | 'anonymous' | 'mobile' | 'residential' | 'unknown';
@@ -51,6 +52,22 @@ interface AbuseSignalOptions {
weight?: number;
}
interface PeerIpClassHint {
ipClass: IpClass;
expiresAtMs: number;
}
interface OutboundIpClass {
lookupIp: string;
ipClass: IpClass;
}
interface ResolvedBanClass {
ipClass: IpClass;
authoritative: boolean;
blocked: boolean;
}
const WINDOW_MS = positiveNumberFromEnv('FLUXER_ABUSE_WINDOW_MS', 60_000);
const THRESHOLD_DATACENTER = positiveNumberFromEnv('FLUXER_ABUSE_THRESHOLD_DATACENTER', 20);
const THRESHOLD_ANONYMOUS = positiveNumberFromEnv('FLUXER_ABUSE_THRESHOLD_ANONYMOUS', 500);
@@ -73,6 +90,14 @@ const REQUIRED_SCORE_WINDOWS_FOR_AUTO_BAN = positiveNumberFromEnv(
3,
);
const REPLICATION_CHANNEL = 'abuse_tracker:ticks';
const IP_CLASS_CHANNEL = 'abuse_tracker:ipclass';
const IP_CLASS_CLAIM_PREFIX = 'abuse:ipclass:claim:';
const IP_CLASS_CLAIM_ENABLED = process.env.FLUXER_ABUSE_IP_CLASS_CLAIM_ENABLED !== '0';
const IP_CLASS_CLAIM_TTL_SECONDS = positiveNumberFromEnv('FLUXER_ABUSE_IP_CLASS_CLAIM_TTL_SEC', 15);
const DEFAULT_IP_CLASS_PENDING_TTL_MS = positiveNumberFromEnv('FLUXER_ABUSE_IP_CLASS_PENDING_TTL_MS', 20_000);
const DEFAULT_IP_CLASS_NEGATIVE_TTL_MS = positiveNumberFromEnv('FLUXER_ABUSE_IP_CLASS_NEGATIVE_TTL_MS', 300_000);
const DEFAULT_IP_CLASS_HINT_TTL_MS = positiveNumberFromEnv('FLUXER_ABUSE_IP_CLASS_HINT_TTL_MS', 600_000);
const IP_CLASSES = ['datacenter', 'anonymous', 'mobile', 'residential', 'unknown'] as const;
const POD_ID = process.env.HOSTNAME ?? randomUUID();
type ReplicatedTick = [banKey: string, scoreDelta: number, tokenHashes: Array<string>, lookupIp: string];
@@ -83,11 +108,23 @@ interface ReplicationMessage {
ts: number;
}
type IpClassEntry = [banKey: string, lookupIp: string, ipClass: IpClass];
interface IpClassMessage {
sender: string;
entries: Array<IpClassEntry>;
ts: number;
}
const records = new Map<string, AbuseRecord>();
const outboundDeltas = new Map<string, OutboundEntry>();
const persistentScoreWindows = new Map<string, PersistentScoreState>();
const ipClassCache = new Map<string, IpClass>();
const ipClassPending = new Set<string>();
const ipClassPending = new Map<string, number>();
const ipClassNegativeUntil = new Map<string, number>();
const peerIpClassHints = new Map<string, PeerIpClassHint>();
const outboundIpClasses = new Map<string, OutboundIpClass>();
const pendingIpClassTasks = new Set<Promise<void>>();
const recordedClientErrorRequests = new WeakSet<Request>();
const pendingAutoBanTasks = new Set<Promise<void>>();
const adminRepository = new AdminRepository();
@@ -97,6 +134,9 @@ let flushTimer: NodeJS.Timeout | null = null;
let kvSubscription: IKVSubscription | null = null;
let messageHandler: ((channel: string, message: string) => void) | null = null;
let errorHandler: ((error: Error) => void) | null = null;
let ipClassPendingTtlMs = DEFAULT_IP_CLASS_PENDING_TTL_MS;
let ipClassNegativeTtlMs = DEFAULT_IP_CLASS_NEGATIVE_TTL_MS;
let ipClassHintTtlMs = DEFAULT_IP_CLASS_HINT_TTL_MS;
function positiveNumberFromEnv(name: string, fallback: number): number {
const raw = process.env[name];
@@ -164,13 +204,99 @@ function shouldSkipAutoBanForIpClass(ipClass: IpClass): boolean {
return ipClass === 'mobile';
}
function isIpClass(value: unknown): value is IpClass {
return typeof value === 'string' && (IP_CLASSES as ReadonlyArray<string>).includes(value);
}
function getOwnIpClass(key: string, now: number): IpClass | null {
const cached = ipClassCache.get(key);
if (cached === undefined) return null;
const negativeUntilMs = ipClassNegativeUntil.get(key);
if (negativeUntilMs !== undefined && negativeUntilMs <= now) {
ipClassCache.delete(key);
ipClassNegativeUntil.delete(key);
return null;
}
return cached;
}
function isOwnIpClassNegative(key: string, now: number): boolean {
const negativeUntilMs = ipClassNegativeUntil.get(key);
return negativeUntilMs !== undefined && negativeUntilMs > now;
}
function setOwnIpClass(key: string, lookupIp: string, ipClass: IpClass, negative: boolean): void {
ipClassCache.set(key, ipClass);
if (negative) {
ipClassNegativeUntil.set(key, Date.now() + ipClassNegativeTtlMs);
} else {
ipClassNegativeUntil.delete(key);
peerIpClassHints.delete(key);
}
ipClassPending.delete(key);
if (!negative && ipClass !== 'unknown') {
queueOutboundIpClass(key, lookupIp, ipClass);
}
const rec = records.get(key);
if (rec) maybeFireAutoBan(key, rec);
}
function isIpClassPending(key: string, now: number): boolean {
const expiresAtMs = ipClassPending.get(key);
if (expiresAtMs === undefined) return false;
if (expiresAtMs <= now) {
ipClassPending.delete(key);
return false;
}
return true;
}
function getPeerClassHint(key: string, now: number): IpClass | null {
const hint = peerIpClassHints.get(key);
if (!hint) return null;
if (hint.expiresAtMs <= now) {
peerIpClassHints.delete(key);
return null;
}
return hint.ipClass;
}
function recordPeerClassHint(key: string, ipClass: IpClass): void {
peerIpClassHints.set(key, {ipClass, expiresAtMs: Date.now() + ipClassHintTtlMs});
}
function isStricterThanUnknown(ipClass: IpClass): boolean {
return (
scoreThresholdFor(ipClass) <= scoreThresholdFor('unknown') &&
tokenDiversityThresholdFor(ipClass) <= tokenDiversityThresholdFor('unknown')
);
}
function resolveClassForBan(key: string, now: number): ResolvedBanClass {
const own = getOwnIpClass(key, now);
if (own !== null && !isOwnIpClassNegative(key, now)) return {ipClass: own, authoritative: true, blocked: false};
const hint = getPeerClassHint(key, now);
if (hint !== null && isStricterThanUnknown(hint)) return {ipClass: hint, authoritative: true, blocked: false};
if (hint !== null) return {ipClass: 'unknown', authoritative: false, blocked: true};
if (own !== null) return {ipClass: own, authoritative: true, blocked: false};
return {ipClass: 'unknown', authoritative: false, blocked: isIpClassPending(key, now)};
}
function pruneIfNeeded(now: number): void {
if (records.size < MAX_TRACKED_IPS) return;
for (const [key, expiresAtMs] of ipClassPending) {
if (expiresAtMs <= now) ipClassPending.delete(key);
}
for (const [key, hint] of peerIpClassHints) {
if (hint.expiresAtMs <= now) peerIpClassHints.delete(key);
}
for (const [key, rec] of records) {
if (rec.windowStartMs + WINDOW_MS < now) {
if (rec.windowStartMs + WINDOW_MS < now && !ipClassPending.has(key)) {
records.delete(key);
ipClassCache.delete(key);
ipClassPending.delete(key);
ipClassNegativeUntil.delete(key);
peerIpClassHints.delete(key);
outboundIpClasses.delete(key);
}
if (records.size < MAX_TRACKED_IPS * 0.9) return;
}
@@ -222,9 +348,11 @@ function queueOutboundDelta(
}
}
function shouldEnsureIpClassLookup(key: string, rec: AbuseRecord): boolean {
if (ipClassCache.has(key) || ipClassPending.has(key)) return false;
return rec.score >= MIN_SCORE_FOR_IP_LOOKUP || rec.distinctTokenHashes.size >= MIN_TOKENS_FOR_IP_LOOKUP;
function shouldEnsureIpClassLookup(key: string, rec: AbuseRecord, now: number): boolean {
if (getOwnIpClass(key, now) !== null || isIpClassPending(key, now)) return false;
if (getPeerClassHint(key, now) !== null) return false;
if (rec.score < MIN_SCORE_FOR_IP_LOOKUP && rec.distinctTokenHashes.size < MIN_TOKENS_FOR_IP_LOOKUP) return false;
return ipBanCache.getMatch(rec.lookupIp) === null;
}
function markScoreThresholdWindow(key: string, rec: AbuseRecord, now: number): number {
@@ -247,33 +375,53 @@ function markScoreThresholdWindow(key: string, rec: AbuseRecord, now: number): n
return state.count;
}
async function claimIpClassLookup(key: string): Promise<boolean> {
if (!IP_CLASS_CLAIM_ENABLED) return true;
if (!kvPublisher) return true;
try {
return await getKVClient().setnx(`${IP_CLASS_CLAIM_PREFIX}${key}`, POD_ID, IP_CLASS_CLAIM_TTL_SECONDS);
} catch {
return true;
}
}
async function runIpClassLookup(key: string, lookupIp: string): Promise<void> {
try {
if (!(await claimIpClassLookup(key))) return;
const result = await getIpInfoService().lookup(lookupIp, {source: 'AbusiveIpAutoBanner', reason: 'classify'});
setOwnIpClass(key, lookupIp, classifyIpInfo(result), !result.available);
} catch (err) {
setOwnIpClass(key, lookupIp, 'unknown', true);
Logger.warn({err, ip: lookupIp}, '[abuse-auto-ban] IP classification lookup failed');
}
}
function ensureIpClassLookup(key: string, lookupIp: string): void {
if (ipClassCache.has(key) || ipClassPending.has(key)) return;
ipClassPending.add(key);
void (async () => {
try {
const result = await getIpInfoService().lookup(lookupIp, {source: 'AbusiveIpAutoBanner', reason: 'classify'});
ipClassCache.set(key, classifyIpInfo(result));
} catch (err) {
ipClassCache.set(key, 'unknown');
Logger.warn({err, ip: lookupIp}, '[abuse-auto-ban] IP classification lookup failed');
} finally {
ipClassPending.delete(key);
const rec = records.get(key);
if (rec) maybeFireAutoBan(key, rec);
}
})();
const now = Date.now();
if (getOwnIpClass(key, now) !== null || isIpClassPending(key, now)) return;
ipClassPending.set(key, now + ipClassPendingTtlMs);
const task = runIpClassLookup(key, lookupIp);
pendingIpClassTasks.add(task);
void task.finally(() => {
pendingIpClassTasks.delete(task);
});
}
function maybeFireAutoBan(key: string, rec: AbuseRecord): void {
if (rec.autoBanFired) return;
const ipClass = ipClassCache.get(key) ?? 'unknown';
const now = Date.now();
if (ipBanCache.getMatch(rec.lookupIp) !== null) {
rec.autoBanFired = true;
return;
}
const resolved = resolveClassForBan(key, now);
const ipClass = resolved.ipClass;
const scoreThreshold = scoreThresholdFor(ipClass);
const tokenThreshold = tokenDiversityThresholdFor(ipClass);
const overScore = rec.score >= scoreThreshold;
const overTokenDiversity = rec.distinctTokenHashes.size >= tokenThreshold;
if (!overScore && !overTokenDiversity) return;
if (!ipClassCache.has(key) && ipClassPending.has(key)) {
if (resolved.blocked) {
return;
}
if (shouldSkipAutoBanForIpClass(ipClass)) {
@@ -369,7 +517,7 @@ export function recordAbuseSignal(ip: string | null, reason: string, opts: Abuse
queuedTokenHash = opts.tokenHash;
}
queueOutboundDelta(signalIp, weight, queuedTokenHash, hadToken);
if (shouldEnsureIpClassLookup(signalIp.banKey, rec)) {
if (shouldEnsureIpClassLookup(signalIp.banKey, rec, now)) {
ensureIpClassLookup(signalIp.banKey, signalIp.lookupIp);
}
maybeFireAutoBan(signalIp.banKey, rec);
@@ -404,7 +552,7 @@ function applyReplicatedTick(tick: ReplicatedTick): void {
if (rec.distinctTokenHashes.size >= MAX_TOKEN_HASHES_PER_IP) break;
rec.distinctTokenHashes.add(tokenHash);
}
if (shouldEnsureIpClassLookup(banKey, rec)) {
if (shouldEnsureIpClassLookup(banKey, rec, now)) {
ensureIpClassLookup(banKey, lookupIp);
}
maybeFireAutoBan(banKey, rec);
@@ -435,7 +583,56 @@ async function flushOutbound(): Promise<void> {
}
}
function queueOutboundIpClass(key: string, lookupIp: string, ipClass: IpClass): void {
if (!kvPublisher) return;
if (!outboundIpClasses.has(key) && outboundIpClasses.size >= MAX_TRACKED_IPS) return;
outboundIpClasses.set(key, {lookupIp, ipClass});
}
async function flushOutboundIpClasses(): Promise<void> {
if (!kvPublisher || outboundIpClasses.size === 0) return;
const entries: Array<IpClassEntry> = [];
const selectedKeys: Array<string> = [];
for (const [key, entry] of outboundIpClasses) {
entries.push([key, entry.lookupIp, entry.ipClass]);
selectedKeys.push(key);
if (entries.length >= MAX_BATCH_TICKS) break;
}
const message: IpClassMessage = {sender: POD_ID, entries, ts: Date.now()};
try {
await kvPublisher.publish(IP_CLASS_CHANNEL, JSON.stringify(message));
for (const key of selectedKeys) {
outboundIpClasses.delete(key);
}
} catch (err) {
Logger.warn({err, entryCount: entries.length}, '[abuse-auto-ban] Failed to publish abuse IP class batch');
}
}
function handleIpClassMessage(message: string): void {
const msg = parseJsonRecord(message);
if (!msg || msg.sender === POD_ID || !Array.isArray(msg.entries)) return;
for (const rawEntry of msg.entries) {
if (!Array.isArray(rawEntry) || rawEntry.length < 3) continue;
const [banKey, lookupIp, ipClass] = rawEntry;
if (typeof banKey !== 'string' || typeof lookupIp !== 'string' || !isIpClass(ipClass)) continue;
if (ipClass === 'unknown') continue;
const signalIp = normalizeSignalIp(lookupIp);
if (!signalIp || signalIp.banKey !== banKey) continue;
const rec = records.get(banKey);
if (!rec) continue;
const now = Date.now();
if (getOwnIpClass(banKey, now) !== null && !isOwnIpClassNegative(banKey, now)) continue;
recordPeerClassHint(banKey, ipClass);
maybeFireAutoBan(banKey, rec);
}
}
function handleReplicationMessage(channel: string, message: string): void {
if (channel === IP_CLASS_CHANNEL) {
handleIpClassMessage(message);
return;
}
if (channel !== REPLICATION_CHANNEL) return;
const msg = parseJsonRecord(message);
if (!msg || msg.sender === POD_ID || !Array.isArray(msg.ticks)) return;
@@ -469,11 +666,12 @@ export async function startAbuseReplicationSubscriber(kvClient: IKVProvider | nu
};
try {
await subscription.connect();
await subscription.subscribe(REPLICATION_CHANNEL);
await subscription.subscribe(REPLICATION_CHANNEL, IP_CLASS_CHANNEL);
subscription.on('message', messageHandler);
subscription.on('error', errorHandler);
flushTimer = setInterval(() => {
void flushOutbound();
void flushOutboundIpClasses();
}, BATCH_FLUSH_MS);
if (typeof flushTimer === 'object' && flushTimer && 'unref' in flushTimer) {
(flushTimer as {unref(): void}).unref();
@@ -514,11 +712,28 @@ export async function drainAbuseAutoBanTasksForTests(): Promise<void> {
await Promise.all([...pendingAutoBanTasks]);
}
export async function drainAbuseIpClassLookupsForTests(): Promise<void> {
await Promise.all([...pendingIpClassTasks]);
}
export function setAbuseIpClassTtlsForTests(opts: {negativeMs?: number; hintMs?: number; pendingMs?: number}): void {
if (opts.negativeMs !== undefined) ipClassNegativeTtlMs = opts.negativeMs;
if (opts.hintMs !== undefined) ipClassHintTtlMs = opts.hintMs;
if (opts.pendingMs !== undefined) ipClassPendingTtlMs = opts.pendingMs;
}
export function resetAbuseTrackingForTests(): void {
records.clear();
outboundDeltas.clear();
persistentScoreWindows.clear();
ipClassCache.clear();
ipClassPending.clear();
ipClassNegativeUntil.clear();
peerIpClassHints.clear();
outboundIpClasses.clear();
pendingIpClassTasks.clear();
pendingAutoBanTasks.clear();
ipClassPendingTtlMs = DEFAULT_IP_CLASS_PENDING_TTL_MS;
ipClassNegativeTtlMs = DEFAULT_IP_CLASS_NEGATIVE_TTL_MS;
ipClassHintTtlMs = DEFAULT_IP_CLASS_HINT_TTL_MS;
}
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import crypto from 'node:crypto';
import {lookupAsnByIp, lookupGeoipByIp} from '@pkgs/geoip/src/GeoipLookup';
import {createIpInfoService, createUnavailableIpInfoService, type IpInfoService} from '@pkgs/geoip/src/IpInfoService';
import {createMiddleware} from 'hono/factory';
import type {ApiContext} from '../ApiContext';
@@ -58,8 +59,14 @@ import {createIpInfoChecker} from '../risk/adapters/IpInfoAdapter';
import {createReverseDnsLookup} from '../risk/adapters/ReverseDnsAdapter';
import {DeterministicRiskEngine} from '../risk/DeterministicRiskEngine';
import {CassandraHistoricalOutcomeRepository} from '../risk/HistoricalOutcomeRepository';
import {createKvIpInfoLookupBudget} from '../risk/IpInfoBudget';
import {buildIpInfoCache, buildIpInfoRequestAuditLogger} from '../risk/IpInfoCacheFactory';
import {CassandraRegistrationEventsRepository} from '../risk/RegistrationEventsRepository';
import {
type IpInfoPrescreenVerdict,
ipInfoPrescreenOptionsFromEnv,
prescreenIpInfoLookup,
} from '../risk/RegistrationIpPrescreen';
import {CassandraRiskAssessmentRepository} from '../risk/RiskAssessmentRepository';
import {createRiskToolbox} from '../risk/RiskToolboxFactory';
import {CassandraSuspiciousIpRepository} from '../risk/SuspiciousIpRepository';
@@ -268,6 +275,7 @@ export function getIpInfoService(): IpInfoService {
apiKey: Config.risk.ipinfoApiKey,
cache,
auditLogger: buildIpInfoRequestAuditLogger(),
budget: createKvIpInfoLookupBudget({getKvClient: getKVClient}),
});
return _ipInfoService;
}
@@ -293,7 +301,14 @@ function getRegistrationRiskEvaluator(): IRegistrationRiskEvaluator {
return _registrationRiskEvaluator;
}
const ipInfoService = getIpInfoService();
const ipInfoChecker = Config.risk.ipinfoApiKey ? createIpInfoChecker({ipInfoService}) : undefined;
const lookupLocalCity = (ip: string) => lookupGeoipByIp(ip, Config.geoip.maxmindDbPath);
const lookupLocalAsn = (ip: string) => lookupAsnByIp(ip, Config.geoip.maxmindAsnDbPath);
const prescreenOptions = ipInfoPrescreenOptionsFromEnv();
const prescreen = async (ip: string): Promise<IpInfoPrescreenVerdict> => {
const [city, asn] = await Promise.all([lookupLocalCity(ip), lookupLocalAsn(ip)]);
return prescreenIpInfoLookup({countryIso: city.countryCode, asn: asn.asn, asnOrg: asn.asnOrg}, prescreenOptions);
};
const ipInfoChecker = Config.risk.ipinfoApiKey ? createIpInfoChecker({ipInfoService, prescreen}) : undefined;
const cacheService = getCacheService();
const reverseDnsLookup = createReverseDnsLookup({cacheService});
const toolbox = createRiskToolbox({
@@ -305,6 +320,8 @@ function getRegistrationRiskEvaluator(): IRegistrationRiskEvaluator {
historicalOutcomeRepository: getHistoricalOutcomeRepository(),
suspiciousIpRepository: getSuspiciousIpRepository(),
cacheService,
lookupLocalCity,
lookupLocalAsn,
});
const engine = new DeterministicRiskEngine(toolbox, {
logger: Logger,
@@ -1,15 +1,21 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {getSameIpDecisionKey} from '@fluxer/ip_utils/src/IpAddress';
import type {IpInfoLookupResult} from '@pkgs/geoip/src/IpInfoService';
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
import {AdminRepository} from '../../admin/AdminRepository';
import type {ApiTestHarness} from '../../test/ApiTestHarness';
import {createApiTestHarness} from '../../test/ApiTestHarness';
import type {MockKVProvider} from '../../test/mocks/MockKVProvider';
import {
drainAbuseAutoBanTasksForTests,
drainAbuseIpClassLookupsForTests,
hashAuthToken,
recordAbuseSignal,
resetAbuseTrackingForTests,
setAbuseIpClassTtlsForTests,
startAbuseReplicationSubscriber,
stopAbuseReplicationSubscriber,
} from '../AbusiveIpAutoBanner';
import {ipBanCache} from '../IpBanMiddleware';
import {setInjectedIpInfoService} from '../ServiceMiddleware';
@@ -65,6 +71,11 @@ function ipInfoResult(ip: string, overrides: Partial<IpInfoLookupResult> = {}):
};
}
function claimCallCount(harness: ApiTestHarness, banKey: string): number {
const kvProvider = harness.kvProvider as MockKVProvider;
return kvProvider.setnxSpy.mock.calls.filter(([key]) => key === `abuse:ipclass:claim:${banKey}`).length;
}
async function waitForAssertion(assertion: () => void): Promise<void> {
const deadline = Date.now() + 1000;
let lastError: unknown;
@@ -84,6 +95,7 @@ async function waitForAssertion(assertion: () => void): Promise<void> {
describe('AbusiveIpAutoBanner', () => {
let harness: ApiTestHarness;
let adminRepository: AdminRepository;
let lookupCount = 0;
beforeAll(async () => {
harness = await createApiTestHarness();
adminRepository = new AdminRepository();
@@ -92,13 +104,18 @@ describe('AbusiveIpAutoBanner', () => {
await harness.reset();
resetAbuseTrackingForTests();
ipBanCache.resetCaches();
lookupCount = 0;
setInjectedIpInfoService({
async lookup(ip: string) {
lookupCount += 1;
return ipInfoResult(ip);
},
});
await stopAbuseReplicationSubscriber();
await startAbuseReplicationSubscriber(harness.kvProvider);
});
afterAll(async () => {
await stopAbuseReplicationSubscriber();
setInjectedIpInfoService(undefined);
await harness.shutdown();
});
@@ -117,7 +134,7 @@ describe('AbusiveIpAutoBanner', () => {
it('does not auto-ban after a single score-only spike', async () => {
const ip = '8.8.4.4';
recordAbuseSignal(ip, 'http_429', {weight: 150});
await new Promise((resolve) => setTimeout(resolve, 25));
await drainAbuseIpClassLookupsForTests();
await drainAbuseAutoBanTasksForTests();
expect(ipBanCache.isBanned(ip)).toBe(false);
await expect(adminRepository.isIpBanned(ip)).resolves.toBe(false);
@@ -135,7 +152,7 @@ describe('AbusiveIpAutoBanner', () => {
for (let i = 0; i < 10; i += 1) {
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`mobile-invalid-${i}`)});
}
await new Promise((resolve) => setTimeout(resolve, 25));
await drainAbuseIpClassLookupsForTests();
await drainAbuseAutoBanTasksForTests();
expect(ipBanCache.isBanned(ip)).toBe(false);
await expect(adminRepository.isIpBanned(ip)).resolves.toBe(false);
@@ -153,7 +170,7 @@ describe('AbusiveIpAutoBanner', () => {
for (let i = 0; i < 100; i += 1) {
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`mobile-threshold-${i}`)});
}
await new Promise((resolve) => setTimeout(resolve, 25));
await drainAbuseIpClassLookupsForTests();
await drainAbuseAutoBanTasksForTests();
expect(ipBanCache.isBanned(ip)).toBe(false);
await expect(adminRepository.isIpBanned(ip)).resolves.toBe(false);
@@ -168,4 +185,68 @@ describe('AbusiveIpAutoBanner', () => {
await expect(adminRepository.isIpBanned(ip)).resolves.toBe(false);
}
});
it('claims the class lookup exactly once for a burst on the same IP', async () => {
const ip = '8.8.8.8';
for (let i = 0; i < 10; i += 1) {
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`claim-${i}`)});
}
await drainAbuseIpClassLookupsForTests();
await drainAbuseAutoBanTasksForTests();
expect(claimCallCount(harness, ip)).toBe(1);
expect(lookupCount).toBe(1);
});
it('does not pay for a lookup or ban when another pod owns the class claim', async () => {
const ip = '8.8.8.8';
await harness.kvProvider.setnx(`abuse:ipclass:claim:${ip}`, 'other-pod', 60);
for (let i = 0; i < 10; i += 1) {
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`claim-loser-${i}`)});
}
await drainAbuseIpClassLookupsForTests();
await drainAbuseAutoBanTasksForTests();
expect(lookupCount).toBe(0);
expect(ipBanCache.isBanned(ip)).toBe(false);
});
it('does not classify an IPv4 address that is already banned', async () => {
const ip = '8.8.8.8';
ipBanCache.banTemp(ip, 3600);
for (let i = 0; i < 20; i += 1) {
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`already-banned-${i}`)});
}
await drainAbuseIpClassLookupsForTests();
await drainAbuseAutoBanTasksForTests();
expect(lookupCount).toBe(0);
expect(claimCallCount(harness, ip)).toBe(0);
});
it('does not classify an IPv6 address inside an already banned /64', async () => {
const ip = '2606:4700:4700::1111';
const banKey = getSameIpDecisionKey(ip) ?? ip;
ipBanCache.banTemp(banKey, 3600);
for (let i = 0; i < 20; i += 1) {
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`already-banned-v6-${i}`)});
}
await drainAbuseIpClassLookupsForTests();
await drainAbuseAutoBanTasksForTests();
expect(lookupCount).toBe(0);
expect(claimCallCount(harness, banKey)).toBe(0);
});
it('retries a failed classification once the negative TTL elapses', async () => {
const ip = '9.9.9.9';
setAbuseIpClassTtlsForTests({negativeMs: 50});
setInjectedIpInfoService({
async lookup(candidateIp: string) {
lookupCount += 1;
return ipInfoResult(candidateIp, {available: false});
},
});
recordAbuseSignal(ip, 'http_429', {weight: 25});
await drainAbuseIpClassLookupsForTests();
await drainAbuseAutoBanTasksForTests();
expect(lookupCount).toBe(1);
await new Promise((resolve) => setTimeout(resolve, 60));
await harness.kvProvider.del(`abuse:ipclass:claim:${ip}`);
recordAbuseSignal(ip, 'http_429', {weight: 25});
await drainAbuseIpClassLookupsForTests();
await drainAbuseAutoBanTasksForTests();
expect(lookupCount).toBe(2);
});
});
@@ -0,0 +1,203 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {IpInfoLookupResult} from '@pkgs/geoip/src/IpInfoService';
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
import type {ApiTestHarness} from '../../test/ApiTestHarness';
import {createApiTestHarness} from '../../test/ApiTestHarness';
import type {MockKVProvider} from '../../test/mocks/MockKVProvider';
import {
drainAbuseAutoBanTasksForTests,
drainAbuseIpClassLookupsForTests,
hashAuthToken,
recordAbuseSignal,
resetAbuseTrackingForTests,
setAbuseIpClassTtlsForTests,
startAbuseReplicationSubscriber,
stopAbuseReplicationSubscriber,
} from '../AbusiveIpAutoBanner';
import {ipBanCache} from '../IpBanMiddleware';
import {setInjectedIpInfoService} from '../ServiceMiddleware';
function ipInfoResult(ip: string, overrides: Partial<IpInfoLookupResult> = {}): IpInfoLookupResult {
return {
ip,
available: true,
riskNote: 'test',
geo: {
countryCode: 'US',
countryName: 'United States',
continent: 'North America',
continentCode: 'NA',
region: null,
regionCode: null,
city: null,
postalCode: null,
timezone: null,
latitude: null,
longitude: null,
accuracyRadiusKm: null,
},
asn: {
asn: 'AS64500',
number: 64500,
name: 'Test ISP',
domain: null,
type: null,
},
mobile: {
name: null,
mcc: null,
mnc: null,
},
anonymous: {
isAnonymous: false,
providerName: null,
isVpn: false,
isProxy: false,
isResidentialProxy: false,
isTor: false,
isRelay: false,
percentDaysSeen: null,
},
flags: {
isAnycast: false,
isHosting: false,
isMobile: false,
isSatellite: false,
},
...overrides,
};
}
function publishPeerIpClasses(harness: ApiTestHarness, entries: Array<[string, string, unknown]>): void {
const kvProvider = harness.kvProvider as MockKVProvider;
kvProvider
.getSubscription()
.simulateMessage('abuse_tracker:ipclass', JSON.stringify({sender: 'other-pod', entries, ts: Date.now()}));
}
function claimCallCount(harness: ApiTestHarness, banKey: string): number {
const kvProvider = harness.kvProvider as MockKVProvider;
return kvProvider.setnxSpy.mock.calls.filter(([key]) => key === `abuse:ipclass:claim:${banKey}`).length;
}
function recordTokenSignals(ip: string, prefix: string, from: number, to: number): void {
for (let i = from; i < to; i += 1) {
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`${prefix}-${i}`)});
}
}
async function drainAbuseWork(): Promise<void> {
await drainAbuseIpClassLookupsForTests();
await drainAbuseAutoBanTasksForTests();
}
describe('AbusiveIpAutoBanner IP class replication', () => {
let harness: ApiTestHarness;
let lookupCount = 0;
beforeAll(async () => {
harness = await createApiTestHarness();
});
beforeEach(async () => {
await harness.reset();
resetAbuseTrackingForTests();
ipBanCache.resetCaches();
lookupCount = 0;
setInjectedIpInfoService({
async lookup(ip: string) {
lookupCount += 1;
return ipInfoResult(ip);
},
});
await stopAbuseReplicationSubscriber();
await startAbuseReplicationSubscriber(harness.kvProvider);
});
afterAll(async () => {
await stopAbuseReplicationSubscriber();
setInjectedIpInfoService(undefined);
await harness.shutdown();
});
it('adopts a datacenter class from a peer without paying for its own lookup', async () => {
const ip = '8.8.8.8';
recordTokenSignals(ip, 'peer-datacenter', 0, 3);
publishPeerIpClasses(harness, [[ip, ip, 'datacenter']]);
recordTokenSignals(ip, 'peer-datacenter', 3, 10);
await drainAbuseWork();
expect(lookupCount).toBe(0);
expect(claimCallCount(harness, ip)).toBe(0);
expect(ipBanCache.isBanned(ip)).toBe(true);
});
it('never adopts a mobile class from a peer and leaves the IP bannable', async () => {
const ip = '8.8.4.4';
recordTokenSignals(ip, 'peer-mobile', 0, 3);
publishPeerIpClasses(harness, [[ip, ip, 'mobile']]);
recordTokenSignals(ip, 'peer-mobile', 3, 10);
await drainAbuseWork();
expect(lookupCount).toBe(0);
expect(ipBanCache.isBanned(ip)).toBe(false);
publishPeerIpClasses(harness, [[ip, ip, 'datacenter']]);
await drainAbuseWork();
expect(ipBanCache.isBanned(ip)).toBe(true);
});
it('ignores an unrecognised class from a peer and classifies the IP itself', async () => {
const ip = '4.4.4.4';
recordTokenSignals(ip, 'peer-invalid', 0, 3);
publishPeerIpClasses(harness, [
[ip, ip, 'datacentre'],
[ip, ip, 42],
]);
recordTokenSignals(ip, 'peer-invalid', 3, 10);
await drainAbuseWork();
expect(lookupCount).toBe(1);
expect(ipBanCache.isBanned(ip)).toBe(true);
});
it('ignores an unknown class from a peer and classifies the IP itself', async () => {
const ip = '9.9.9.9';
recordTokenSignals(ip, 'peer-unknown', 0, 3);
publishPeerIpClasses(harness, [[ip, ip, 'unknown']]);
recordTokenSignals(ip, 'peer-unknown', 3, 10);
await drainAbuseWork();
expect(lookupCount).toBe(1);
expect(ipBanCache.isBanned(ip)).toBe(true);
});
it('ignores a peer class for an IP it is not tracking', async () => {
const ip = '208.67.222.222';
publishPeerIpClasses(harness, [[ip, ip, 'datacenter']]);
recordTokenSignals(ip, 'peer-untracked', 0, 10);
await drainAbuseWork();
expect(lookupCount).toBe(1);
expect(claimCallCount(harness, ip)).toBe(1);
});
it('accepts a peer mobile class after its own lookup failed and keeps the IP unbanned', async () => {
const ip = '199.85.126.10';
setInjectedIpInfoService({
async lookup(lookupIp: string) {
lookupCount += 1;
return ipInfoResult(lookupIp, {available: false});
},
});
recordAbuseSignal(ip, 'http_429', {weight: 25});
await drainAbuseWork();
expect(lookupCount).toBe(1);
publishPeerIpClasses(harness, [[ip, ip, 'mobile']]);
recordTokenSignals(ip, 'negative-then-mobile', 0, 10);
await drainAbuseWork();
expect(lookupCount).toBe(1);
expect(ipBanCache.isBanned(ip)).toBe(false);
});
it('resumes classifying once a peer class hint expires', async () => {
const ip = '77.88.8.8';
setAbuseIpClassTtlsForTests({hintMs: 50});
recordAbuseSignal(ip, 'http_429', {weight: 3});
publishPeerIpClasses(harness, [[ip, ip, 'datacenter']]);
recordAbuseSignal(ip, 'http_429', {weight: 25});
await drainAbuseWork();
expect(lookupCount).toBe(0);
await new Promise((resolve) => setTimeout(resolve, 80));
recordAbuseSignal(ip, 'http_429', {weight: 25});
await drainAbuseWork();
expect(lookupCount).toBe(1);
expect(claimCallCount(harness, ip)).toBe(1);
expect(ipBanCache.isBanned(ip)).toBe(false);
});
});
+105
View File
@@ -0,0 +1,105 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {IpInfoLookupBudget, IpInfoLookupPriority} from '@pkgs/geoip/src/IpInfoService';
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
import {Logger} from '../Logger';
const BURST_KEY_PREFIX = 'ipinfo:budget:burst:';
const MONTH_KEY_PREFIX = 'ipinfo:budget:month:';
const MONTH_KEY_TTL_SECONDS = 40 * 24 * 3600;
const BURST_REFILL_INTERVAL_MS = 60_000;
const BUDGET_LOG_INTERVAL_MS = 60_000;
let lastBudgetErrorLogMs = 0;
function positiveNumberFromEnv(name: string, fallback: number): number {
const raw = process.env[name];
if (!raw) return fallback;
const parsed = Number(raw);
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
}
function budgetEnabled(): boolean {
return process.env.FLUXER_IPINFO_BUDGET_ENABLED !== '0';
}
function burstConfigFor(priority: IpInfoLookupPriority): {maxTokens: number; refillPerMin: number} {
if (priority === 'critical') {
return {
maxTokens: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_CRITICAL_BURST', 60),
refillPerMin: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_CRITICAL_REFILL_PER_MIN', 60),
};
}
if (priority === 'background') {
return {
maxTokens: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_BACKGROUND_BURST', 120),
refillPerMin: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_BACKGROUND_REFILL_PER_MIN', 30),
};
}
return {
maxTokens: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_STANDARD_BURST', 240),
refillPerMin: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_STANDARD_REFILL_PER_MIN', 120),
};
}
function monthlyCeilingFor(priority: IpInfoLookupPriority): number {
const monthlyMax = positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_MONTHLY_MAX', 140000);
if (priority === 'critical') return monthlyMax;
const percent =
priority === 'background'
? positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_BACKGROUND_MONTHLY_PCT', 60)
: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_STANDARD_MONTHLY_PCT', 90);
return Math.floor((monthlyMax * Math.min(percent, 100)) / 100);
}
function currentMonthKey(): string {
const now = new Date();
const month = String(now.getUTCMonth() + 1).padStart(2, '0');
return `${MONTH_KEY_PREFIX}${now.getUTCFullYear()}-${month}`;
}
function logThrottled(payload: Record<string, unknown>, message: string): void {
const now = Date.now();
if (now - lastBudgetErrorLogMs < BUDGET_LOG_INTERVAL_MS) return;
lastBudgetErrorLogMs = now;
Logger.warn(payload, message);
}
export function createKvIpInfoLookupBudget(opts: {getKvClient: () => IKVProvider}): IpInfoLookupBudget {
return {
async tryConsume(priority: IpInfoLookupPriority): Promise<boolean> {
if (!budgetEnabled()) {
return true;
}
try {
const kv = opts.getKvClient();
const burst = burstConfigFor(priority);
const consumed = await kv.tryConsumeTokens(
`${BURST_KEY_PREFIX}${priority}`,
1,
burst.maxTokens,
burst.refillPerMin,
BURST_REFILL_INTERVAL_MS,
);
if (consumed < 1) {
logThrottled({priority, reason: 'burst'}, 'IPInfo lookup budget shed');
return false;
}
const monthKey = currentMonthKey();
const used = Number((await kv.get(monthKey)) ?? '0');
if (used >= monthlyCeilingFor(priority)) {
logThrottled({priority, reason: 'monthly', used}, 'IPInfo lookup budget shed');
return false;
}
const value = await kv.incr(monthKey);
if (value === 1) {
await kv.expire(monthKey, MONTH_KEY_TTL_SECONDS);
}
return true;
} catch (error) {
logThrottled({error, priority}, 'IPInfo lookup budget check failed, admitting lookup');
return true;
}
},
};
}
@@ -3,7 +3,7 @@
import {getDefaultCassandraClient} from '@pkgs/cassandra/src/Client';
import {createCassandraIpInfoCache} from '@pkgs/geoip/src/CassandraIpInfoCache';
import {createCassandraIpInfoRequestAuditLogger} from '@pkgs/geoip/src/CassandraIpInfoRequestAudit';
import type {IpInfoCache, IpInfoRequestAuditLogger} from '@pkgs/geoip/src/IpInfoService';
import {type IpInfoCache, type IpInfoRequestAuditLogger, isCachedIpInfoFailure} from '@pkgs/geoip/src/IpInfoService';
import {createPostgresIpInfoCache, createPostgresIpInfoRequestAuditLogger} from '@pkgs/geoip/src/PostgresIpInfoKv';
import {createTieredIpInfoCache} from '@pkgs/geoip/src/TieredIpInfoCache';
import {getDefaultPostgresClient} from '@pkgs/postgres/src/Client';
@@ -22,11 +22,13 @@ export function buildIpInfoCache(options: BuildIpInfoCacheOptions): IpInfoCache
getClient: getDefaultPostgresClient,
onError: (error, operation) => Logger.warn({error, operation}, 'Postgres IPInfo cache operation failed'),
}),
skipColdWrite: isCachedIpInfoFailure,
});
}
return createTieredIpInfoCache({
hot: options.hot,
cold: createCassandraIpInfoCache({getClient: getDefaultCassandraClient}),
skipColdWrite: isCachedIpInfoFailure,
});
}
@@ -0,0 +1,60 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
classifyAccountPolicyReverseDnsHostname,
isAccountPolicyEducationOrganizationName,
isAccountPolicyTrustedCommercialPrivacyProvider,
} from './AccountPolicyService';
const ASN_ENTRY_REGEX = /^\d+$/u;
export interface LocalIpIntel {
countryIso: string | null;
asn: number | null;
asnOrg: string | null;
}
export interface IpInfoPrescreenOptions {
enabled: boolean;
allowedAsns: ReadonlySet<number>;
}
export type IpInfoPrescreenVerdict = 'consult' | 'skip';
export function prescreenIpInfoLookup(local: LocalIpIntel, opts: IpInfoPrescreenOptions): IpInfoPrescreenVerdict {
if (!opts.enabled) return 'consult';
if (opts.allowedAsns.size === 0) return 'consult';
if (local.countryIso === null) return 'consult';
if (local.asn === null) return 'consult';
if (!opts.allowedAsns.has(local.asn)) return 'consult';
if (isAccountPolicyTrustedCommercialPrivacyProvider(local.asnOrg)) return 'consult';
if (isAccountPolicyEducationOrganizationName(local.asnOrg)) return 'consult';
if (classifyAccountPolicyReverseDnsHostname(local.asnOrg) === 'cellular') return 'consult';
return 'skip';
}
export function ipInfoPrescreenOptionsFromEnv(): IpInfoPrescreenOptions {
return {
enabled: booleanFromEnv(process.env.FLUXER_RISK_IPINFO_PRESCREEN_ENABLED),
allowedAsns: asnSetFromEnv(process.env.FLUXER_RISK_IPINFO_PRESCREEN_ALLOW_ASNS),
};
}
function booleanFromEnv(rawValue: string | undefined): boolean {
if (!rawValue) return false;
const normalized = rawValue.trim().toLowerCase();
return normalized === '1' || normalized === 'true';
}
function asnSetFromEnv(rawValue: string | undefined): ReadonlySet<number> {
const allowedAsns = new Set<number>();
if (!rawValue) return allowedAsns;
for (const entry of rawValue.split(',')) {
const trimmed = entry.trim();
if (!ASN_ENTRY_REGEX.test(trimmed)) continue;
const asn = Number.parseInt(trimmed, 10);
if (!Number.isSafeInteger(asn)) continue;
allowedAsns.add(asn);
}
return allowedAsns;
}
+13 -21
View File
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {ICacheService} from '@pkgs/cache/src/ICacheService';
import type {GeoipAsnResult, GeoipResult} from '@pkgs/geoip/src/GeoipLookup';
import type {IpInfoService} from '@pkgs/geoip/src/IpInfoService';
import type {IAdminRepository} from '../admin/IAdminRepository';
import {createDisposableDomainChecker} from './adapters/DisposableDomainChecker';
@@ -9,6 +10,7 @@ import {createDomainAgeChecker} from './adapters/DomainAgeChecker';
import {analyzeEmailSyntax} from './adapters/EmailSyntaxAnalyzer';
import {createGeoIpAsnAdapter, createGeoIpCityAdapter} from './adapters/GeoIpAdapters';
import {createHistoricalOutcomeAdapter} from './adapters/HistoricalOutcomeAdapter';
import {unavailableIpInfoAnonymousResult} from './adapters/IpInfoAdapter';
import {checkGeoVsLocale} from './adapters/LocaleGeoMatcher';
import {analyzeRegistrationTiming} from './adapters/RegistrationTimingAnalyzer';
import {analyzeUserAgent} from './adapters/UserAgentAnalyzer';
@@ -29,12 +31,20 @@ interface RiskToolboxFactoryOptions {
mxResolver?: MxResolver;
mxCacheTtlMs?: number;
cacheService?: ICacheService;
lookupLocalCity?: (ip: string) => Promise<GeoipResult>;
lookupLocalAsn?: (ip: string) => Promise<GeoipAsnResult>;
}
export function createRiskToolbox(opts: RiskToolboxFactoryOptions): RiskToolbox {
const checkDomainDisposable = createDisposableDomainChecker({adminRepository: opts.adminRepository});
const lookupGeoIpCity = createGeoIpCityAdapter({ipInfoService: opts.ipInfoService});
const lookupGeoIpAsn = createGeoIpAsnAdapter({ipInfoService: opts.ipInfoService});
const lookupGeoIpCity = createGeoIpCityAdapter({
ipInfoService: opts.ipInfoService,
lookupLocalCity: opts.lookupLocalCity,
});
const lookupGeoIpAsn = createGeoIpAsnAdapter({
ipInfoService: opts.ipInfoService,
lookupLocalAsn: opts.lookupLocalAsn,
});
const checkMx = createDnsMxChecker({
resolver: opts.mxResolver ?? new NodeDnsMxResolver(),
cacheTtlMs: opts.mxCacheTtlMs,
@@ -46,25 +56,7 @@ export function createRiskToolbox(opts: RiskToolboxFactoryOptions): RiskToolbox
});
const lookupIpInfo = opts.ipInfoChecker
? async (args: {ip: string}) => opts.ipInfoChecker!(args.ip)
: async (args: {ip: string}) =>
({
ip: args.ip,
available: false,
isAnonymous: false,
providerName: null,
isVpn: false,
isProxy: false,
isResidentialProxy: false,
isTor: false,
isRelay: false,
isHosting: false,
isMobile: false,
asnType: null,
asnOrg: null,
connectionType: 'unknown',
percentDaysSeen: null,
riskNote: 'IPInfo not configured (no API key)',
}) as IpInfoAnonymousResult;
: async (args: {ip: string}) => unavailableIpInfoAnonymousResult(args.ip, 'IPInfo not configured (no API key)');
const lookupReverseDns = opts.reverseDnsLookup
? async (args: {ip: string}) => opts.reverseDnsLookup!(args.ip)
: async (args: {ip: string}) => ({
@@ -0,0 +1,289 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {GeoipAsnResult, GeoipResult} from '@pkgs/geoip/src/GeoipLookup';
import type {IpInfoLookupResult, IpInfoService} from '@pkgs/geoip/src/IpInfoService';
import {describe, expect, it} from 'vitest';
import {createGeoIpAsnAdapter, createGeoIpCityAdapter} from '../adapters/GeoIpAdapters';
function throwingIpInfoService(): IpInfoService {
return {
lookup: async () => {
throw new Error('ipinfo must not be consulted');
},
};
}
function countingIpInfoService(result: IpInfoLookupResult): {service: IpInfoService; calls: () => number} {
let calls = 0;
return {
service: {
lookup: async () => {
calls += 1;
return result;
},
},
calls: () => calls,
};
}
function ipInfoResult(overrides: Partial<IpInfoLookupResult> = {}): IpInfoLookupResult {
return {
ip: '198.51.100.1',
available: true,
riskNote: 'test',
geo: {
countryCode: 'US',
countryName: 'United States',
continent: 'North America',
continentCode: 'NA',
region: 'California',
regionCode: 'CA',
city: 'San Jose',
postalCode: null,
timezone: 'America/Los_Angeles',
latitude: 37.3,
longitude: -121.9,
accuracyRadiusKm: 20,
},
asn: {
asn: 'AS64500',
number: 64500,
name: 'Test ISP',
domain: null,
type: null,
},
mobile: {
name: null,
mcc: null,
mnc: null,
},
anonymous: {
isAnonymous: false,
providerName: null,
isVpn: false,
isProxy: false,
isResidentialProxy: false,
isTor: false,
isRelay: false,
percentDaysSeen: null,
},
flags: {
isAnycast: false,
isHosting: false,
isMobile: false,
isSatellite: false,
},
...overrides,
};
}
function geoipResult(overrides: Partial<GeoipResult> = {}): GeoipResult {
return {
countryCode: 'SE',
normalizedIp: '198.51.100.7',
city: 'Stockholm',
region: 'Stockholm County',
regionCode: 'AB',
countryName: 'Sweden',
latitude: 59.33,
longitude: 18.06,
accuracyRadiusKm: 5,
timeZone: 'Europe/Stockholm',
...overrides,
};
}
function geoipAsnResult(overrides: Partial<GeoipAsnResult> = {}): GeoipAsnResult {
return {
normalizedIp: '198.51.100.7',
asn: 64510,
asnOrg: 'Example Broadband ISP',
available: true,
...overrides,
};
}
describe('createGeoIpCityAdapter', () => {
it('answers from the local city database without touching IPInfo', async () => {
const lookupGeoIpCity = createGeoIpCityAdapter({
ipInfoService: throwingIpInfoService(),
lookupLocalCity: async () => geoipResult(),
});
await expect(lookupGeoIpCity({ip: '198.51.100.7'})).resolves.toEqual({
ip: '198.51.100.7',
available: true,
found: true,
countryIso: 'SE',
country: 'Sweden',
region: 'Stockholm County',
city: 'Stockholm',
latitude: 59.33,
longitude: 18.06,
accuracyRadiusKm: 5,
timeZone: 'Europe/Stockholm',
});
});
it('normalizes the IP before handing it to the local city database', async () => {
const seen: Array<string> = [];
const lookupGeoIpCity = createGeoIpCityAdapter({
ipInfoService: throwingIpInfoService(),
lookupLocalCity: async (ip) => {
seen.push(ip);
return geoipResult();
},
});
await lookupGeoIpCity({ip: '[2001:0db8:0000::0001]'});
expect(seen).toEqual(['2001:db8::1']);
});
it('coalesces missing optional local fields to null', async () => {
const lookupGeoIpCity = createGeoIpCityAdapter({
ipInfoService: throwingIpInfoService(),
lookupLocalCity: async () => ({
countryCode: 'SE',
normalizedIp: '198.51.100.7',
city: null,
region: null,
countryName: null,
}),
});
await expect(lookupGeoIpCity({ip: '198.51.100.7'})).resolves.toEqual({
ip: '198.51.100.7',
available: true,
found: true,
countryIso: 'SE',
country: null,
region: null,
city: null,
latitude: null,
longitude: null,
accuracyRadiusKm: null,
timeZone: null,
});
});
it('falls back to IPInfo when the local city database has no country', async () => {
const counting = countingIpInfoService(ipInfoResult());
const lookupGeoIpCity = createGeoIpCityAdapter({
ipInfoService: counting.service,
lookupLocalCity: async () => geoipResult({countryCode: null}),
});
await expect(lookupGeoIpCity({ip: '198.51.100.1'})).resolves.toEqual({
ip: '198.51.100.1',
available: true,
found: true,
countryIso: 'US',
country: 'United States',
region: 'California',
city: 'San Jose',
latitude: 37.3,
longitude: -121.9,
accuracyRadiusKm: 20,
timeZone: 'America/Los_Angeles',
});
expect(counting.calls()).toBe(1);
});
it('falls back to IPInfo when no local city lookup is wired', async () => {
const counting = countingIpInfoService(ipInfoResult());
const lookupGeoIpCity = createGeoIpCityAdapter({ipInfoService: counting.service});
const result = await lookupGeoIpCity({ip: '198.51.100.1'});
expect(result.countryIso).toBe('US');
expect(counting.calls()).toBe(1);
});
it('returns an available not-found result for an unparseable IP without any lookup', async () => {
const lookupGeoIpCity = createGeoIpCityAdapter({
ipInfoService: throwingIpInfoService(),
lookupLocalCity: async () => {
throw new Error('local city must not be consulted');
},
});
await expect(lookupGeoIpCity({ip: 'not-an-ip'})).resolves.toEqual({
ip: 'not-an-ip',
available: true,
found: false,
countryIso: null,
country: null,
region: null,
city: null,
latitude: null,
longitude: null,
accuracyRadiusKm: null,
timeZone: null,
});
});
});
describe('createGeoIpAsnAdapter', () => {
it('answers from the local ASN database without touching IPInfo', async () => {
const lookupGeoIpAsn = createGeoIpAsnAdapter({
ipInfoService: throwingIpInfoService(),
lookupLocalAsn: async () => geoipAsnResult(),
});
await expect(lookupGeoIpAsn({ip: '198.51.100.7'})).resolves.toEqual({
ip: '198.51.100.7',
available: true,
found: true,
asn: 64510,
asnOrg: 'Example Broadband ISP',
});
});
it('normalizes the IP before handing it to the local ASN database', async () => {
const seen: Array<string> = [];
const lookupGeoIpAsn = createGeoIpAsnAdapter({
ipInfoService: throwingIpInfoService(),
lookupLocalAsn: async (ip) => {
seen.push(ip);
return geoipAsnResult();
},
});
await lookupGeoIpAsn({ip: '[2001:0db8:0000::0001]'});
expect(seen).toEqual(['2001:db8::1']);
});
it('falls back to IPInfo when the local ASN database has no ASN', async () => {
const counting = countingIpInfoService(ipInfoResult());
const lookupGeoIpAsn = createGeoIpAsnAdapter({
ipInfoService: counting.service,
lookupLocalAsn: async () => geoipAsnResult({asn: null, asnOrg: null}),
});
await expect(lookupGeoIpAsn({ip: '198.51.100.1'})).resolves.toEqual({
ip: '198.51.100.1',
available: true,
found: true,
asn: 64500,
asnOrg: 'Test ISP',
});
expect(counting.calls()).toBe(1);
});
it('falls back to IPInfo when the local ASN database is unavailable', async () => {
const counting = countingIpInfoService(ipInfoResult());
const lookupGeoIpAsn = createGeoIpAsnAdapter({
ipInfoService: counting.service,
lookupLocalAsn: async () => geoipAsnResult({normalizedIp: null, asn: null, asnOrg: null, available: false}),
});
const result = await lookupGeoIpAsn({ip: '198.51.100.1'});
expect(result.asn).toBe(64500);
expect(counting.calls()).toBe(1);
});
it('returns an available not-found result for an unparseable IP without any lookup', async () => {
const lookupGeoIpAsn = createGeoIpAsnAdapter({
ipInfoService: throwingIpInfoService(),
lookupLocalAsn: async () => {
throw new Error('local ASN must not be consulted');
},
});
await expect(lookupGeoIpAsn({ip: 'not-an-ip'})).resolves.toEqual({
ip: 'not-an-ip',
available: true,
found: false,
asn: null,
asnOrg: null,
});
});
});
@@ -0,0 +1,156 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {IpInfoLookupContext, IpInfoLookupResult, IpInfoService} from '@pkgs/geoip/src/IpInfoService';
import {afterEach, beforeEach, describe, expect, it} from 'vitest';
import {createCurrentBehaviorTestAccountPolicyEvaluator} from '../../test/AccountPolicyTestEvaluator';
import {setInjectedAccountPolicyEvaluator} from '../AccountPolicyService';
import {createIpInfoChecker, unavailableIpInfoAnonymousResult} from '../adapters/IpInfoAdapter';
function ipInfoResult(overrides: Partial<IpInfoLookupResult> = {}): IpInfoLookupResult {
return {
ip: '198.51.100.1',
available: true,
riskNote: 'live lookup',
geo: {
countryCode: 'US',
countryName: 'United States',
continent: 'North America',
continentCode: 'NA',
region: null,
regionCode: null,
city: null,
postalCode: null,
timezone: null,
latitude: null,
longitude: null,
accuracyRadiusKm: null,
},
asn: {
asn: 'AS64500',
number: 64500,
name: 'Test ISP',
domain: null,
type: 'isp',
},
mobile: {
name: null,
mcc: null,
mnc: null,
},
anonymous: {
isAnonymous: true,
providerName: 'Example VPN',
isVpn: true,
isProxy: false,
isResidentialProxy: false,
isTor: false,
isRelay: false,
percentDaysSeen: 42,
},
flags: {
isAnycast: false,
isHosting: false,
isMobile: false,
isSatellite: false,
},
...overrides,
};
}
function countingIpInfoService(result: IpInfoLookupResult): {
service: IpInfoService;
calls: () => number;
contexts: () => Array<IpInfoLookupContext | undefined>;
} {
const contexts: Array<IpInfoLookupContext | undefined> = [];
return {
service: {
lookup: async (_ip, context) => {
contexts.push(context);
return result;
},
},
calls: () => contexts.length,
contexts: () => contexts,
};
}
beforeEach(() => {
setInjectedAccountPolicyEvaluator(createCurrentBehaviorTestAccountPolicyEvaluator());
});
afterEach(() => {
setInjectedAccountPolicyEvaluator(undefined);
});
describe('createIpInfoChecker pre-screen', () => {
it('returns an unavailable result with zero lookups when the pre-screen skips', async () => {
const counting = countingIpInfoService(ipInfoResult());
const checkIpInfo = createIpInfoChecker({
ipInfoService: counting.service,
prescreen: async () => 'skip',
});
await expect(checkIpInfo('198.51.100.1')).resolves.toEqual(
unavailableIpInfoAnonymousResult('198.51.100.1', 'IPInfo skipped (local pre-screen)'),
);
expect(counting.calls()).toBe(0);
});
it('calls IPInfo exactly once and preserves the mapping when the pre-screen consults', async () => {
const counting = countingIpInfoService(ipInfoResult());
const checkIpInfo = createIpInfoChecker({
ipInfoService: counting.service,
prescreen: async () => 'consult',
});
await expect(checkIpInfo('198.51.100.1')).resolves.toEqual({
ip: '198.51.100.1',
available: true,
isAnonymous: true,
providerName: 'Example VPN',
isVpn: true,
isProxy: false,
isResidentialProxy: false,
isTor: false,
isRelay: false,
isHosting: false,
isMobile: false,
asnType: 'isp',
asnOrg: 'Test ISP',
connectionType: 'residential',
percentDaysSeen: 42,
riskNote: 'live lookup',
});
expect(counting.calls()).toBe(1);
expect(counting.contexts()).toEqual([{source: 'risk.ipinfo_checker', reason: 'registration_risk'}]);
});
it('is identical to a consulting pre-screen when no pre-screen is wired', async () => {
const withoutPrescreen = countingIpInfoService(ipInfoResult());
const withPrescreen = countingIpInfoService(ipInfoResult());
const baseline = await createIpInfoChecker({ipInfoService: withoutPrescreen.service})('198.51.100.1');
const consulted = await createIpInfoChecker({
ipInfoService: withPrescreen.service,
prescreen: async () => 'consult',
})('198.51.100.1');
expect(baseline).toEqual(consulted);
expect(withoutPrescreen.calls()).toBe(1);
expect(withPrescreen.calls()).toBe(1);
});
it('never synthesizes a clean attestation for a skipped lookup', () => {
const skipped = unavailableIpInfoAnonymousResult('198.51.100.1', 'IPInfo skipped (local pre-screen)');
expect(skipped.available).toBe(false);
expect(skipped.isAnonymous).toBe(false);
expect(skipped.isVpn).toBe(false);
expect(skipped.isProxy).toBe(false);
expect(skipped.isResidentialProxy).toBe(false);
expect(skipped.isTor).toBe(false);
expect(skipped.isRelay).toBe(false);
expect(skipped.isHosting).toBe(false);
expect(skipped.isMobile).toBe(false);
expect(skipped.asnType).toBeNull();
expect(skipped.asnOrg).toBeNull();
expect(skipped.connectionType).toBe('unknown');
expect(skipped.percentDaysSeen).toBeNull();
});
});
@@ -0,0 +1,134 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
import {MockKVProvider} from '../../test/mocks/MockKVProvider';
import {createKvIpInfoLookupBudget} from '../IpInfoBudget';
const BUDGET_ENV_KEYS = [
'FLUXER_IPINFO_BUDGET_ENABLED',
'FLUXER_IPINFO_BUDGET_MONTHLY_MAX',
'FLUXER_IPINFO_BUDGET_BACKGROUND_MONTHLY_PCT',
'FLUXER_IPINFO_BUDGET_STANDARD_MONTHLY_PCT',
'FLUXER_IPINFO_BUDGET_CRITICAL_BURST',
'FLUXER_IPINFO_BUDGET_STANDARD_BURST',
'FLUXER_IPINFO_BUDGET_BACKGROUND_BURST',
'FLUXER_IPINFO_BUDGET_CRITICAL_REFILL_PER_MIN',
'FLUXER_IPINFO_BUDGET_STANDARD_REFILL_PER_MIN',
'FLUXER_IPINFO_BUDGET_BACKGROUND_REFILL_PER_MIN',
];
function monthKey(): string {
const now = new Date();
const month = String(now.getUTCMonth() + 1).padStart(2, '0');
return `ipinfo:budget:month:${now.getUTCFullYear()}-${month}`;
}
describe('IpInfoBudget', () => {
const savedEnv = new Map<string, string | undefined>();
beforeEach(() => {
for (const key of BUDGET_ENV_KEYS) {
savedEnv.set(key, process.env[key]);
delete process.env[key];
}
process.env.FLUXER_IPINFO_BUDGET_MONTHLY_MAX = '100';
});
afterEach(() => {
for (const [key, value] of savedEnv) {
if (value === undefined) {
delete process.env[key];
} else {
process.env[key] = value;
}
}
savedEnv.clear();
});
it('sheds background lookups at the background monthly ceiling while standard still admits', async () => {
const kv = new MockKVProvider();
await kv.set(monthKey(), '60');
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
expect(await budget.tryConsume('background')).toBe(false);
expect(await budget.tryConsume('standard')).toBe(true);
});
it('sheds standard lookups at the standard monthly ceiling', async () => {
const kv = new MockKVProvider();
await kv.set(monthKey(), '90');
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
expect(await budget.tryConsume('standard')).toBe(false);
});
it('admits critical lookups above the standard ceiling', async () => {
const kv = new MockKVProvider();
await kv.set(monthKey(), '95');
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
expect(await budget.tryConsume('critical')).toBe(true);
});
it('never increments the monthly counter for a shed lookup', async () => {
const kv = new MockKVProvider();
await kv.set(monthKey(), '60');
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
const outcomes = [
await budget.tryConsume('background'),
await budget.tryConsume('background'),
await budget.tryConsume('standard'),
await budget.tryConsume('critical'),
];
expect(outcomes).toEqual([false, false, true, true]);
expect(kv.incrSpy).toHaveBeenCalledTimes(2);
expect(await kv.get(monthKey())).toBe('62');
});
it('sheds once the per-priority burst bucket is drained', async () => {
process.env.FLUXER_IPINFO_BUDGET_BACKGROUND_BURST = '3';
process.env.FLUXER_IPINFO_BUDGET_BACKGROUND_REFILL_PER_MIN = '1';
const kv = new MockKVProvider();
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
expect(await budget.tryConsume('background')).toBe(true);
expect(await budget.tryConsume('background')).toBe(true);
expect(await budget.tryConsume('background')).toBe(true);
expect(await budget.tryConsume('background')).toBe(false);
expect(await budget.tryConsume('standard')).toBe(true);
});
it('expires the month key only on the first increment', async () => {
const kv = new MockKVProvider();
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
expect(await budget.tryConsume('standard')).toBe(true);
expect(await budget.tryConsume('standard')).toBe(true);
expect(await budget.tryConsume('standard')).toBe(true);
expect(kv.expireSpy).toHaveBeenCalledTimes(1);
expect(kv.expireSpy).toHaveBeenCalledWith(monthKey(), 40 * 24 * 3600);
});
it('fails open for every priority when the KV provider throws', async () => {
const kv = new MockKVProvider();
vi.spyOn(kv, 'get').mockRejectedValue(new Error('kv unavailable'));
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
expect(await budget.tryConsume('background')).toBe(true);
expect(await budget.tryConsume('standard')).toBe(true);
expect(await budget.tryConsume('critical')).toBe(true);
});
it('admits everything when the budget is disabled', async () => {
process.env.FLUXER_IPINFO_BUDGET_ENABLED = '0';
const kv = new MockKVProvider();
await kv.set(monthKey(), '1000');
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
expect(await budget.tryConsume('background')).toBe(true);
expect(kv.tryConsumeTokensSpy).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,276 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {
CachedIpInfoFailure,
IpInfoCache,
IpInfoLookupBudget,
IpInfoLookupPriority,
IpInfoRequestAuditEvent,
IpInfoRequestAuditLogger,
} from '@pkgs/geoip/src/IpInfoService';
import {createIpInfoService, resolveIpInfoLookupPriority} from '@pkgs/geoip/src/IpInfoService';
import {delay, HttpResponse, http} from 'msw';
import {describe, expect, it} from 'vitest';
import {server} from '../../test/msw/server';
interface RecordedSet {
key: string;
value: unknown;
ttlSeconds: number | undefined;
}
interface RecordingCache {
cache: IpInfoCache;
sets: Array<RecordedSet>;
}
function createRecordingCache(): RecordingCache {
const store = new Map<string, unknown>();
const sets: Array<RecordedSet> = [];
return {
sets,
cache: {
async get<T>(key: string): Promise<T | null> {
return (store.get(key) as T | undefined) ?? null;
},
async set<T>(key: string, value: T, ttlSeconds?: number): Promise<void> {
store.set(key, value);
sets.push({key, value, ttlSeconds});
},
},
};
}
function createRecordingAuditLogger(): {logger: IpInfoRequestAuditLogger; events: Array<IpInfoRequestAuditEvent>} {
const events: Array<IpInfoRequestAuditEvent> = [];
return {
events,
logger: {
async record(event: IpInfoRequestAuditEvent): Promise<void> {
events.push(event);
},
},
};
}
function useLookupHandler(handler: () => Response | Promise<Response>): {count: () => number} {
let calls = 0;
server.use(
http.get('https://api.ipinfo.io/lookup/:ip', async () => {
calls += 1;
return await handler();
}),
);
return {count: () => calls};
}
function successPayload(ip: string, anonymous: Record<string, boolean> = {}): Response {
return HttpResponse.json({
ip,
geo: {country_code: 'US', country: 'United States'},
as: {asn: 'AS64500', name: 'Test ISP'},
anonymous,
});
}
describe('IpInfoService caching', () => {
it('negative-caches an HTTP error and serves the second lookup without a request', async () => {
const requests = useLookupHandler(() => new HttpResponse(null, {status: 500}));
const {cache, sets} = createRecordingCache();
const service = createIpInfoService({apiKey: 'token', cache});
const first = await service.lookup('203.0.113.1');
const second = await service.lookup('203.0.113.1');
expect(first.available).toBe(false);
expect(second.available).toBe(false);
expect(requests.count()).toBe(1);
expect(sets).toHaveLength(1);
expect(sets[0]?.ttlSeconds).toBe(300);
});
it('negative-caches a request failure for a short window', async () => {
useLookupHandler(async () => {
await delay(5000);
return successPayload('203.0.113.2');
});
const {cache, sets} = createRecordingCache();
const service = createIpInfoService({apiKey: 'token', cache});
const result = await service.lookup('203.0.113.2');
expect(result.available).toBe(false);
expect(sets[0]?.ttlSeconds).toBe(60);
expect((sets[0]?.value as CachedIpInfoFailure).failureOutcome).toBe('request_failed');
});
it('negative-caches a schema mismatch', async () => {
useLookupHandler(() => HttpResponse.json({}));
const {cache, sets} = createRecordingCache();
const service = createIpInfoService({apiKey: 'token', cache});
const result = await service.lookup('203.0.113.3');
expect(result.available).toBe(false);
expect(sets[0]?.ttlSeconds).toBe(600);
expect((sets[0]?.value as CachedIpInfoFailure).failureOutcome).toBe('schema_mismatch');
expect((sets[0]?.value as CachedIpInfoFailure).failureHttpStatus).toBe(200);
});
it('negative-caches a quota rejection for longer', async () => {
useLookupHandler(() => new HttpResponse(null, {status: 429}));
const {cache, sets} = createRecordingCache();
const service = createIpInfoService({apiKey: 'token', cache});
await service.lookup('203.0.113.4');
expect(sets[0]?.ttlSeconds).toBe(900);
});
it('caps the quota rejection TTL for background lookups', async () => {
useLookupHandler(() => new HttpResponse(null, {status: 429}));
const {cache, sets} = createRecordingCache();
const service = createIpInfoService({apiKey: 'token', cache});
await service.lookup('203.0.113.5', {source: 'AbusiveIpAutoBanner', reason: 'classify'});
expect(sets[0]?.ttlSeconds).toBe(120);
});
it('returns a cached failure as a clean unavailable result', async () => {
useLookupHandler(() => new HttpResponse(null, {status: 500}));
const {cache} = createRecordingCache();
const service = createIpInfoService({apiKey: 'token', cache});
await service.lookup('203.0.113.6');
const cached = await service.lookup('203.0.113.6');
expect(cached).not.toHaveProperty('cachedFailure');
expect(cached).not.toHaveProperty('failureOutcome');
expect(cached).not.toHaveProperty('failureHttpStatus');
expect(cached).not.toHaveProperty('cachedAtMs');
expect(cached.ip).toBe('203.0.113.6');
expect(cached.riskNote).toBe('IPInfo HTTP 500');
});
it('writes a cached failure that older readers can still consume', async () => {
useLookupHandler(() => new HttpResponse(null, {status: 500}));
const {cache, sets} = createRecordingCache();
const service = createIpInfoService({apiKey: 'token', cache});
await service.lookup('203.0.113.7');
const entry = sets[0]?.value as CachedIpInfoFailure;
expect(entry.cachedFailure).toBe(true);
expect(entry.failureOutcome).toBe('http_error');
expect(entry.failureHttpStatus).toBe(500);
expect(typeof entry.cachedAtMs).toBe('number');
const legacyView = {...entry, ip: '203.0.113.7'};
expect(legacyView.available).toBe(false);
expect(legacyView.geo.countryCode).toBeNull();
expect(legacyView.asn.number).toBeNull();
expect(legacyView.mobile.name).toBeNull();
expect(legacyView.anonymous.isAnonymous).toBe(false);
expect(legacyView.flags.isMobile).toBe(false);
});
it('keeps the existing success TTL selection', async () => {
useLookupHandler(() => successPayload('203.0.113.8'));
const plain = createRecordingCache();
await createIpInfoService({apiKey: 'token', cache: plain.cache}).lookup('203.0.113.8');
useLookupHandler(() => successPayload('203.0.113.9', {is_vpn: true}));
const anonymous = createRecordingCache();
await createIpInfoService({apiKey: 'token', cache: anonymous.cache}).lookup('203.0.113.9');
expect(plain.sets[0]?.ttlSeconds).toBe(14 * 24 * 60 * 60);
expect(anonymous.sets[0]?.ttlSeconds).toBe(7 * 24 * 60 * 60);
});
it('coalesces concurrent lookups across a failure', async () => {
const requests = useLookupHandler(() => new HttpResponse(null, {status: 500}));
const {cache, sets} = createRecordingCache();
const service = createIpInfoService({apiKey: 'token', cache});
const [first, second] = await Promise.all([service.lookup('203.0.113.10'), service.lookup('203.0.113.10')]);
expect(requests.count()).toBe(1);
expect(sets).toHaveLength(1);
expect(first.available).toBe(false);
expect(second.available).toBe(false);
});
it('sheds a lookup when the budget refuses it', async () => {
const requests = useLookupHandler(() => successPayload('203.0.113.11'));
const {cache, sets} = createRecordingCache();
const {logger, events} = createRecordingAuditLogger();
const refused: Array<IpInfoLookupPriority> = [];
const budget: IpInfoLookupBudget = {
async tryConsume(priority: IpInfoLookupPriority): Promise<boolean> {
refused.push(priority);
return priority !== 'background';
},
};
const service = createIpInfoService({apiKey: 'token', cache, auditLogger: logger, budget});
const result = await service.lookup('203.0.113.11', {source: 'AbusiveIpAutoBanner', reason: 'classify'});
expect(refused).toEqual(['background']);
expect(requests.count()).toBe(0);
expect(result.available).toBe(false);
expect(result.riskNote).toContain('shed');
expect(sets).toHaveLength(0);
expect(events).toHaveLength(1);
expect(events[0]?.outcome).toBe('budget_shed');
expect(events[0]?.httpStatus).toBeNull();
});
it('still coalesces concurrent lookups when a budget is configured', async () => {
const requests = useLookupHandler(() => successPayload('203.0.113.13'));
const {cache} = createRecordingCache();
const consumed: Array<IpInfoLookupPriority> = [];
const budget: IpInfoLookupBudget = {
async tryConsume(priority: IpInfoLookupPriority): Promise<boolean> {
consumed.push(priority);
return true;
},
};
const service = createIpInfoService({apiKey: 'token', cache, budget});
const results = await Promise.all([
service.lookup('203.0.113.13', {source: 'risk.geoip_city'}),
service.lookup('203.0.113.13', {source: 'risk.geoip_asn'}),
service.lookup('203.0.113.13', {source: 'risk.ipinfo_checker'}),
]);
expect(requests.count()).toBe(1);
expect(consumed).toEqual(['standard']);
expect(results.every((result) => result.available)).toBe(true);
});
it('lets an admitting budget through', async () => {
const requests = useLookupHandler(() => successPayload('203.0.113.12'));
const {cache} = createRecordingCache();
const budget: IpInfoLookupBudget = {
async tryConsume(): Promise<boolean> {
return true;
},
};
const service = createIpInfoService({apiKey: 'token', cache, budget});
const result = await service.lookup('203.0.113.12', {source: 'risk.ipinfo_checker'});
expect(requests.count()).toBe(1);
expect(result.available).toBe(true);
});
it('maps every lookup source to a priority', () => {
expect(resolveIpInfoLookupPriority('admin.ip_ban')).toBe('critical');
expect(resolveIpInfoLookupPriority('admin.scheduled_deletion_suspicious_ip')).toBe('critical');
expect(resolveIpInfoLookupPriority('AbusiveIpAutoBanner')).toBe('background');
expect(resolveIpInfoLookupPriority('risk.ipinfo_checker')).toBe('standard');
expect(resolveIpInfoLookupPriority('risk.geoip_city')).toBe('standard');
expect(resolveIpInfoLookupPriority('risk.geoip_asn')).toBe('standard');
expect(resolveIpInfoLookupPriority(undefined)).toBe('standard');
});
});
@@ -0,0 +1,110 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {afterEach, beforeEach, describe, expect, it} from 'vitest';
import {createCurrentBehaviorTestAccountPolicyEvaluator} from '../../test/AccountPolicyTestEvaluator';
import {setInjectedAccountPolicyEvaluator} from '../AccountPolicyService';
import {
type IpInfoPrescreenOptions,
ipInfoPrescreenOptionsFromEnv,
type LocalIpIntel,
prescreenIpInfoLookup,
} from '../RegistrationIpPrescreen';
const CLEAN_LOCAL: LocalIpIntel = {countryIso: 'SE', asn: 64500, asnOrg: 'Example Broadband ISP'};
function options(overrides: Partial<IpInfoPrescreenOptions> = {}): IpInfoPrescreenOptions {
return {enabled: true, allowedAsns: new Set([64500]), ...overrides};
}
function local(overrides: Partial<LocalIpIntel> = {}): LocalIpIntel {
return {...CLEAN_LOCAL, ...overrides};
}
beforeEach(() => {
setInjectedAccountPolicyEvaluator(createCurrentBehaviorTestAccountPolicyEvaluator());
});
afterEach(() => {
setInjectedAccountPolicyEvaluator(undefined);
delete process.env.FLUXER_RISK_IPINFO_PRESCREEN_ENABLED;
delete process.env.FLUXER_RISK_IPINFO_PRESCREEN_ALLOW_ASNS;
});
describe('prescreenIpInfoLookup', () => {
it('skips the lookup only when every local signal is clean and allowlisted', () => {
expect(prescreenIpInfoLookup(local(), options())).toBe('skip');
});
it('consults when the pre-screen is disabled', () => {
expect(prescreenIpInfoLookup(local(), options({enabled: false}))).toBe('consult');
});
it('consults when the allowlist is empty', () => {
expect(prescreenIpInfoLookup(local(), options({allowedAsns: new Set()}))).toBe('consult');
});
it('consults when the local city database has no country', () => {
expect(prescreenIpInfoLookup(local({countryIso: null}), options())).toBe('consult');
});
it('consults when the local ASN database has no ASN', () => {
expect(prescreenIpInfoLookup(local({asn: null}), options())).toBe('consult');
});
it('consults when the ASN is not on the allowlist', () => {
expect(prescreenIpInfoLookup(local({asn: 64501}), options())).toBe('consult');
});
it('consults for a trusted commercial privacy provider even on the allowlist', () => {
expect(prescreenIpInfoLookup(local({asnOrg: 'Example Privacy Relay LLC'}), options())).toBe('consult');
});
it('consults for an education organization even on the allowlist', () => {
expect(prescreenIpInfoLookup(local({asnOrg: 'North Example Academy'}), options())).toBe('consult');
});
it('consults for a cellular organization even on the allowlist', () => {
expect(prescreenIpInfoLookup(local({asnOrg: 'Example cell-net Wireless'}), options())).toBe('consult');
});
it('skips when the allowlisted ASN carries no organization name to veto', () => {
expect(prescreenIpInfoLookup(local({asnOrg: null}), options())).toBe('skip');
});
});
describe('ipInfoPrescreenOptionsFromEnv', () => {
it('is disabled with an empty allowlist by default', () => {
const parsed = ipInfoPrescreenOptionsFromEnv();
expect(parsed.enabled).toBe(false);
expect(parsed.allowedAsns.size).toBe(0);
});
it('treats only 1 and true as enabled', () => {
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ENABLED = '1';
expect(ipInfoPrescreenOptionsFromEnv().enabled).toBe(true);
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ENABLED = 'TRUE';
expect(ipInfoPrescreenOptionsFromEnv().enabled).toBe(true);
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ENABLED = 'yes';
expect(ipInfoPrescreenOptionsFromEnv().enabled).toBe(false);
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ENABLED = '0';
expect(ipInfoPrescreenOptionsFromEnv().enabled).toBe(false);
});
it('parses a comma separated allowlist and drops non numeric entries', () => {
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ALLOW_ASNS = ' 64500, 64501 ,,notanasn,64502x,-3,64503 ';
expect([...ipInfoPrescreenOptionsFromEnv().allowedAsns]).toEqual([64500, 64501, 64503]);
});
it('parses an empty allowlist from an empty or whitespace value', () => {
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ALLOW_ASNS = '';
expect(ipInfoPrescreenOptionsFromEnv().allowedAsns.size).toBe(0);
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ALLOW_ASNS = ' , ,';
expect(ipInfoPrescreenOptionsFromEnv().allowedAsns.size).toBe(0);
});
it('yields a consult verdict for every input with the shipped defaults', () => {
const shipped = ipInfoPrescreenOptionsFromEnv();
expect(prescreenIpInfoLookup(local(), shipped)).toBe('consult');
expect(prescreenIpInfoLookup(local({asnOrg: null}), shipped)).toBe('consult');
});
});
@@ -0,0 +1,130 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {IpInfoCache} from '@pkgs/geoip/src/IpInfoService';
import {createTieredIpInfoCache} from '@pkgs/geoip/src/TieredIpInfoCache';
import {describe, expect, it} from 'vitest';
interface RecordedSet {
key: string;
value: unknown;
ttlSeconds: number | undefined;
}
interface RecordingCache {
cache: IpInfoCache;
store: Map<string, unknown>;
sets: Array<RecordedSet>;
}
function createRecordingCache(): RecordingCache {
const store = new Map<string, unknown>();
const sets: Array<RecordedSet> = [];
return {
store,
sets,
cache: {
async get<T>(key: string): Promise<T | null> {
return (store.get(key) as T | undefined) ?? null;
},
async set<T>(key: string, value: T, ttlSeconds?: number): Promise<void> {
store.set(key, value);
sets.push({key, value, ttlSeconds});
},
},
};
}
describe('TieredIpInfoCache', () => {
it('clamps the hot TTL to the requested TTL and passes the raw TTL to the cold tier', async () => {
const hot = createRecordingCache();
const cold = createRecordingCache();
const tiered = createTieredIpInfoCache({hot: hot.cache, cold: cold.cache});
await tiered.set('a', {available: false}, 60);
expect(hot.sets).toEqual([{key: 'a', value: {available: false}, ttlSeconds: 60}]);
expect(cold.sets).toEqual([{key: 'a', value: {available: false}, ttlSeconds: 60}]);
});
it('caps the hot TTL at the configured hot window', async () => {
const hot = createRecordingCache();
const cold = createRecordingCache();
const tiered = createTieredIpInfoCache({hot: hot.cache, cold: cold.cache});
await tiered.set('a', {available: true}, 100000);
expect(hot.sets[0]?.ttlSeconds).toBe(600);
expect(cold.sets[0]?.ttlSeconds).toBe(100000);
});
it('uses the hot window when no TTL is supplied', async () => {
const hot = createRecordingCache();
const cold = createRecordingCache();
const tiered = createTieredIpInfoCache({hot: hot.cache, cold: cold.cache});
await tiered.set('a', {available: true});
expect(hot.sets[0]?.ttlSeconds).toBe(600);
expect(cold.sets[0]?.ttlSeconds).toBeUndefined();
});
it('skips the cold write when skipColdWrite matches', async () => {
const hot = createRecordingCache();
const cold = createRecordingCache();
const tiered = createTieredIpInfoCache({
hot: hot.cache,
cold: cold.cache,
skipColdWrite: (value) => (value as {available?: unknown}).available === false,
});
await tiered.set('a', {available: false}, 60);
await tiered.set('b', {available: true}, 60);
expect(hot.sets.map((entry) => entry.key)).toEqual(['a', 'b']);
expect(cold.sets.map((entry) => entry.key)).toEqual(['b']);
});
it('promotes a cold hit into the hot tier', async () => {
const hot = createRecordingCache();
const cold = createRecordingCache();
cold.store.set('a', {available: true});
const tiered = createTieredIpInfoCache({hot: hot.cache, cold: cold.cache});
const hit = await tiered.get('a');
expect(hit).toEqual({available: true});
expect(hot.sets).toEqual([{key: 'a', value: {available: true}, ttlSeconds: 600}]);
});
it('never promotes a cold hit that skipColdWrite matches', async () => {
const hot = createRecordingCache();
const cold = createRecordingCache();
cold.store.set('a', {available: false});
const tiered = createTieredIpInfoCache({
hot: hot.cache,
cold: cold.cache,
skipColdWrite: (value) => (value as {available?: unknown}).available === false,
});
const hit = await tiered.get('a');
expect(hit).toEqual({available: false});
expect(hot.sets).toEqual([]);
});
it('never writes a zero TTL', async () => {
const hot = createRecordingCache();
const cold = createRecordingCache();
const tiered = createTieredIpInfoCache({hot: hot.cache, cold: cold.cache});
await tiered.set('a', {available: false}, 60);
await tiered.set('b', {available: true}, 100000);
await tiered.set('c', {available: true});
cold.store.set('d', {available: true});
await tiered.get('d');
for (const entry of [...hot.sets, ...cold.sets]) {
expect(entry.ttlSeconds === undefined || entry.ttlSeconds > 0).toBe(true);
}
});
});
@@ -1,15 +1,18 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {parseIpAddress} from '@fluxer/ip_utils/src/IpAddress';
import type {GeoipAsnResult, GeoipResult} from '@pkgs/geoip/src/GeoipLookup';
import type {IpInfoService} from '@pkgs/geoip/src/IpInfoService';
import type {GeoIpAsnResult, GeoIpCityResult} from '../RiskTypes';
interface GeoIpCityContext {
ipInfoService: IpInfoService;
lookupLocalCity?: (ip: string) => Promise<GeoipResult>;
}
interface GeoIpAsnContext {
ipInfoService: IpInfoService;
lookupLocalAsn?: (ip: string) => Promise<GeoipAsnResult>;
}
export function createGeoIpCityAdapter(ctx: GeoIpCityContext) {
@@ -19,6 +22,22 @@ export function createGeoIpCityAdapter(ctx: GeoIpCityContext) {
if (!parsed) {
return notFound(ip, true);
}
const local = ctx.lookupLocalCity ? await ctx.lookupLocalCity(parsed.normalized) : null;
if (local && local.countryCode !== null) {
return {
ip,
available: true,
found: true,
countryIso: local.countryCode,
country: local.countryName,
region: local.region,
city: local.city,
latitude: local.latitude ?? null,
longitude: local.longitude ?? null,
accuracyRadiusKm: local.accuracyRadiusKm ?? null,
timeZone: local.timeZone ?? null,
};
}
const info = await ctx.ipInfoService.lookup(parsed.normalized, {
source: 'risk.geoip_city',
reason: 'registration_risk',
@@ -63,6 +82,10 @@ export function createGeoIpAsnAdapter(ctx: GeoIpAsnContext) {
if (!parsed) {
return {ip, available: true, found: false, asn: null, asnOrg: null};
}
const local = ctx.lookupLocalAsn ? await ctx.lookupLocalAsn(parsed.normalized) : null;
if (local && local.asn !== null) {
return {ip, available: true, found: true, asn: local.asn, asnOrg: local.asnOrg};
}
const info = await ctx.ipInfoService.lookup(parsed.normalized, {
source: 'risk.geoip_asn',
reason: 'registration_risk',
@@ -2,14 +2,40 @@
import type {IpInfoService} from '@pkgs/geoip/src/IpInfoService';
import {isAccountPolicyEducationOrganizationName} from '../AccountPolicyService';
import type {IpInfoPrescreenVerdict} from '../RegistrationIpPrescreen';
import type {IpConnectionType, IpInfoAnonymousResult} from '../RiskTypes';
interface IpInfoCheckerContext {
ipInfoService: IpInfoService;
prescreen?: (ip: string) => Promise<IpInfoPrescreenVerdict>;
}
export function unavailableIpInfoAnonymousResult(ip: string, riskNote: string): IpInfoAnonymousResult {
return {
ip,
available: false,
isAnonymous: false,
providerName: null,
isVpn: false,
isProxy: false,
isResidentialProxy: false,
isTor: false,
isRelay: false,
isHosting: false,
isMobile: false,
asnType: null,
asnOrg: null,
connectionType: 'unknown',
percentDaysSeen: null,
riskNote,
};
}
export function createIpInfoChecker(ctx: IpInfoCheckerContext) {
return async function checkIpInfo(ip: string): Promise<IpInfoAnonymousResult> {
if (ctx.prescreen && (await ctx.prescreen(ip)) === 'skip') {
return unavailableIpInfoAnonymousResult(ip, 'IPInfo skipped (local pre-screen)');
}
const result = await ctx.ipInfoService.lookup(ip, {
source: 'risk.ipinfo_checker',
reason: 'registration_risk',
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {resetGeoipReadersForTesting} from '@pkgs/geoip/src/GeoipLookup';
import {resetSudoModeServiceForTesting} from '../auth/services/SudoModeService';
import {resetSsoRequestUrlPolicyForTesting} from '../instance/SsoConfigValidation';
import {resetGlobalLimitConfigServiceForTesting} from '../limits/LimitConfigService';
@@ -32,6 +33,7 @@ export function resetServiceStateForTesting(): void {
torExitListCache.shutdown();
torExitListCache.clearForTesting();
urlBlocklistCache.resetForTesting();
resetGeoipReadersForTesting();
fileShaCache.resetForTesting();
phraseBlocklistCache.resetForTesting();
bannedAvatarHashCache.resetForTesting();
@@ -0,0 +1,28 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {HttpResponse, http} from 'msw';
export function createIpInfoLookupHandler() {
return http.get('https://api.ipinfo.io/lookup/:ip', ({params}) => {
const ip = typeof params.ip === 'string' ? params.ip : '198.51.100.1';
return HttpResponse.json({
ip,
geo: {
city: 'Ashburn',
region: 'Virginia',
region_code: 'VA',
country: 'United States',
country_code: 'US',
continent: 'North America',
continent_code: 'NA',
},
as: {
asn: 'AS64500',
name: 'Test ISP',
domain: 'example.com',
type: 'isp',
},
anonymous: {},
});
});
}
+2
View File
@@ -2,6 +2,7 @@
import {setupServer} from 'msw/node';
import {createBunnyEdgeHandlers} from './handlers/BunnyEdgeHandlers';
import {createIpInfoLookupHandler} from './handlers/IpInfoHandlers';
import {createNcmecHandlers} from './handlers/NcmecHandlers';
import {createOnionooDetailsHandler} from './handlers/OnionooHandlers';
import {createOpenNsfwHandlers} from './handlers/OpenNsfwHandlers';
@@ -11,6 +12,7 @@ export const server = setupServer(
...createBunnyEdgeHandlers(),
...createNcmecHandlers(),
...createOpenNsfwHandlers(),
createIpInfoLookupHandler(),
createOnionooDetailsHandler(),
createPwnedPasswordsRangeHandler(),
);