mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-18 21:45:31 +09:00
fix(api): dedupe and budget ipinfo lookups across api pods (#2584)
This commit is contained in:
1 parent
a2a68847fd
commit
824b5c86c9
24 files changed
+2125
-66
No files matched your search
@@ -15,6 +15,7 @@ import type {HonoEnv} from '../types/HonoEnv';
|
||||
import {parseJsonRecord} from '../utils/JsonBoundaryUtils';
|
||||
import {ipBanCache} from './IpBanMiddleware';
|
||||
import {getIpInfoService} from './ServiceMiddleware';
|
||||
import {getKVClient} from './ServiceRegistry';
|
||||
import {getCacheService} from './ServiceSingletons';
|
||||
|
||||
type IpClass = 'datacenter' | 'anonymous' | 'mobile' | 'residential' | 'unknown';
|
||||
@@ -51,6 +52,22 @@ interface AbuseSignalOptions {
|
||||
weight?: number;
|
||||
}
|
||||
|
||||
interface PeerIpClassHint {
|
||||
ipClass: IpClass;
|
||||
expiresAtMs: number;
|
||||
}
|
||||
|
||||
interface OutboundIpClass {
|
||||
lookupIp: string;
|
||||
ipClass: IpClass;
|
||||
}
|
||||
|
||||
interface ResolvedBanClass {
|
||||
ipClass: IpClass;
|
||||
authoritative: boolean;
|
||||
blocked: boolean;
|
||||
}
|
||||
|
||||
const WINDOW_MS = positiveNumberFromEnv('FLUXER_ABUSE_WINDOW_MS', 60_000);
|
||||
const THRESHOLD_DATACENTER = positiveNumberFromEnv('FLUXER_ABUSE_THRESHOLD_DATACENTER', 20);
|
||||
const THRESHOLD_ANONYMOUS = positiveNumberFromEnv('FLUXER_ABUSE_THRESHOLD_ANONYMOUS', 500);
|
||||
@@ -73,6 +90,14 @@ const REQUIRED_SCORE_WINDOWS_FOR_AUTO_BAN = positiveNumberFromEnv(
|
||||
3,
|
||||
);
|
||||
const REPLICATION_CHANNEL = 'abuse_tracker:ticks';
|
||||
const IP_CLASS_CHANNEL = 'abuse_tracker:ipclass';
|
||||
const IP_CLASS_CLAIM_PREFIX = 'abuse:ipclass:claim:';
|
||||
const IP_CLASS_CLAIM_ENABLED = process.env.FLUXER_ABUSE_IP_CLASS_CLAIM_ENABLED !== '0';
|
||||
const IP_CLASS_CLAIM_TTL_SECONDS = positiveNumberFromEnv('FLUXER_ABUSE_IP_CLASS_CLAIM_TTL_SEC', 15);
|
||||
const DEFAULT_IP_CLASS_PENDING_TTL_MS = positiveNumberFromEnv('FLUXER_ABUSE_IP_CLASS_PENDING_TTL_MS', 20_000);
|
||||
const DEFAULT_IP_CLASS_NEGATIVE_TTL_MS = positiveNumberFromEnv('FLUXER_ABUSE_IP_CLASS_NEGATIVE_TTL_MS', 300_000);
|
||||
const DEFAULT_IP_CLASS_HINT_TTL_MS = positiveNumberFromEnv('FLUXER_ABUSE_IP_CLASS_HINT_TTL_MS', 600_000);
|
||||
const IP_CLASSES = ['datacenter', 'anonymous', 'mobile', 'residential', 'unknown'] as const;
|
||||
const POD_ID = process.env.HOSTNAME ?? randomUUID();
|
||||
|
||||
type ReplicatedTick = [banKey: string, scoreDelta: number, tokenHashes: Array<string>, lookupIp: string];
|
||||
@@ -83,11 +108,23 @@ interface ReplicationMessage {
|
||||
ts: number;
|
||||
}
|
||||
|
||||
type IpClassEntry = [banKey: string, lookupIp: string, ipClass: IpClass];
|
||||
|
||||
interface IpClassMessage {
|
||||
sender: string;
|
||||
entries: Array<IpClassEntry>;
|
||||
ts: number;
|
||||
}
|
||||
|
||||
const records = new Map<string, AbuseRecord>();
|
||||
const outboundDeltas = new Map<string, OutboundEntry>();
|
||||
const persistentScoreWindows = new Map<string, PersistentScoreState>();
|
||||
const ipClassCache = new Map<string, IpClass>();
|
||||
const ipClassPending = new Set<string>();
|
||||
const ipClassPending = new Map<string, number>();
|
||||
const ipClassNegativeUntil = new Map<string, number>();
|
||||
const peerIpClassHints = new Map<string, PeerIpClassHint>();
|
||||
const outboundIpClasses = new Map<string, OutboundIpClass>();
|
||||
const pendingIpClassTasks = new Set<Promise<void>>();
|
||||
const recordedClientErrorRequests = new WeakSet<Request>();
|
||||
const pendingAutoBanTasks = new Set<Promise<void>>();
|
||||
const adminRepository = new AdminRepository();
|
||||
@@ -97,6 +134,9 @@ let flushTimer: NodeJS.Timeout | null = null;
|
||||
let kvSubscription: IKVSubscription | null = null;
|
||||
let messageHandler: ((channel: string, message: string) => void) | null = null;
|
||||
let errorHandler: ((error: Error) => void) | null = null;
|
||||
let ipClassPendingTtlMs = DEFAULT_IP_CLASS_PENDING_TTL_MS;
|
||||
let ipClassNegativeTtlMs = DEFAULT_IP_CLASS_NEGATIVE_TTL_MS;
|
||||
let ipClassHintTtlMs = DEFAULT_IP_CLASS_HINT_TTL_MS;
|
||||
|
||||
function positiveNumberFromEnv(name: string, fallback: number): number {
|
||||
const raw = process.env[name];
|
||||
@@ -164,13 +204,99 @@ function shouldSkipAutoBanForIpClass(ipClass: IpClass): boolean {
|
||||
return ipClass === 'mobile';
|
||||
}
|
||||
|
||||
function isIpClass(value: unknown): value is IpClass {
|
||||
return typeof value === 'string' && (IP_CLASSES as ReadonlyArray<string>).includes(value);
|
||||
}
|
||||
|
||||
function getOwnIpClass(key: string, now: number): IpClass | null {
|
||||
const cached = ipClassCache.get(key);
|
||||
if (cached === undefined) return null;
|
||||
const negativeUntilMs = ipClassNegativeUntil.get(key);
|
||||
if (negativeUntilMs !== undefined && negativeUntilMs <= now) {
|
||||
ipClassCache.delete(key);
|
||||
ipClassNegativeUntil.delete(key);
|
||||
return null;
|
||||
}
|
||||
return cached;
|
||||
}
|
||||
|
||||
function isOwnIpClassNegative(key: string, now: number): boolean {
|
||||
const negativeUntilMs = ipClassNegativeUntil.get(key);
|
||||
return negativeUntilMs !== undefined && negativeUntilMs > now;
|
||||
}
|
||||
|
||||
function setOwnIpClass(key: string, lookupIp: string, ipClass: IpClass, negative: boolean): void {
|
||||
ipClassCache.set(key, ipClass);
|
||||
if (negative) {
|
||||
ipClassNegativeUntil.set(key, Date.now() + ipClassNegativeTtlMs);
|
||||
} else {
|
||||
ipClassNegativeUntil.delete(key);
|
||||
peerIpClassHints.delete(key);
|
||||
}
|
||||
ipClassPending.delete(key);
|
||||
if (!negative && ipClass !== 'unknown') {
|
||||
queueOutboundIpClass(key, lookupIp, ipClass);
|
||||
}
|
||||
const rec = records.get(key);
|
||||
if (rec) maybeFireAutoBan(key, rec);
|
||||
}
|
||||
|
||||
function isIpClassPending(key: string, now: number): boolean {
|
||||
const expiresAtMs = ipClassPending.get(key);
|
||||
if (expiresAtMs === undefined) return false;
|
||||
if (expiresAtMs <= now) {
|
||||
ipClassPending.delete(key);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function getPeerClassHint(key: string, now: number): IpClass | null {
|
||||
const hint = peerIpClassHints.get(key);
|
||||
if (!hint) return null;
|
||||
if (hint.expiresAtMs <= now) {
|
||||
peerIpClassHints.delete(key);
|
||||
return null;
|
||||
}
|
||||
return hint.ipClass;
|
||||
}
|
||||
|
||||
function recordPeerClassHint(key: string, ipClass: IpClass): void {
|
||||
peerIpClassHints.set(key, {ipClass, expiresAtMs: Date.now() + ipClassHintTtlMs});
|
||||
}
|
||||
|
||||
function isStricterThanUnknown(ipClass: IpClass): boolean {
|
||||
return (
|
||||
scoreThresholdFor(ipClass) <= scoreThresholdFor('unknown') &&
|
||||
tokenDiversityThresholdFor(ipClass) <= tokenDiversityThresholdFor('unknown')
|
||||
);
|
||||
}
|
||||
|
||||
function resolveClassForBan(key: string, now: number): ResolvedBanClass {
|
||||
const own = getOwnIpClass(key, now);
|
||||
if (own !== null && !isOwnIpClassNegative(key, now)) return {ipClass: own, authoritative: true, blocked: false};
|
||||
const hint = getPeerClassHint(key, now);
|
||||
if (hint !== null && isStricterThanUnknown(hint)) return {ipClass: hint, authoritative: true, blocked: false};
|
||||
if (hint !== null) return {ipClass: 'unknown', authoritative: false, blocked: true};
|
||||
if (own !== null) return {ipClass: own, authoritative: true, blocked: false};
|
||||
return {ipClass: 'unknown', authoritative: false, blocked: isIpClassPending(key, now)};
|
||||
}
|
||||
|
||||
function pruneIfNeeded(now: number): void {
|
||||
if (records.size < MAX_TRACKED_IPS) return;
|
||||
for (const [key, expiresAtMs] of ipClassPending) {
|
||||
if (expiresAtMs <= now) ipClassPending.delete(key);
|
||||
}
|
||||
for (const [key, hint] of peerIpClassHints) {
|
||||
if (hint.expiresAtMs <= now) peerIpClassHints.delete(key);
|
||||
}
|
||||
for (const [key, rec] of records) {
|
||||
if (rec.windowStartMs + WINDOW_MS < now) {
|
||||
if (rec.windowStartMs + WINDOW_MS < now && !ipClassPending.has(key)) {
|
||||
records.delete(key);
|
||||
ipClassCache.delete(key);
|
||||
ipClassPending.delete(key);
|
||||
ipClassNegativeUntil.delete(key);
|
||||
peerIpClassHints.delete(key);
|
||||
outboundIpClasses.delete(key);
|
||||
}
|
||||
if (records.size < MAX_TRACKED_IPS * 0.9) return;
|
||||
}
|
||||
@@ -222,9 +348,11 @@ function queueOutboundDelta(
|
||||
}
|
||||
}
|
||||
|
||||
function shouldEnsureIpClassLookup(key: string, rec: AbuseRecord): boolean {
|
||||
if (ipClassCache.has(key) || ipClassPending.has(key)) return false;
|
||||
return rec.score >= MIN_SCORE_FOR_IP_LOOKUP || rec.distinctTokenHashes.size >= MIN_TOKENS_FOR_IP_LOOKUP;
|
||||
function shouldEnsureIpClassLookup(key: string, rec: AbuseRecord, now: number): boolean {
|
||||
if (getOwnIpClass(key, now) !== null || isIpClassPending(key, now)) return false;
|
||||
if (getPeerClassHint(key, now) !== null) return false;
|
||||
if (rec.score < MIN_SCORE_FOR_IP_LOOKUP && rec.distinctTokenHashes.size < MIN_TOKENS_FOR_IP_LOOKUP) return false;
|
||||
return ipBanCache.getMatch(rec.lookupIp) === null;
|
||||
}
|
||||
|
||||
function markScoreThresholdWindow(key: string, rec: AbuseRecord, now: number): number {
|
||||
@@ -247,33 +375,53 @@ function markScoreThresholdWindow(key: string, rec: AbuseRecord, now: number): n
|
||||
return state.count;
|
||||
}
|
||||
|
||||
async function claimIpClassLookup(key: string): Promise<boolean> {
|
||||
if (!IP_CLASS_CLAIM_ENABLED) return true;
|
||||
if (!kvPublisher) return true;
|
||||
try {
|
||||
return await getKVClient().setnx(`${IP_CLASS_CLAIM_PREFIX}${key}`, POD_ID, IP_CLASS_CLAIM_TTL_SECONDS);
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
async function runIpClassLookup(key: string, lookupIp: string): Promise<void> {
|
||||
try {
|
||||
if (!(await claimIpClassLookup(key))) return;
|
||||
const result = await getIpInfoService().lookup(lookupIp, {source: 'AbusiveIpAutoBanner', reason: 'classify'});
|
||||
setOwnIpClass(key, lookupIp, classifyIpInfo(result), !result.available);
|
||||
} catch (err) {
|
||||
setOwnIpClass(key, lookupIp, 'unknown', true);
|
||||
Logger.warn({err, ip: lookupIp}, '[abuse-auto-ban] IP classification lookup failed');
|
||||
}
|
||||
}
|
||||
|
||||
function ensureIpClassLookup(key: string, lookupIp: string): void {
|
||||
if (ipClassCache.has(key) || ipClassPending.has(key)) return;
|
||||
ipClassPending.add(key);
|
||||
void (async () => {
|
||||
try {
|
||||
const result = await getIpInfoService().lookup(lookupIp, {source: 'AbusiveIpAutoBanner', reason: 'classify'});
|
||||
ipClassCache.set(key, classifyIpInfo(result));
|
||||
} catch (err) {
|
||||
ipClassCache.set(key, 'unknown');
|
||||
Logger.warn({err, ip: lookupIp}, '[abuse-auto-ban] IP classification lookup failed');
|
||||
} finally {
|
||||
ipClassPending.delete(key);
|
||||
const rec = records.get(key);
|
||||
if (rec) maybeFireAutoBan(key, rec);
|
||||
}
|
||||
})();
|
||||
const now = Date.now();
|
||||
if (getOwnIpClass(key, now) !== null || isIpClassPending(key, now)) return;
|
||||
ipClassPending.set(key, now + ipClassPendingTtlMs);
|
||||
const task = runIpClassLookup(key, lookupIp);
|
||||
pendingIpClassTasks.add(task);
|
||||
void task.finally(() => {
|
||||
pendingIpClassTasks.delete(task);
|
||||
});
|
||||
}
|
||||
|
||||
function maybeFireAutoBan(key: string, rec: AbuseRecord): void {
|
||||
if (rec.autoBanFired) return;
|
||||
const ipClass = ipClassCache.get(key) ?? 'unknown';
|
||||
const now = Date.now();
|
||||
if (ipBanCache.getMatch(rec.lookupIp) !== null) {
|
||||
rec.autoBanFired = true;
|
||||
return;
|
||||
}
|
||||
const resolved = resolveClassForBan(key, now);
|
||||
const ipClass = resolved.ipClass;
|
||||
const scoreThreshold = scoreThresholdFor(ipClass);
|
||||
const tokenThreshold = tokenDiversityThresholdFor(ipClass);
|
||||
const overScore = rec.score >= scoreThreshold;
|
||||
const overTokenDiversity = rec.distinctTokenHashes.size >= tokenThreshold;
|
||||
if (!overScore && !overTokenDiversity) return;
|
||||
if (!ipClassCache.has(key) && ipClassPending.has(key)) {
|
||||
if (resolved.blocked) {
|
||||
return;
|
||||
}
|
||||
if (shouldSkipAutoBanForIpClass(ipClass)) {
|
||||
@@ -369,7 +517,7 @@ export function recordAbuseSignal(ip: string | null, reason: string, opts: Abuse
|
||||
queuedTokenHash = opts.tokenHash;
|
||||
}
|
||||
queueOutboundDelta(signalIp, weight, queuedTokenHash, hadToken);
|
||||
if (shouldEnsureIpClassLookup(signalIp.banKey, rec)) {
|
||||
if (shouldEnsureIpClassLookup(signalIp.banKey, rec, now)) {
|
||||
ensureIpClassLookup(signalIp.banKey, signalIp.lookupIp);
|
||||
}
|
||||
maybeFireAutoBan(signalIp.banKey, rec);
|
||||
@@ -404,7 +552,7 @@ function applyReplicatedTick(tick: ReplicatedTick): void {
|
||||
if (rec.distinctTokenHashes.size >= MAX_TOKEN_HASHES_PER_IP) break;
|
||||
rec.distinctTokenHashes.add(tokenHash);
|
||||
}
|
||||
if (shouldEnsureIpClassLookup(banKey, rec)) {
|
||||
if (shouldEnsureIpClassLookup(banKey, rec, now)) {
|
||||
ensureIpClassLookup(banKey, lookupIp);
|
||||
}
|
||||
maybeFireAutoBan(banKey, rec);
|
||||
@@ -435,7 +583,56 @@ async function flushOutbound(): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
function queueOutboundIpClass(key: string, lookupIp: string, ipClass: IpClass): void {
|
||||
if (!kvPublisher) return;
|
||||
if (!outboundIpClasses.has(key) && outboundIpClasses.size >= MAX_TRACKED_IPS) return;
|
||||
outboundIpClasses.set(key, {lookupIp, ipClass});
|
||||
}
|
||||
|
||||
async function flushOutboundIpClasses(): Promise<void> {
|
||||
if (!kvPublisher || outboundIpClasses.size === 0) return;
|
||||
const entries: Array<IpClassEntry> = [];
|
||||
const selectedKeys: Array<string> = [];
|
||||
for (const [key, entry] of outboundIpClasses) {
|
||||
entries.push([key, entry.lookupIp, entry.ipClass]);
|
||||
selectedKeys.push(key);
|
||||
if (entries.length >= MAX_BATCH_TICKS) break;
|
||||
}
|
||||
const message: IpClassMessage = {sender: POD_ID, entries, ts: Date.now()};
|
||||
try {
|
||||
await kvPublisher.publish(IP_CLASS_CHANNEL, JSON.stringify(message));
|
||||
for (const key of selectedKeys) {
|
||||
outboundIpClasses.delete(key);
|
||||
}
|
||||
} catch (err) {
|
||||
Logger.warn({err, entryCount: entries.length}, '[abuse-auto-ban] Failed to publish abuse IP class batch');
|
||||
}
|
||||
}
|
||||
|
||||
function handleIpClassMessage(message: string): void {
|
||||
const msg = parseJsonRecord(message);
|
||||
if (!msg || msg.sender === POD_ID || !Array.isArray(msg.entries)) return;
|
||||
for (const rawEntry of msg.entries) {
|
||||
if (!Array.isArray(rawEntry) || rawEntry.length < 3) continue;
|
||||
const [banKey, lookupIp, ipClass] = rawEntry;
|
||||
if (typeof banKey !== 'string' || typeof lookupIp !== 'string' || !isIpClass(ipClass)) continue;
|
||||
if (ipClass === 'unknown') continue;
|
||||
const signalIp = normalizeSignalIp(lookupIp);
|
||||
if (!signalIp || signalIp.banKey !== banKey) continue;
|
||||
const rec = records.get(banKey);
|
||||
if (!rec) continue;
|
||||
const now = Date.now();
|
||||
if (getOwnIpClass(banKey, now) !== null && !isOwnIpClassNegative(banKey, now)) continue;
|
||||
recordPeerClassHint(banKey, ipClass);
|
||||
maybeFireAutoBan(banKey, rec);
|
||||
}
|
||||
}
|
||||
|
||||
function handleReplicationMessage(channel: string, message: string): void {
|
||||
if (channel === IP_CLASS_CHANNEL) {
|
||||
handleIpClassMessage(message);
|
||||
return;
|
||||
}
|
||||
if (channel !== REPLICATION_CHANNEL) return;
|
||||
const msg = parseJsonRecord(message);
|
||||
if (!msg || msg.sender === POD_ID || !Array.isArray(msg.ticks)) return;
|
||||
@@ -469,11 +666,12 @@ export async function startAbuseReplicationSubscriber(kvClient: IKVProvider | nu
|
||||
};
|
||||
try {
|
||||
await subscription.connect();
|
||||
await subscription.subscribe(REPLICATION_CHANNEL);
|
||||
await subscription.subscribe(REPLICATION_CHANNEL, IP_CLASS_CHANNEL);
|
||||
subscription.on('message', messageHandler);
|
||||
subscription.on('error', errorHandler);
|
||||
flushTimer = setInterval(() => {
|
||||
void flushOutbound();
|
||||
void flushOutboundIpClasses();
|
||||
}, BATCH_FLUSH_MS);
|
||||
if (typeof flushTimer === 'object' && flushTimer && 'unref' in flushTimer) {
|
||||
(flushTimer as {unref(): void}).unref();
|
||||
@@ -514,11 +712,28 @@ export async function drainAbuseAutoBanTasksForTests(): Promise<void> {
|
||||
await Promise.all([...pendingAutoBanTasks]);
|
||||
}
|
||||
|
||||
export async function drainAbuseIpClassLookupsForTests(): Promise<void> {
|
||||
await Promise.all([...pendingIpClassTasks]);
|
||||
}
|
||||
|
||||
export function setAbuseIpClassTtlsForTests(opts: {negativeMs?: number; hintMs?: number; pendingMs?: number}): void {
|
||||
if (opts.negativeMs !== undefined) ipClassNegativeTtlMs = opts.negativeMs;
|
||||
if (opts.hintMs !== undefined) ipClassHintTtlMs = opts.hintMs;
|
||||
if (opts.pendingMs !== undefined) ipClassPendingTtlMs = opts.pendingMs;
|
||||
}
|
||||
|
||||
export function resetAbuseTrackingForTests(): void {
|
||||
records.clear();
|
||||
outboundDeltas.clear();
|
||||
persistentScoreWindows.clear();
|
||||
ipClassCache.clear();
|
||||
ipClassPending.clear();
|
||||
ipClassNegativeUntil.clear();
|
||||
peerIpClassHints.clear();
|
||||
outboundIpClasses.clear();
|
||||
pendingIpClassTasks.clear();
|
||||
pendingAutoBanTasks.clear();
|
||||
ipClassPendingTtlMs = DEFAULT_IP_CLASS_PENDING_TTL_MS;
|
||||
ipClassNegativeTtlMs = DEFAULT_IP_CLASS_NEGATIVE_TTL_MS;
|
||||
ipClassHintTtlMs = DEFAULT_IP_CLASS_HINT_TTL_MS;
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import crypto from 'node:crypto';
|
||||
import {lookupAsnByIp, lookupGeoipByIp} from '@pkgs/geoip/src/GeoipLookup';
|
||||
import {createIpInfoService, createUnavailableIpInfoService, type IpInfoService} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {createMiddleware} from 'hono/factory';
|
||||
import type {ApiContext} from '../ApiContext';
|
||||
@@ -58,8 +59,14 @@ import {createIpInfoChecker} from '../risk/adapters/IpInfoAdapter';
|
||||
import {createReverseDnsLookup} from '../risk/adapters/ReverseDnsAdapter';
|
||||
import {DeterministicRiskEngine} from '../risk/DeterministicRiskEngine';
|
||||
import {CassandraHistoricalOutcomeRepository} from '../risk/HistoricalOutcomeRepository';
|
||||
import {createKvIpInfoLookupBudget} from '../risk/IpInfoBudget';
|
||||
import {buildIpInfoCache, buildIpInfoRequestAuditLogger} from '../risk/IpInfoCacheFactory';
|
||||
import {CassandraRegistrationEventsRepository} from '../risk/RegistrationEventsRepository';
|
||||
import {
|
||||
type IpInfoPrescreenVerdict,
|
||||
ipInfoPrescreenOptionsFromEnv,
|
||||
prescreenIpInfoLookup,
|
||||
} from '../risk/RegistrationIpPrescreen';
|
||||
import {CassandraRiskAssessmentRepository} from '../risk/RiskAssessmentRepository';
|
||||
import {createRiskToolbox} from '../risk/RiskToolboxFactory';
|
||||
import {CassandraSuspiciousIpRepository} from '../risk/SuspiciousIpRepository';
|
||||
@@ -268,6 +275,7 @@ export function getIpInfoService(): IpInfoService {
|
||||
apiKey: Config.risk.ipinfoApiKey,
|
||||
cache,
|
||||
auditLogger: buildIpInfoRequestAuditLogger(),
|
||||
budget: createKvIpInfoLookupBudget({getKvClient: getKVClient}),
|
||||
});
|
||||
return _ipInfoService;
|
||||
}
|
||||
@@ -293,7 +301,14 @@ function getRegistrationRiskEvaluator(): IRegistrationRiskEvaluator {
|
||||
return _registrationRiskEvaluator;
|
||||
}
|
||||
const ipInfoService = getIpInfoService();
|
||||
const ipInfoChecker = Config.risk.ipinfoApiKey ? createIpInfoChecker({ipInfoService}) : undefined;
|
||||
const lookupLocalCity = (ip: string) => lookupGeoipByIp(ip, Config.geoip.maxmindDbPath);
|
||||
const lookupLocalAsn = (ip: string) => lookupAsnByIp(ip, Config.geoip.maxmindAsnDbPath);
|
||||
const prescreenOptions = ipInfoPrescreenOptionsFromEnv();
|
||||
const prescreen = async (ip: string): Promise<IpInfoPrescreenVerdict> => {
|
||||
const [city, asn] = await Promise.all([lookupLocalCity(ip), lookupLocalAsn(ip)]);
|
||||
return prescreenIpInfoLookup({countryIso: city.countryCode, asn: asn.asn, asnOrg: asn.asnOrg}, prescreenOptions);
|
||||
};
|
||||
const ipInfoChecker = Config.risk.ipinfoApiKey ? createIpInfoChecker({ipInfoService, prescreen}) : undefined;
|
||||
const cacheService = getCacheService();
|
||||
const reverseDnsLookup = createReverseDnsLookup({cacheService});
|
||||
const toolbox = createRiskToolbox({
|
||||
@@ -305,6 +320,8 @@ function getRegistrationRiskEvaluator(): IRegistrationRiskEvaluator {
|
||||
historicalOutcomeRepository: getHistoricalOutcomeRepository(),
|
||||
suspiciousIpRepository: getSuspiciousIpRepository(),
|
||||
cacheService,
|
||||
lookupLocalCity,
|
||||
lookupLocalAsn,
|
||||
});
|
||||
const engine = new DeterministicRiskEngine(toolbox, {
|
||||
logger: Logger,
|
||||
|
||||
@@ -1,15 +1,21 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {getSameIpDecisionKey} from '@fluxer/ip_utils/src/IpAddress';
|
||||
import type {IpInfoLookupResult} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
|
||||
import {AdminRepository} from '../../admin/AdminRepository';
|
||||
import type {ApiTestHarness} from '../../test/ApiTestHarness';
|
||||
import {createApiTestHarness} from '../../test/ApiTestHarness';
|
||||
import type {MockKVProvider} from '../../test/mocks/MockKVProvider';
|
||||
import {
|
||||
drainAbuseAutoBanTasksForTests,
|
||||
drainAbuseIpClassLookupsForTests,
|
||||
hashAuthToken,
|
||||
recordAbuseSignal,
|
||||
resetAbuseTrackingForTests,
|
||||
setAbuseIpClassTtlsForTests,
|
||||
startAbuseReplicationSubscriber,
|
||||
stopAbuseReplicationSubscriber,
|
||||
} from '../AbusiveIpAutoBanner';
|
||||
import {ipBanCache} from '../IpBanMiddleware';
|
||||
import {setInjectedIpInfoService} from '../ServiceMiddleware';
|
||||
@@ -65,6 +71,11 @@ function ipInfoResult(ip: string, overrides: Partial<IpInfoLookupResult> = {}):
|
||||
};
|
||||
}
|
||||
|
||||
function claimCallCount(harness: ApiTestHarness, banKey: string): number {
|
||||
const kvProvider = harness.kvProvider as MockKVProvider;
|
||||
return kvProvider.setnxSpy.mock.calls.filter(([key]) => key === `abuse:ipclass:claim:${banKey}`).length;
|
||||
}
|
||||
|
||||
async function waitForAssertion(assertion: () => void): Promise<void> {
|
||||
const deadline = Date.now() + 1000;
|
||||
let lastError: unknown;
|
||||
@@ -84,6 +95,7 @@ async function waitForAssertion(assertion: () => void): Promise<void> {
|
||||
describe('AbusiveIpAutoBanner', () => {
|
||||
let harness: ApiTestHarness;
|
||||
let adminRepository: AdminRepository;
|
||||
let lookupCount = 0;
|
||||
beforeAll(async () => {
|
||||
harness = await createApiTestHarness();
|
||||
adminRepository = new AdminRepository();
|
||||
@@ -92,13 +104,18 @@ describe('AbusiveIpAutoBanner', () => {
|
||||
await harness.reset();
|
||||
resetAbuseTrackingForTests();
|
||||
ipBanCache.resetCaches();
|
||||
lookupCount = 0;
|
||||
setInjectedIpInfoService({
|
||||
async lookup(ip: string) {
|
||||
lookupCount += 1;
|
||||
return ipInfoResult(ip);
|
||||
},
|
||||
});
|
||||
await stopAbuseReplicationSubscriber();
|
||||
await startAbuseReplicationSubscriber(harness.kvProvider);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await stopAbuseReplicationSubscriber();
|
||||
setInjectedIpInfoService(undefined);
|
||||
await harness.shutdown();
|
||||
});
|
||||
@@ -117,7 +134,7 @@ describe('AbusiveIpAutoBanner', () => {
|
||||
it('does not auto-ban after a single score-only spike', async () => {
|
||||
const ip = '8.8.4.4';
|
||||
recordAbuseSignal(ip, 'http_429', {weight: 150});
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
expect(ipBanCache.isBanned(ip)).toBe(false);
|
||||
await expect(adminRepository.isIpBanned(ip)).resolves.toBe(false);
|
||||
@@ -135,7 +152,7 @@ describe('AbusiveIpAutoBanner', () => {
|
||||
for (let i = 0; i < 10; i += 1) {
|
||||
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`mobile-invalid-${i}`)});
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
expect(ipBanCache.isBanned(ip)).toBe(false);
|
||||
await expect(adminRepository.isIpBanned(ip)).resolves.toBe(false);
|
||||
@@ -153,7 +170,7 @@ describe('AbusiveIpAutoBanner', () => {
|
||||
for (let i = 0; i < 100; i += 1) {
|
||||
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`mobile-threshold-${i}`)});
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
expect(ipBanCache.isBanned(ip)).toBe(false);
|
||||
await expect(adminRepository.isIpBanned(ip)).resolves.toBe(false);
|
||||
@@ -168,4 +185,68 @@ describe('AbusiveIpAutoBanner', () => {
|
||||
await expect(adminRepository.isIpBanned(ip)).resolves.toBe(false);
|
||||
}
|
||||
});
|
||||
it('claims the class lookup exactly once for a burst on the same IP', async () => {
|
||||
const ip = '8.8.8.8';
|
||||
for (let i = 0; i < 10; i += 1) {
|
||||
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`claim-${i}`)});
|
||||
}
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
expect(claimCallCount(harness, ip)).toBe(1);
|
||||
expect(lookupCount).toBe(1);
|
||||
});
|
||||
it('does not pay for a lookup or ban when another pod owns the class claim', async () => {
|
||||
const ip = '8.8.8.8';
|
||||
await harness.kvProvider.setnx(`abuse:ipclass:claim:${ip}`, 'other-pod', 60);
|
||||
for (let i = 0; i < 10; i += 1) {
|
||||
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`claim-loser-${i}`)});
|
||||
}
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
expect(lookupCount).toBe(0);
|
||||
expect(ipBanCache.isBanned(ip)).toBe(false);
|
||||
});
|
||||
it('does not classify an IPv4 address that is already banned', async () => {
|
||||
const ip = '8.8.8.8';
|
||||
ipBanCache.banTemp(ip, 3600);
|
||||
for (let i = 0; i < 20; i += 1) {
|
||||
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`already-banned-${i}`)});
|
||||
}
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
expect(lookupCount).toBe(0);
|
||||
expect(claimCallCount(harness, ip)).toBe(0);
|
||||
});
|
||||
it('does not classify an IPv6 address inside an already banned /64', async () => {
|
||||
const ip = '2606:4700:4700::1111';
|
||||
const banKey = getSameIpDecisionKey(ip) ?? ip;
|
||||
ipBanCache.banTemp(banKey, 3600);
|
||||
for (let i = 0; i < 20; i += 1) {
|
||||
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`already-banned-v6-${i}`)});
|
||||
}
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
expect(lookupCount).toBe(0);
|
||||
expect(claimCallCount(harness, banKey)).toBe(0);
|
||||
});
|
||||
it('retries a failed classification once the negative TTL elapses', async () => {
|
||||
const ip = '9.9.9.9';
|
||||
setAbuseIpClassTtlsForTests({negativeMs: 50});
|
||||
setInjectedIpInfoService({
|
||||
async lookup(candidateIp: string) {
|
||||
lookupCount += 1;
|
||||
return ipInfoResult(candidateIp, {available: false});
|
||||
},
|
||||
});
|
||||
recordAbuseSignal(ip, 'http_429', {weight: 25});
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
expect(lookupCount).toBe(1);
|
||||
await new Promise((resolve) => setTimeout(resolve, 60));
|
||||
await harness.kvProvider.del(`abuse:ipclass:claim:${ip}`);
|
||||
recordAbuseSignal(ip, 'http_429', {weight: 25});
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
expect(lookupCount).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,203 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {IpInfoLookupResult} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
|
||||
import type {ApiTestHarness} from '../../test/ApiTestHarness';
|
||||
import {createApiTestHarness} from '../../test/ApiTestHarness';
|
||||
import type {MockKVProvider} from '../../test/mocks/MockKVProvider';
|
||||
import {
|
||||
drainAbuseAutoBanTasksForTests,
|
||||
drainAbuseIpClassLookupsForTests,
|
||||
hashAuthToken,
|
||||
recordAbuseSignal,
|
||||
resetAbuseTrackingForTests,
|
||||
setAbuseIpClassTtlsForTests,
|
||||
startAbuseReplicationSubscriber,
|
||||
stopAbuseReplicationSubscriber,
|
||||
} from '../AbusiveIpAutoBanner';
|
||||
import {ipBanCache} from '../IpBanMiddleware';
|
||||
import {setInjectedIpInfoService} from '../ServiceMiddleware';
|
||||
|
||||
function ipInfoResult(ip: string, overrides: Partial<IpInfoLookupResult> = {}): IpInfoLookupResult {
|
||||
return {
|
||||
ip,
|
||||
available: true,
|
||||
riskNote: 'test',
|
||||
geo: {
|
||||
countryCode: 'US',
|
||||
countryName: 'United States',
|
||||
continent: 'North America',
|
||||
continentCode: 'NA',
|
||||
region: null,
|
||||
regionCode: null,
|
||||
city: null,
|
||||
postalCode: null,
|
||||
timezone: null,
|
||||
latitude: null,
|
||||
longitude: null,
|
||||
accuracyRadiusKm: null,
|
||||
},
|
||||
asn: {
|
||||
asn: 'AS64500',
|
||||
number: 64500,
|
||||
name: 'Test ISP',
|
||||
domain: null,
|
||||
type: null,
|
||||
},
|
||||
mobile: {
|
||||
name: null,
|
||||
mcc: null,
|
||||
mnc: null,
|
||||
},
|
||||
anonymous: {
|
||||
isAnonymous: false,
|
||||
providerName: null,
|
||||
isVpn: false,
|
||||
isProxy: false,
|
||||
isResidentialProxy: false,
|
||||
isTor: false,
|
||||
isRelay: false,
|
||||
percentDaysSeen: null,
|
||||
},
|
||||
flags: {
|
||||
isAnycast: false,
|
||||
isHosting: false,
|
||||
isMobile: false,
|
||||
isSatellite: false,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function publishPeerIpClasses(harness: ApiTestHarness, entries: Array<[string, string, unknown]>): void {
|
||||
const kvProvider = harness.kvProvider as MockKVProvider;
|
||||
kvProvider
|
||||
.getSubscription()
|
||||
.simulateMessage('abuse_tracker:ipclass', JSON.stringify({sender: 'other-pod', entries, ts: Date.now()}));
|
||||
}
|
||||
|
||||
function claimCallCount(harness: ApiTestHarness, banKey: string): number {
|
||||
const kvProvider = harness.kvProvider as MockKVProvider;
|
||||
return kvProvider.setnxSpy.mock.calls.filter(([key]) => key === `abuse:ipclass:claim:${banKey}`).length;
|
||||
}
|
||||
|
||||
function recordTokenSignals(ip: string, prefix: string, from: number, to: number): void {
|
||||
for (let i = from; i < to; i += 1) {
|
||||
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`${prefix}-${i}`)});
|
||||
}
|
||||
}
|
||||
|
||||
async function drainAbuseWork(): Promise<void> {
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
}
|
||||
|
||||
describe('AbusiveIpAutoBanner IP class replication', () => {
|
||||
let harness: ApiTestHarness;
|
||||
let lookupCount = 0;
|
||||
beforeAll(async () => {
|
||||
harness = await createApiTestHarness();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await harness.reset();
|
||||
resetAbuseTrackingForTests();
|
||||
ipBanCache.resetCaches();
|
||||
lookupCount = 0;
|
||||
setInjectedIpInfoService({
|
||||
async lookup(ip: string) {
|
||||
lookupCount += 1;
|
||||
return ipInfoResult(ip);
|
||||
},
|
||||
});
|
||||
await stopAbuseReplicationSubscriber();
|
||||
await startAbuseReplicationSubscriber(harness.kvProvider);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await stopAbuseReplicationSubscriber();
|
||||
setInjectedIpInfoService(undefined);
|
||||
await harness.shutdown();
|
||||
});
|
||||
it('adopts a datacenter class from a peer without paying for its own lookup', async () => {
|
||||
const ip = '8.8.8.8';
|
||||
recordTokenSignals(ip, 'peer-datacenter', 0, 3);
|
||||
publishPeerIpClasses(harness, [[ip, ip, 'datacenter']]);
|
||||
recordTokenSignals(ip, 'peer-datacenter', 3, 10);
|
||||
await drainAbuseWork();
|
||||
expect(lookupCount).toBe(0);
|
||||
expect(claimCallCount(harness, ip)).toBe(0);
|
||||
expect(ipBanCache.isBanned(ip)).toBe(true);
|
||||
});
|
||||
it('never adopts a mobile class from a peer and leaves the IP bannable', async () => {
|
||||
const ip = '8.8.4.4';
|
||||
recordTokenSignals(ip, 'peer-mobile', 0, 3);
|
||||
publishPeerIpClasses(harness, [[ip, ip, 'mobile']]);
|
||||
recordTokenSignals(ip, 'peer-mobile', 3, 10);
|
||||
await drainAbuseWork();
|
||||
expect(lookupCount).toBe(0);
|
||||
expect(ipBanCache.isBanned(ip)).toBe(false);
|
||||
publishPeerIpClasses(harness, [[ip, ip, 'datacenter']]);
|
||||
await drainAbuseWork();
|
||||
expect(ipBanCache.isBanned(ip)).toBe(true);
|
||||
});
|
||||
it('ignores an unrecognised class from a peer and classifies the IP itself', async () => {
|
||||
const ip = '4.4.4.4';
|
||||
recordTokenSignals(ip, 'peer-invalid', 0, 3);
|
||||
publishPeerIpClasses(harness, [
|
||||
[ip, ip, 'datacentre'],
|
||||
[ip, ip, 42],
|
||||
]);
|
||||
recordTokenSignals(ip, 'peer-invalid', 3, 10);
|
||||
await drainAbuseWork();
|
||||
expect(lookupCount).toBe(1);
|
||||
expect(ipBanCache.isBanned(ip)).toBe(true);
|
||||
});
|
||||
it('ignores an unknown class from a peer and classifies the IP itself', async () => {
|
||||
const ip = '9.9.9.9';
|
||||
recordTokenSignals(ip, 'peer-unknown', 0, 3);
|
||||
publishPeerIpClasses(harness, [[ip, ip, 'unknown']]);
|
||||
recordTokenSignals(ip, 'peer-unknown', 3, 10);
|
||||
await drainAbuseWork();
|
||||
expect(lookupCount).toBe(1);
|
||||
expect(ipBanCache.isBanned(ip)).toBe(true);
|
||||
});
|
||||
it('ignores a peer class for an IP it is not tracking', async () => {
|
||||
const ip = '208.67.222.222';
|
||||
publishPeerIpClasses(harness, [[ip, ip, 'datacenter']]);
|
||||
recordTokenSignals(ip, 'peer-untracked', 0, 10);
|
||||
await drainAbuseWork();
|
||||
expect(lookupCount).toBe(1);
|
||||
expect(claimCallCount(harness, ip)).toBe(1);
|
||||
});
|
||||
it('accepts a peer mobile class after its own lookup failed and keeps the IP unbanned', async () => {
|
||||
const ip = '199.85.126.10';
|
||||
setInjectedIpInfoService({
|
||||
async lookup(lookupIp: string) {
|
||||
lookupCount += 1;
|
||||
return ipInfoResult(lookupIp, {available: false});
|
||||
},
|
||||
});
|
||||
recordAbuseSignal(ip, 'http_429', {weight: 25});
|
||||
await drainAbuseWork();
|
||||
expect(lookupCount).toBe(1);
|
||||
publishPeerIpClasses(harness, [[ip, ip, 'mobile']]);
|
||||
recordTokenSignals(ip, 'negative-then-mobile', 0, 10);
|
||||
await drainAbuseWork();
|
||||
expect(lookupCount).toBe(1);
|
||||
expect(ipBanCache.isBanned(ip)).toBe(false);
|
||||
});
|
||||
it('resumes classifying once a peer class hint expires', async () => {
|
||||
const ip = '77.88.8.8';
|
||||
setAbuseIpClassTtlsForTests({hintMs: 50});
|
||||
recordAbuseSignal(ip, 'http_429', {weight: 3});
|
||||
publishPeerIpClasses(harness, [[ip, ip, 'datacenter']]);
|
||||
recordAbuseSignal(ip, 'http_429', {weight: 25});
|
||||
await drainAbuseWork();
|
||||
expect(lookupCount).toBe(0);
|
||||
await new Promise((resolve) => setTimeout(resolve, 80));
|
||||
recordAbuseSignal(ip, 'http_429', {weight: 25});
|
||||
await drainAbuseWork();
|
||||
expect(lookupCount).toBe(1);
|
||||
expect(claimCallCount(harness, ip)).toBe(1);
|
||||
expect(ipBanCache.isBanned(ip)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {IpInfoLookupBudget, IpInfoLookupPriority} from '@pkgs/geoip/src/IpInfoService';
|
||||
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
|
||||
import {Logger} from '../Logger';
|
||||
|
||||
const BURST_KEY_PREFIX = 'ipinfo:budget:burst:';
|
||||
const MONTH_KEY_PREFIX = 'ipinfo:budget:month:';
|
||||
const MONTH_KEY_TTL_SECONDS = 40 * 24 * 3600;
|
||||
const BURST_REFILL_INTERVAL_MS = 60_000;
|
||||
const BUDGET_LOG_INTERVAL_MS = 60_000;
|
||||
|
||||
let lastBudgetErrorLogMs = 0;
|
||||
|
||||
function positiveNumberFromEnv(name: string, fallback: number): number {
|
||||
const raw = process.env[name];
|
||||
if (!raw) return fallback;
|
||||
const parsed = Number(raw);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function budgetEnabled(): boolean {
|
||||
return process.env.FLUXER_IPINFO_BUDGET_ENABLED !== '0';
|
||||
}
|
||||
|
||||
function burstConfigFor(priority: IpInfoLookupPriority): {maxTokens: number; refillPerMin: number} {
|
||||
if (priority === 'critical') {
|
||||
return {
|
||||
maxTokens: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_CRITICAL_BURST', 60),
|
||||
refillPerMin: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_CRITICAL_REFILL_PER_MIN', 60),
|
||||
};
|
||||
}
|
||||
if (priority === 'background') {
|
||||
return {
|
||||
maxTokens: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_BACKGROUND_BURST', 120),
|
||||
refillPerMin: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_BACKGROUND_REFILL_PER_MIN', 30),
|
||||
};
|
||||
}
|
||||
return {
|
||||
maxTokens: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_STANDARD_BURST', 240),
|
||||
refillPerMin: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_STANDARD_REFILL_PER_MIN', 120),
|
||||
};
|
||||
}
|
||||
|
||||
function monthlyCeilingFor(priority: IpInfoLookupPriority): number {
|
||||
const monthlyMax = positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_MONTHLY_MAX', 140000);
|
||||
if (priority === 'critical') return monthlyMax;
|
||||
const percent =
|
||||
priority === 'background'
|
||||
? positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_BACKGROUND_MONTHLY_PCT', 60)
|
||||
: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_STANDARD_MONTHLY_PCT', 90);
|
||||
return Math.floor((monthlyMax * Math.min(percent, 100)) / 100);
|
||||
}
|
||||
|
||||
function currentMonthKey(): string {
|
||||
const now = new Date();
|
||||
const month = String(now.getUTCMonth() + 1).padStart(2, '0');
|
||||
return `${MONTH_KEY_PREFIX}${now.getUTCFullYear()}-${month}`;
|
||||
}
|
||||
|
||||
function logThrottled(payload: Record<string, unknown>, message: string): void {
|
||||
const now = Date.now();
|
||||
if (now - lastBudgetErrorLogMs < BUDGET_LOG_INTERVAL_MS) return;
|
||||
lastBudgetErrorLogMs = now;
|
||||
Logger.warn(payload, message);
|
||||
}
|
||||
|
||||
export function createKvIpInfoLookupBudget(opts: {getKvClient: () => IKVProvider}): IpInfoLookupBudget {
|
||||
return {
|
||||
async tryConsume(priority: IpInfoLookupPriority): Promise<boolean> {
|
||||
if (!budgetEnabled()) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
const kv = opts.getKvClient();
|
||||
const burst = burstConfigFor(priority);
|
||||
const consumed = await kv.tryConsumeTokens(
|
||||
`${BURST_KEY_PREFIX}${priority}`,
|
||||
1,
|
||||
burst.maxTokens,
|
||||
burst.refillPerMin,
|
||||
BURST_REFILL_INTERVAL_MS,
|
||||
);
|
||||
if (consumed < 1) {
|
||||
logThrottled({priority, reason: 'burst'}, 'IPInfo lookup budget shed');
|
||||
return false;
|
||||
}
|
||||
const monthKey = currentMonthKey();
|
||||
const used = Number((await kv.get(monthKey)) ?? '0');
|
||||
if (used >= monthlyCeilingFor(priority)) {
|
||||
logThrottled({priority, reason: 'monthly', used}, 'IPInfo lookup budget shed');
|
||||
return false;
|
||||
}
|
||||
const value = await kv.incr(monthKey);
|
||||
if (value === 1) {
|
||||
await kv.expire(monthKey, MONTH_KEY_TTL_SECONDS);
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
logThrottled({error, priority}, 'IPInfo lookup budget check failed, admitting lookup');
|
||||
return true;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
import {getDefaultCassandraClient} from '@pkgs/cassandra/src/Client';
|
||||
import {createCassandraIpInfoCache} from '@pkgs/geoip/src/CassandraIpInfoCache';
|
||||
import {createCassandraIpInfoRequestAuditLogger} from '@pkgs/geoip/src/CassandraIpInfoRequestAudit';
|
||||
import type {IpInfoCache, IpInfoRequestAuditLogger} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {type IpInfoCache, type IpInfoRequestAuditLogger, isCachedIpInfoFailure} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {createPostgresIpInfoCache, createPostgresIpInfoRequestAuditLogger} from '@pkgs/geoip/src/PostgresIpInfoKv';
|
||||
import {createTieredIpInfoCache} from '@pkgs/geoip/src/TieredIpInfoCache';
|
||||
import {getDefaultPostgresClient} from '@pkgs/postgres/src/Client';
|
||||
@@ -22,11 +22,13 @@ export function buildIpInfoCache(options: BuildIpInfoCacheOptions): IpInfoCache
|
||||
getClient: getDefaultPostgresClient,
|
||||
onError: (error, operation) => Logger.warn({error, operation}, 'Postgres IPInfo cache operation failed'),
|
||||
}),
|
||||
skipColdWrite: isCachedIpInfoFailure,
|
||||
});
|
||||
}
|
||||
return createTieredIpInfoCache({
|
||||
hot: options.hot,
|
||||
cold: createCassandraIpInfoCache({getClient: getDefaultCassandraClient}),
|
||||
skipColdWrite: isCachedIpInfoFailure,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {
|
||||
classifyAccountPolicyReverseDnsHostname,
|
||||
isAccountPolicyEducationOrganizationName,
|
||||
isAccountPolicyTrustedCommercialPrivacyProvider,
|
||||
} from './AccountPolicyService';
|
||||
|
||||
const ASN_ENTRY_REGEX = /^\d+$/u;
|
||||
|
||||
export interface LocalIpIntel {
|
||||
countryIso: string | null;
|
||||
asn: number | null;
|
||||
asnOrg: string | null;
|
||||
}
|
||||
|
||||
export interface IpInfoPrescreenOptions {
|
||||
enabled: boolean;
|
||||
allowedAsns: ReadonlySet<number>;
|
||||
}
|
||||
|
||||
export type IpInfoPrescreenVerdict = 'consult' | 'skip';
|
||||
|
||||
export function prescreenIpInfoLookup(local: LocalIpIntel, opts: IpInfoPrescreenOptions): IpInfoPrescreenVerdict {
|
||||
if (!opts.enabled) return 'consult';
|
||||
if (opts.allowedAsns.size === 0) return 'consult';
|
||||
if (local.countryIso === null) return 'consult';
|
||||
if (local.asn === null) return 'consult';
|
||||
if (!opts.allowedAsns.has(local.asn)) return 'consult';
|
||||
if (isAccountPolicyTrustedCommercialPrivacyProvider(local.asnOrg)) return 'consult';
|
||||
if (isAccountPolicyEducationOrganizationName(local.asnOrg)) return 'consult';
|
||||
if (classifyAccountPolicyReverseDnsHostname(local.asnOrg) === 'cellular') return 'consult';
|
||||
return 'skip';
|
||||
}
|
||||
|
||||
export function ipInfoPrescreenOptionsFromEnv(): IpInfoPrescreenOptions {
|
||||
return {
|
||||
enabled: booleanFromEnv(process.env.FLUXER_RISK_IPINFO_PRESCREEN_ENABLED),
|
||||
allowedAsns: asnSetFromEnv(process.env.FLUXER_RISK_IPINFO_PRESCREEN_ALLOW_ASNS),
|
||||
};
|
||||
}
|
||||
|
||||
function booleanFromEnv(rawValue: string | undefined): boolean {
|
||||
if (!rawValue) return false;
|
||||
const normalized = rawValue.trim().toLowerCase();
|
||||
return normalized === '1' || normalized === 'true';
|
||||
}
|
||||
|
||||
function asnSetFromEnv(rawValue: string | undefined): ReadonlySet<number> {
|
||||
const allowedAsns = new Set<number>();
|
||||
if (!rawValue) return allowedAsns;
|
||||
for (const entry of rawValue.split(',')) {
|
||||
const trimmed = entry.trim();
|
||||
if (!ASN_ENTRY_REGEX.test(trimmed)) continue;
|
||||
const asn = Number.parseInt(trimmed, 10);
|
||||
if (!Number.isSafeInteger(asn)) continue;
|
||||
allowedAsns.add(asn);
|
||||
}
|
||||
return allowedAsns;
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {ICacheService} from '@pkgs/cache/src/ICacheService';
|
||||
import type {GeoipAsnResult, GeoipResult} from '@pkgs/geoip/src/GeoipLookup';
|
||||
import type {IpInfoService} from '@pkgs/geoip/src/IpInfoService';
|
||||
import type {IAdminRepository} from '../admin/IAdminRepository';
|
||||
import {createDisposableDomainChecker} from './adapters/DisposableDomainChecker';
|
||||
@@ -9,6 +10,7 @@ import {createDomainAgeChecker} from './adapters/DomainAgeChecker';
|
||||
import {analyzeEmailSyntax} from './adapters/EmailSyntaxAnalyzer';
|
||||
import {createGeoIpAsnAdapter, createGeoIpCityAdapter} from './adapters/GeoIpAdapters';
|
||||
import {createHistoricalOutcomeAdapter} from './adapters/HistoricalOutcomeAdapter';
|
||||
import {unavailableIpInfoAnonymousResult} from './adapters/IpInfoAdapter';
|
||||
import {checkGeoVsLocale} from './adapters/LocaleGeoMatcher';
|
||||
import {analyzeRegistrationTiming} from './adapters/RegistrationTimingAnalyzer';
|
||||
import {analyzeUserAgent} from './adapters/UserAgentAnalyzer';
|
||||
@@ -29,12 +31,20 @@ interface RiskToolboxFactoryOptions {
|
||||
mxResolver?: MxResolver;
|
||||
mxCacheTtlMs?: number;
|
||||
cacheService?: ICacheService;
|
||||
lookupLocalCity?: (ip: string) => Promise<GeoipResult>;
|
||||
lookupLocalAsn?: (ip: string) => Promise<GeoipAsnResult>;
|
||||
}
|
||||
|
||||
export function createRiskToolbox(opts: RiskToolboxFactoryOptions): RiskToolbox {
|
||||
const checkDomainDisposable = createDisposableDomainChecker({adminRepository: opts.adminRepository});
|
||||
const lookupGeoIpCity = createGeoIpCityAdapter({ipInfoService: opts.ipInfoService});
|
||||
const lookupGeoIpAsn = createGeoIpAsnAdapter({ipInfoService: opts.ipInfoService});
|
||||
const lookupGeoIpCity = createGeoIpCityAdapter({
|
||||
ipInfoService: opts.ipInfoService,
|
||||
lookupLocalCity: opts.lookupLocalCity,
|
||||
});
|
||||
const lookupGeoIpAsn = createGeoIpAsnAdapter({
|
||||
ipInfoService: opts.ipInfoService,
|
||||
lookupLocalAsn: opts.lookupLocalAsn,
|
||||
});
|
||||
const checkMx = createDnsMxChecker({
|
||||
resolver: opts.mxResolver ?? new NodeDnsMxResolver(),
|
||||
cacheTtlMs: opts.mxCacheTtlMs,
|
||||
@@ -46,25 +56,7 @@ export function createRiskToolbox(opts: RiskToolboxFactoryOptions): RiskToolbox
|
||||
});
|
||||
const lookupIpInfo = opts.ipInfoChecker
|
||||
? async (args: {ip: string}) => opts.ipInfoChecker!(args.ip)
|
||||
: async (args: {ip: string}) =>
|
||||
({
|
||||
ip: args.ip,
|
||||
available: false,
|
||||
isAnonymous: false,
|
||||
providerName: null,
|
||||
isVpn: false,
|
||||
isProxy: false,
|
||||
isResidentialProxy: false,
|
||||
isTor: false,
|
||||
isRelay: false,
|
||||
isHosting: false,
|
||||
isMobile: false,
|
||||
asnType: null,
|
||||
asnOrg: null,
|
||||
connectionType: 'unknown',
|
||||
percentDaysSeen: null,
|
||||
riskNote: 'IPInfo not configured (no API key)',
|
||||
}) as IpInfoAnonymousResult;
|
||||
: async (args: {ip: string}) => unavailableIpInfoAnonymousResult(args.ip, 'IPInfo not configured (no API key)');
|
||||
const lookupReverseDns = opts.reverseDnsLookup
|
||||
? async (args: {ip: string}) => opts.reverseDnsLookup!(args.ip)
|
||||
: async (args: {ip: string}) => ({
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {GeoipAsnResult, GeoipResult} from '@pkgs/geoip/src/GeoipLookup';
|
||||
import type {IpInfoLookupResult, IpInfoService} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import {createGeoIpAsnAdapter, createGeoIpCityAdapter} from '../adapters/GeoIpAdapters';
|
||||
|
||||
function throwingIpInfoService(): IpInfoService {
|
||||
return {
|
||||
lookup: async () => {
|
||||
throw new Error('ipinfo must not be consulted');
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function countingIpInfoService(result: IpInfoLookupResult): {service: IpInfoService; calls: () => number} {
|
||||
let calls = 0;
|
||||
return {
|
||||
service: {
|
||||
lookup: async () => {
|
||||
calls += 1;
|
||||
return result;
|
||||
},
|
||||
},
|
||||
calls: () => calls,
|
||||
};
|
||||
}
|
||||
|
||||
function ipInfoResult(overrides: Partial<IpInfoLookupResult> = {}): IpInfoLookupResult {
|
||||
return {
|
||||
ip: '198.51.100.1',
|
||||
available: true,
|
||||
riskNote: 'test',
|
||||
geo: {
|
||||
countryCode: 'US',
|
||||
countryName: 'United States',
|
||||
continent: 'North America',
|
||||
continentCode: 'NA',
|
||||
region: 'California',
|
||||
regionCode: 'CA',
|
||||
city: 'San Jose',
|
||||
postalCode: null,
|
||||
timezone: 'America/Los_Angeles',
|
||||
latitude: 37.3,
|
||||
longitude: -121.9,
|
||||
accuracyRadiusKm: 20,
|
||||
},
|
||||
asn: {
|
||||
asn: 'AS64500',
|
||||
number: 64500,
|
||||
name: 'Test ISP',
|
||||
domain: null,
|
||||
type: null,
|
||||
},
|
||||
mobile: {
|
||||
name: null,
|
||||
mcc: null,
|
||||
mnc: null,
|
||||
},
|
||||
anonymous: {
|
||||
isAnonymous: false,
|
||||
providerName: null,
|
||||
isVpn: false,
|
||||
isProxy: false,
|
||||
isResidentialProxy: false,
|
||||
isTor: false,
|
||||
isRelay: false,
|
||||
percentDaysSeen: null,
|
||||
},
|
||||
flags: {
|
||||
isAnycast: false,
|
||||
isHosting: false,
|
||||
isMobile: false,
|
||||
isSatellite: false,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function geoipResult(overrides: Partial<GeoipResult> = {}): GeoipResult {
|
||||
return {
|
||||
countryCode: 'SE',
|
||||
normalizedIp: '198.51.100.7',
|
||||
city: 'Stockholm',
|
||||
region: 'Stockholm County',
|
||||
regionCode: 'AB',
|
||||
countryName: 'Sweden',
|
||||
latitude: 59.33,
|
||||
longitude: 18.06,
|
||||
accuracyRadiusKm: 5,
|
||||
timeZone: 'Europe/Stockholm',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function geoipAsnResult(overrides: Partial<GeoipAsnResult> = {}): GeoipAsnResult {
|
||||
return {
|
||||
normalizedIp: '198.51.100.7',
|
||||
asn: 64510,
|
||||
asnOrg: 'Example Broadband ISP',
|
||||
available: true,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('createGeoIpCityAdapter', () => {
|
||||
it('answers from the local city database without touching IPInfo', async () => {
|
||||
const lookupGeoIpCity = createGeoIpCityAdapter({
|
||||
ipInfoService: throwingIpInfoService(),
|
||||
lookupLocalCity: async () => geoipResult(),
|
||||
});
|
||||
await expect(lookupGeoIpCity({ip: '198.51.100.7'})).resolves.toEqual({
|
||||
ip: '198.51.100.7',
|
||||
available: true,
|
||||
found: true,
|
||||
countryIso: 'SE',
|
||||
country: 'Sweden',
|
||||
region: 'Stockholm County',
|
||||
city: 'Stockholm',
|
||||
latitude: 59.33,
|
||||
longitude: 18.06,
|
||||
accuracyRadiusKm: 5,
|
||||
timeZone: 'Europe/Stockholm',
|
||||
});
|
||||
});
|
||||
|
||||
it('normalizes the IP before handing it to the local city database', async () => {
|
||||
const seen: Array<string> = [];
|
||||
const lookupGeoIpCity = createGeoIpCityAdapter({
|
||||
ipInfoService: throwingIpInfoService(),
|
||||
lookupLocalCity: async (ip) => {
|
||||
seen.push(ip);
|
||||
return geoipResult();
|
||||
},
|
||||
});
|
||||
await lookupGeoIpCity({ip: '[2001:0db8:0000::0001]'});
|
||||
expect(seen).toEqual(['2001:db8::1']);
|
||||
});
|
||||
|
||||
it('coalesces missing optional local fields to null', async () => {
|
||||
const lookupGeoIpCity = createGeoIpCityAdapter({
|
||||
ipInfoService: throwingIpInfoService(),
|
||||
lookupLocalCity: async () => ({
|
||||
countryCode: 'SE',
|
||||
normalizedIp: '198.51.100.7',
|
||||
city: null,
|
||||
region: null,
|
||||
countryName: null,
|
||||
}),
|
||||
});
|
||||
await expect(lookupGeoIpCity({ip: '198.51.100.7'})).resolves.toEqual({
|
||||
ip: '198.51.100.7',
|
||||
available: true,
|
||||
found: true,
|
||||
countryIso: 'SE',
|
||||
country: null,
|
||||
region: null,
|
||||
city: null,
|
||||
latitude: null,
|
||||
longitude: null,
|
||||
accuracyRadiusKm: null,
|
||||
timeZone: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('falls back to IPInfo when the local city database has no country', async () => {
|
||||
const counting = countingIpInfoService(ipInfoResult());
|
||||
const lookupGeoIpCity = createGeoIpCityAdapter({
|
||||
ipInfoService: counting.service,
|
||||
lookupLocalCity: async () => geoipResult({countryCode: null}),
|
||||
});
|
||||
await expect(lookupGeoIpCity({ip: '198.51.100.1'})).resolves.toEqual({
|
||||
ip: '198.51.100.1',
|
||||
available: true,
|
||||
found: true,
|
||||
countryIso: 'US',
|
||||
country: 'United States',
|
||||
region: 'California',
|
||||
city: 'San Jose',
|
||||
latitude: 37.3,
|
||||
longitude: -121.9,
|
||||
accuracyRadiusKm: 20,
|
||||
timeZone: 'America/Los_Angeles',
|
||||
});
|
||||
expect(counting.calls()).toBe(1);
|
||||
});
|
||||
|
||||
it('falls back to IPInfo when no local city lookup is wired', async () => {
|
||||
const counting = countingIpInfoService(ipInfoResult());
|
||||
const lookupGeoIpCity = createGeoIpCityAdapter({ipInfoService: counting.service});
|
||||
const result = await lookupGeoIpCity({ip: '198.51.100.1'});
|
||||
expect(result.countryIso).toBe('US');
|
||||
expect(counting.calls()).toBe(1);
|
||||
});
|
||||
|
||||
it('returns an available not-found result for an unparseable IP without any lookup', async () => {
|
||||
const lookupGeoIpCity = createGeoIpCityAdapter({
|
||||
ipInfoService: throwingIpInfoService(),
|
||||
lookupLocalCity: async () => {
|
||||
throw new Error('local city must not be consulted');
|
||||
},
|
||||
});
|
||||
await expect(lookupGeoIpCity({ip: 'not-an-ip'})).resolves.toEqual({
|
||||
ip: 'not-an-ip',
|
||||
available: true,
|
||||
found: false,
|
||||
countryIso: null,
|
||||
country: null,
|
||||
region: null,
|
||||
city: null,
|
||||
latitude: null,
|
||||
longitude: null,
|
||||
accuracyRadiusKm: null,
|
||||
timeZone: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('createGeoIpAsnAdapter', () => {
|
||||
it('answers from the local ASN database without touching IPInfo', async () => {
|
||||
const lookupGeoIpAsn = createGeoIpAsnAdapter({
|
||||
ipInfoService: throwingIpInfoService(),
|
||||
lookupLocalAsn: async () => geoipAsnResult(),
|
||||
});
|
||||
await expect(lookupGeoIpAsn({ip: '198.51.100.7'})).resolves.toEqual({
|
||||
ip: '198.51.100.7',
|
||||
available: true,
|
||||
found: true,
|
||||
asn: 64510,
|
||||
asnOrg: 'Example Broadband ISP',
|
||||
});
|
||||
});
|
||||
|
||||
it('normalizes the IP before handing it to the local ASN database', async () => {
|
||||
const seen: Array<string> = [];
|
||||
const lookupGeoIpAsn = createGeoIpAsnAdapter({
|
||||
ipInfoService: throwingIpInfoService(),
|
||||
lookupLocalAsn: async (ip) => {
|
||||
seen.push(ip);
|
||||
return geoipAsnResult();
|
||||
},
|
||||
});
|
||||
await lookupGeoIpAsn({ip: '[2001:0db8:0000::0001]'});
|
||||
expect(seen).toEqual(['2001:db8::1']);
|
||||
});
|
||||
|
||||
it('falls back to IPInfo when the local ASN database has no ASN', async () => {
|
||||
const counting = countingIpInfoService(ipInfoResult());
|
||||
const lookupGeoIpAsn = createGeoIpAsnAdapter({
|
||||
ipInfoService: counting.service,
|
||||
lookupLocalAsn: async () => geoipAsnResult({asn: null, asnOrg: null}),
|
||||
});
|
||||
await expect(lookupGeoIpAsn({ip: '198.51.100.1'})).resolves.toEqual({
|
||||
ip: '198.51.100.1',
|
||||
available: true,
|
||||
found: true,
|
||||
asn: 64500,
|
||||
asnOrg: 'Test ISP',
|
||||
});
|
||||
expect(counting.calls()).toBe(1);
|
||||
});
|
||||
|
||||
it('falls back to IPInfo when the local ASN database is unavailable', async () => {
|
||||
const counting = countingIpInfoService(ipInfoResult());
|
||||
const lookupGeoIpAsn = createGeoIpAsnAdapter({
|
||||
ipInfoService: counting.service,
|
||||
lookupLocalAsn: async () => geoipAsnResult({normalizedIp: null, asn: null, asnOrg: null, available: false}),
|
||||
});
|
||||
const result = await lookupGeoIpAsn({ip: '198.51.100.1'});
|
||||
expect(result.asn).toBe(64500);
|
||||
expect(counting.calls()).toBe(1);
|
||||
});
|
||||
|
||||
it('returns an available not-found result for an unparseable IP without any lookup', async () => {
|
||||
const lookupGeoIpAsn = createGeoIpAsnAdapter({
|
||||
ipInfoService: throwingIpInfoService(),
|
||||
lookupLocalAsn: async () => {
|
||||
throw new Error('local ASN must not be consulted');
|
||||
},
|
||||
});
|
||||
await expect(lookupGeoIpAsn({ip: 'not-an-ip'})).resolves.toEqual({
|
||||
ip: 'not-an-ip',
|
||||
available: true,
|
||||
found: false,
|
||||
asn: null,
|
||||
asnOrg: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,156 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {IpInfoLookupContext, IpInfoLookupResult, IpInfoService} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {afterEach, beforeEach, describe, expect, it} from 'vitest';
|
||||
import {createCurrentBehaviorTestAccountPolicyEvaluator} from '../../test/AccountPolicyTestEvaluator';
|
||||
import {setInjectedAccountPolicyEvaluator} from '../AccountPolicyService';
|
||||
import {createIpInfoChecker, unavailableIpInfoAnonymousResult} from '../adapters/IpInfoAdapter';
|
||||
|
||||
function ipInfoResult(overrides: Partial<IpInfoLookupResult> = {}): IpInfoLookupResult {
|
||||
return {
|
||||
ip: '198.51.100.1',
|
||||
available: true,
|
||||
riskNote: 'live lookup',
|
||||
geo: {
|
||||
countryCode: 'US',
|
||||
countryName: 'United States',
|
||||
continent: 'North America',
|
||||
continentCode: 'NA',
|
||||
region: null,
|
||||
regionCode: null,
|
||||
city: null,
|
||||
postalCode: null,
|
||||
timezone: null,
|
||||
latitude: null,
|
||||
longitude: null,
|
||||
accuracyRadiusKm: null,
|
||||
},
|
||||
asn: {
|
||||
asn: 'AS64500',
|
||||
number: 64500,
|
||||
name: 'Test ISP',
|
||||
domain: null,
|
||||
type: 'isp',
|
||||
},
|
||||
mobile: {
|
||||
name: null,
|
||||
mcc: null,
|
||||
mnc: null,
|
||||
},
|
||||
anonymous: {
|
||||
isAnonymous: true,
|
||||
providerName: 'Example VPN',
|
||||
isVpn: true,
|
||||
isProxy: false,
|
||||
isResidentialProxy: false,
|
||||
isTor: false,
|
||||
isRelay: false,
|
||||
percentDaysSeen: 42,
|
||||
},
|
||||
flags: {
|
||||
isAnycast: false,
|
||||
isHosting: false,
|
||||
isMobile: false,
|
||||
isSatellite: false,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function countingIpInfoService(result: IpInfoLookupResult): {
|
||||
service: IpInfoService;
|
||||
calls: () => number;
|
||||
contexts: () => Array<IpInfoLookupContext | undefined>;
|
||||
} {
|
||||
const contexts: Array<IpInfoLookupContext | undefined> = [];
|
||||
return {
|
||||
service: {
|
||||
lookup: async (_ip, context) => {
|
||||
contexts.push(context);
|
||||
return result;
|
||||
},
|
||||
},
|
||||
calls: () => contexts.length,
|
||||
contexts: () => contexts,
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
setInjectedAccountPolicyEvaluator(createCurrentBehaviorTestAccountPolicyEvaluator());
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
setInjectedAccountPolicyEvaluator(undefined);
|
||||
});
|
||||
|
||||
describe('createIpInfoChecker pre-screen', () => {
|
||||
it('returns an unavailable result with zero lookups when the pre-screen skips', async () => {
|
||||
const counting = countingIpInfoService(ipInfoResult());
|
||||
const checkIpInfo = createIpInfoChecker({
|
||||
ipInfoService: counting.service,
|
||||
prescreen: async () => 'skip',
|
||||
});
|
||||
await expect(checkIpInfo('198.51.100.1')).resolves.toEqual(
|
||||
unavailableIpInfoAnonymousResult('198.51.100.1', 'IPInfo skipped (local pre-screen)'),
|
||||
);
|
||||
expect(counting.calls()).toBe(0);
|
||||
});
|
||||
|
||||
it('calls IPInfo exactly once and preserves the mapping when the pre-screen consults', async () => {
|
||||
const counting = countingIpInfoService(ipInfoResult());
|
||||
const checkIpInfo = createIpInfoChecker({
|
||||
ipInfoService: counting.service,
|
||||
prescreen: async () => 'consult',
|
||||
});
|
||||
await expect(checkIpInfo('198.51.100.1')).resolves.toEqual({
|
||||
ip: '198.51.100.1',
|
||||
available: true,
|
||||
isAnonymous: true,
|
||||
providerName: 'Example VPN',
|
||||
isVpn: true,
|
||||
isProxy: false,
|
||||
isResidentialProxy: false,
|
||||
isTor: false,
|
||||
isRelay: false,
|
||||
isHosting: false,
|
||||
isMobile: false,
|
||||
asnType: 'isp',
|
||||
asnOrg: 'Test ISP',
|
||||
connectionType: 'residential',
|
||||
percentDaysSeen: 42,
|
||||
riskNote: 'live lookup',
|
||||
});
|
||||
expect(counting.calls()).toBe(1);
|
||||
expect(counting.contexts()).toEqual([{source: 'risk.ipinfo_checker', reason: 'registration_risk'}]);
|
||||
});
|
||||
|
||||
it('is identical to a consulting pre-screen when no pre-screen is wired', async () => {
|
||||
const withoutPrescreen = countingIpInfoService(ipInfoResult());
|
||||
const withPrescreen = countingIpInfoService(ipInfoResult());
|
||||
const baseline = await createIpInfoChecker({ipInfoService: withoutPrescreen.service})('198.51.100.1');
|
||||
const consulted = await createIpInfoChecker({
|
||||
ipInfoService: withPrescreen.service,
|
||||
prescreen: async () => 'consult',
|
||||
})('198.51.100.1');
|
||||
expect(baseline).toEqual(consulted);
|
||||
expect(withoutPrescreen.calls()).toBe(1);
|
||||
expect(withPrescreen.calls()).toBe(1);
|
||||
});
|
||||
|
||||
it('never synthesizes a clean attestation for a skipped lookup', () => {
|
||||
const skipped = unavailableIpInfoAnonymousResult('198.51.100.1', 'IPInfo skipped (local pre-screen)');
|
||||
expect(skipped.available).toBe(false);
|
||||
expect(skipped.isAnonymous).toBe(false);
|
||||
expect(skipped.isVpn).toBe(false);
|
||||
expect(skipped.isProxy).toBe(false);
|
||||
expect(skipped.isResidentialProxy).toBe(false);
|
||||
expect(skipped.isTor).toBe(false);
|
||||
expect(skipped.isRelay).toBe(false);
|
||||
expect(skipped.isHosting).toBe(false);
|
||||
expect(skipped.isMobile).toBe(false);
|
||||
expect(skipped.asnType).toBeNull();
|
||||
expect(skipped.asnOrg).toBeNull();
|
||||
expect(skipped.connectionType).toBe('unknown');
|
||||
expect(skipped.percentDaysSeen).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,134 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
|
||||
import {MockKVProvider} from '../../test/mocks/MockKVProvider';
|
||||
import {createKvIpInfoLookupBudget} from '../IpInfoBudget';
|
||||
|
||||
const BUDGET_ENV_KEYS = [
|
||||
'FLUXER_IPINFO_BUDGET_ENABLED',
|
||||
'FLUXER_IPINFO_BUDGET_MONTHLY_MAX',
|
||||
'FLUXER_IPINFO_BUDGET_BACKGROUND_MONTHLY_PCT',
|
||||
'FLUXER_IPINFO_BUDGET_STANDARD_MONTHLY_PCT',
|
||||
'FLUXER_IPINFO_BUDGET_CRITICAL_BURST',
|
||||
'FLUXER_IPINFO_BUDGET_STANDARD_BURST',
|
||||
'FLUXER_IPINFO_BUDGET_BACKGROUND_BURST',
|
||||
'FLUXER_IPINFO_BUDGET_CRITICAL_REFILL_PER_MIN',
|
||||
'FLUXER_IPINFO_BUDGET_STANDARD_REFILL_PER_MIN',
|
||||
'FLUXER_IPINFO_BUDGET_BACKGROUND_REFILL_PER_MIN',
|
||||
];
|
||||
|
||||
function monthKey(): string {
|
||||
const now = new Date();
|
||||
const month = String(now.getUTCMonth() + 1).padStart(2, '0');
|
||||
return `ipinfo:budget:month:${now.getUTCFullYear()}-${month}`;
|
||||
}
|
||||
|
||||
describe('IpInfoBudget', () => {
|
||||
const savedEnv = new Map<string, string | undefined>();
|
||||
|
||||
beforeEach(() => {
|
||||
for (const key of BUDGET_ENV_KEYS) {
|
||||
savedEnv.set(key, process.env[key]);
|
||||
delete process.env[key];
|
||||
}
|
||||
process.env.FLUXER_IPINFO_BUDGET_MONTHLY_MAX = '100';
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const [key, value] of savedEnv) {
|
||||
if (value === undefined) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = value;
|
||||
}
|
||||
}
|
||||
savedEnv.clear();
|
||||
});
|
||||
|
||||
it('sheds background lookups at the background monthly ceiling while standard still admits', async () => {
|
||||
const kv = new MockKVProvider();
|
||||
await kv.set(monthKey(), '60');
|
||||
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
|
||||
|
||||
expect(await budget.tryConsume('background')).toBe(false);
|
||||
expect(await budget.tryConsume('standard')).toBe(true);
|
||||
});
|
||||
|
||||
it('sheds standard lookups at the standard monthly ceiling', async () => {
|
||||
const kv = new MockKVProvider();
|
||||
await kv.set(monthKey(), '90');
|
||||
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
|
||||
|
||||
expect(await budget.tryConsume('standard')).toBe(false);
|
||||
});
|
||||
|
||||
it('admits critical lookups above the standard ceiling', async () => {
|
||||
const kv = new MockKVProvider();
|
||||
await kv.set(monthKey(), '95');
|
||||
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
|
||||
|
||||
expect(await budget.tryConsume('critical')).toBe(true);
|
||||
});
|
||||
|
||||
it('never increments the monthly counter for a shed lookup', async () => {
|
||||
const kv = new MockKVProvider();
|
||||
await kv.set(monthKey(), '60');
|
||||
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
|
||||
|
||||
const outcomes = [
|
||||
await budget.tryConsume('background'),
|
||||
await budget.tryConsume('background'),
|
||||
await budget.tryConsume('standard'),
|
||||
await budget.tryConsume('critical'),
|
||||
];
|
||||
|
||||
expect(outcomes).toEqual([false, false, true, true]);
|
||||
expect(kv.incrSpy).toHaveBeenCalledTimes(2);
|
||||
expect(await kv.get(monthKey())).toBe('62');
|
||||
});
|
||||
|
||||
it('sheds once the per-priority burst bucket is drained', async () => {
|
||||
process.env.FLUXER_IPINFO_BUDGET_BACKGROUND_BURST = '3';
|
||||
process.env.FLUXER_IPINFO_BUDGET_BACKGROUND_REFILL_PER_MIN = '1';
|
||||
const kv = new MockKVProvider();
|
||||
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
|
||||
|
||||
expect(await budget.tryConsume('background')).toBe(true);
|
||||
expect(await budget.tryConsume('background')).toBe(true);
|
||||
expect(await budget.tryConsume('background')).toBe(true);
|
||||
expect(await budget.tryConsume('background')).toBe(false);
|
||||
expect(await budget.tryConsume('standard')).toBe(true);
|
||||
});
|
||||
|
||||
it('expires the month key only on the first increment', async () => {
|
||||
const kv = new MockKVProvider();
|
||||
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
|
||||
|
||||
expect(await budget.tryConsume('standard')).toBe(true);
|
||||
expect(await budget.tryConsume('standard')).toBe(true);
|
||||
expect(await budget.tryConsume('standard')).toBe(true);
|
||||
|
||||
expect(kv.expireSpy).toHaveBeenCalledTimes(1);
|
||||
expect(kv.expireSpy).toHaveBeenCalledWith(monthKey(), 40 * 24 * 3600);
|
||||
});
|
||||
|
||||
it('fails open for every priority when the KV provider throws', async () => {
|
||||
const kv = new MockKVProvider();
|
||||
vi.spyOn(kv, 'get').mockRejectedValue(new Error('kv unavailable'));
|
||||
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
|
||||
|
||||
expect(await budget.tryConsume('background')).toBe(true);
|
||||
expect(await budget.tryConsume('standard')).toBe(true);
|
||||
expect(await budget.tryConsume('critical')).toBe(true);
|
||||
});
|
||||
|
||||
it('admits everything when the budget is disabled', async () => {
|
||||
process.env.FLUXER_IPINFO_BUDGET_ENABLED = '0';
|
||||
const kv = new MockKVProvider();
|
||||
await kv.set(monthKey(), '1000');
|
||||
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
|
||||
|
||||
expect(await budget.tryConsume('background')).toBe(true);
|
||||
expect(kv.tryConsumeTokensSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,276 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {
|
||||
CachedIpInfoFailure,
|
||||
IpInfoCache,
|
||||
IpInfoLookupBudget,
|
||||
IpInfoLookupPriority,
|
||||
IpInfoRequestAuditEvent,
|
||||
IpInfoRequestAuditLogger,
|
||||
} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {createIpInfoService, resolveIpInfoLookupPriority} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {delay, HttpResponse, http} from 'msw';
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import {server} from '../../test/msw/server';
|
||||
|
||||
interface RecordedSet {
|
||||
key: string;
|
||||
value: unknown;
|
||||
ttlSeconds: number | undefined;
|
||||
}
|
||||
|
||||
interface RecordingCache {
|
||||
cache: IpInfoCache;
|
||||
sets: Array<RecordedSet>;
|
||||
}
|
||||
|
||||
function createRecordingCache(): RecordingCache {
|
||||
const store = new Map<string, unknown>();
|
||||
const sets: Array<RecordedSet> = [];
|
||||
return {
|
||||
sets,
|
||||
cache: {
|
||||
async get<T>(key: string): Promise<T | null> {
|
||||
return (store.get(key) as T | undefined) ?? null;
|
||||
},
|
||||
async set<T>(key: string, value: T, ttlSeconds?: number): Promise<void> {
|
||||
store.set(key, value);
|
||||
sets.push({key, value, ttlSeconds});
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createRecordingAuditLogger(): {logger: IpInfoRequestAuditLogger; events: Array<IpInfoRequestAuditEvent>} {
|
||||
const events: Array<IpInfoRequestAuditEvent> = [];
|
||||
return {
|
||||
events,
|
||||
logger: {
|
||||
async record(event: IpInfoRequestAuditEvent): Promise<void> {
|
||||
events.push(event);
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function useLookupHandler(handler: () => Response | Promise<Response>): {count: () => number} {
|
||||
let calls = 0;
|
||||
server.use(
|
||||
http.get('https://api.ipinfo.io/lookup/:ip', async () => {
|
||||
calls += 1;
|
||||
return await handler();
|
||||
}),
|
||||
);
|
||||
return {count: () => calls};
|
||||
}
|
||||
|
||||
function successPayload(ip: string, anonymous: Record<string, boolean> = {}): Response {
|
||||
return HttpResponse.json({
|
||||
ip,
|
||||
geo: {country_code: 'US', country: 'United States'},
|
||||
as: {asn: 'AS64500', name: 'Test ISP'},
|
||||
anonymous,
|
||||
});
|
||||
}
|
||||
|
||||
describe('IpInfoService caching', () => {
|
||||
it('negative-caches an HTTP error and serves the second lookup without a request', async () => {
|
||||
const requests = useLookupHandler(() => new HttpResponse(null, {status: 500}));
|
||||
const {cache, sets} = createRecordingCache();
|
||||
const service = createIpInfoService({apiKey: 'token', cache});
|
||||
|
||||
const first = await service.lookup('203.0.113.1');
|
||||
const second = await service.lookup('203.0.113.1');
|
||||
|
||||
expect(first.available).toBe(false);
|
||||
expect(second.available).toBe(false);
|
||||
expect(requests.count()).toBe(1);
|
||||
expect(sets).toHaveLength(1);
|
||||
expect(sets[0]?.ttlSeconds).toBe(300);
|
||||
});
|
||||
|
||||
it('negative-caches a request failure for a short window', async () => {
|
||||
useLookupHandler(async () => {
|
||||
await delay(5000);
|
||||
return successPayload('203.0.113.2');
|
||||
});
|
||||
const {cache, sets} = createRecordingCache();
|
||||
const service = createIpInfoService({apiKey: 'token', cache});
|
||||
|
||||
const result = await service.lookup('203.0.113.2');
|
||||
|
||||
expect(result.available).toBe(false);
|
||||
expect(sets[0]?.ttlSeconds).toBe(60);
|
||||
expect((sets[0]?.value as CachedIpInfoFailure).failureOutcome).toBe('request_failed');
|
||||
});
|
||||
|
||||
it('negative-caches a schema mismatch', async () => {
|
||||
useLookupHandler(() => HttpResponse.json({}));
|
||||
const {cache, sets} = createRecordingCache();
|
||||
const service = createIpInfoService({apiKey: 'token', cache});
|
||||
|
||||
const result = await service.lookup('203.0.113.3');
|
||||
|
||||
expect(result.available).toBe(false);
|
||||
expect(sets[0]?.ttlSeconds).toBe(600);
|
||||
expect((sets[0]?.value as CachedIpInfoFailure).failureOutcome).toBe('schema_mismatch');
|
||||
expect((sets[0]?.value as CachedIpInfoFailure).failureHttpStatus).toBe(200);
|
||||
});
|
||||
|
||||
it('negative-caches a quota rejection for longer', async () => {
|
||||
useLookupHandler(() => new HttpResponse(null, {status: 429}));
|
||||
const {cache, sets} = createRecordingCache();
|
||||
const service = createIpInfoService({apiKey: 'token', cache});
|
||||
|
||||
await service.lookup('203.0.113.4');
|
||||
|
||||
expect(sets[0]?.ttlSeconds).toBe(900);
|
||||
});
|
||||
|
||||
it('caps the quota rejection TTL for background lookups', async () => {
|
||||
useLookupHandler(() => new HttpResponse(null, {status: 429}));
|
||||
const {cache, sets} = createRecordingCache();
|
||||
const service = createIpInfoService({apiKey: 'token', cache});
|
||||
|
||||
await service.lookup('203.0.113.5', {source: 'AbusiveIpAutoBanner', reason: 'classify'});
|
||||
|
||||
expect(sets[0]?.ttlSeconds).toBe(120);
|
||||
});
|
||||
|
||||
it('returns a cached failure as a clean unavailable result', async () => {
|
||||
useLookupHandler(() => new HttpResponse(null, {status: 500}));
|
||||
const {cache} = createRecordingCache();
|
||||
const service = createIpInfoService({apiKey: 'token', cache});
|
||||
|
||||
await service.lookup('203.0.113.6');
|
||||
const cached = await service.lookup('203.0.113.6');
|
||||
|
||||
expect(cached).not.toHaveProperty('cachedFailure');
|
||||
expect(cached).not.toHaveProperty('failureOutcome');
|
||||
expect(cached).not.toHaveProperty('failureHttpStatus');
|
||||
expect(cached).not.toHaveProperty('cachedAtMs');
|
||||
expect(cached.ip).toBe('203.0.113.6');
|
||||
expect(cached.riskNote).toBe('IPInfo HTTP 500');
|
||||
});
|
||||
|
||||
it('writes a cached failure that older readers can still consume', async () => {
|
||||
useLookupHandler(() => new HttpResponse(null, {status: 500}));
|
||||
const {cache, sets} = createRecordingCache();
|
||||
const service = createIpInfoService({apiKey: 'token', cache});
|
||||
|
||||
await service.lookup('203.0.113.7');
|
||||
|
||||
const entry = sets[0]?.value as CachedIpInfoFailure;
|
||||
expect(entry.cachedFailure).toBe(true);
|
||||
expect(entry.failureOutcome).toBe('http_error');
|
||||
expect(entry.failureHttpStatus).toBe(500);
|
||||
expect(typeof entry.cachedAtMs).toBe('number');
|
||||
const legacyView = {...entry, ip: '203.0.113.7'};
|
||||
expect(legacyView.available).toBe(false);
|
||||
expect(legacyView.geo.countryCode).toBeNull();
|
||||
expect(legacyView.asn.number).toBeNull();
|
||||
expect(legacyView.mobile.name).toBeNull();
|
||||
expect(legacyView.anonymous.isAnonymous).toBe(false);
|
||||
expect(legacyView.flags.isMobile).toBe(false);
|
||||
});
|
||||
|
||||
it('keeps the existing success TTL selection', async () => {
|
||||
useLookupHandler(() => successPayload('203.0.113.8'));
|
||||
const plain = createRecordingCache();
|
||||
await createIpInfoService({apiKey: 'token', cache: plain.cache}).lookup('203.0.113.8');
|
||||
|
||||
useLookupHandler(() => successPayload('203.0.113.9', {is_vpn: true}));
|
||||
const anonymous = createRecordingCache();
|
||||
await createIpInfoService({apiKey: 'token', cache: anonymous.cache}).lookup('203.0.113.9');
|
||||
|
||||
expect(plain.sets[0]?.ttlSeconds).toBe(14 * 24 * 60 * 60);
|
||||
expect(anonymous.sets[0]?.ttlSeconds).toBe(7 * 24 * 60 * 60);
|
||||
});
|
||||
|
||||
it('coalesces concurrent lookups across a failure', async () => {
|
||||
const requests = useLookupHandler(() => new HttpResponse(null, {status: 500}));
|
||||
const {cache, sets} = createRecordingCache();
|
||||
const service = createIpInfoService({apiKey: 'token', cache});
|
||||
|
||||
const [first, second] = await Promise.all([service.lookup('203.0.113.10'), service.lookup('203.0.113.10')]);
|
||||
|
||||
expect(requests.count()).toBe(1);
|
||||
expect(sets).toHaveLength(1);
|
||||
expect(first.available).toBe(false);
|
||||
expect(second.available).toBe(false);
|
||||
});
|
||||
|
||||
it('sheds a lookup when the budget refuses it', async () => {
|
||||
const requests = useLookupHandler(() => successPayload('203.0.113.11'));
|
||||
const {cache, sets} = createRecordingCache();
|
||||
const {logger, events} = createRecordingAuditLogger();
|
||||
const refused: Array<IpInfoLookupPriority> = [];
|
||||
const budget: IpInfoLookupBudget = {
|
||||
async tryConsume(priority: IpInfoLookupPriority): Promise<boolean> {
|
||||
refused.push(priority);
|
||||
return priority !== 'background';
|
||||
},
|
||||
};
|
||||
const service = createIpInfoService({apiKey: 'token', cache, auditLogger: logger, budget});
|
||||
|
||||
const result = await service.lookup('203.0.113.11', {source: 'AbusiveIpAutoBanner', reason: 'classify'});
|
||||
|
||||
expect(refused).toEqual(['background']);
|
||||
expect(requests.count()).toBe(0);
|
||||
expect(result.available).toBe(false);
|
||||
expect(result.riskNote).toContain('shed');
|
||||
expect(sets).toHaveLength(0);
|
||||
expect(events).toHaveLength(1);
|
||||
expect(events[0]?.outcome).toBe('budget_shed');
|
||||
expect(events[0]?.httpStatus).toBeNull();
|
||||
});
|
||||
|
||||
it('still coalesces concurrent lookups when a budget is configured', async () => {
|
||||
const requests = useLookupHandler(() => successPayload('203.0.113.13'));
|
||||
const {cache} = createRecordingCache();
|
||||
const consumed: Array<IpInfoLookupPriority> = [];
|
||||
const budget: IpInfoLookupBudget = {
|
||||
async tryConsume(priority: IpInfoLookupPriority): Promise<boolean> {
|
||||
consumed.push(priority);
|
||||
return true;
|
||||
},
|
||||
};
|
||||
const service = createIpInfoService({apiKey: 'token', cache, budget});
|
||||
|
||||
const results = await Promise.all([
|
||||
service.lookup('203.0.113.13', {source: 'risk.geoip_city'}),
|
||||
service.lookup('203.0.113.13', {source: 'risk.geoip_asn'}),
|
||||
service.lookup('203.0.113.13', {source: 'risk.ipinfo_checker'}),
|
||||
]);
|
||||
|
||||
expect(requests.count()).toBe(1);
|
||||
expect(consumed).toEqual(['standard']);
|
||||
expect(results.every((result) => result.available)).toBe(true);
|
||||
});
|
||||
|
||||
it('lets an admitting budget through', async () => {
|
||||
const requests = useLookupHandler(() => successPayload('203.0.113.12'));
|
||||
const {cache} = createRecordingCache();
|
||||
const budget: IpInfoLookupBudget = {
|
||||
async tryConsume(): Promise<boolean> {
|
||||
return true;
|
||||
},
|
||||
};
|
||||
const service = createIpInfoService({apiKey: 'token', cache, budget});
|
||||
|
||||
const result = await service.lookup('203.0.113.12', {source: 'risk.ipinfo_checker'});
|
||||
|
||||
expect(requests.count()).toBe(1);
|
||||
expect(result.available).toBe(true);
|
||||
});
|
||||
|
||||
it('maps every lookup source to a priority', () => {
|
||||
expect(resolveIpInfoLookupPriority('admin.ip_ban')).toBe('critical');
|
||||
expect(resolveIpInfoLookupPriority('admin.scheduled_deletion_suspicious_ip')).toBe('critical');
|
||||
expect(resolveIpInfoLookupPriority('AbusiveIpAutoBanner')).toBe('background');
|
||||
expect(resolveIpInfoLookupPriority('risk.ipinfo_checker')).toBe('standard');
|
||||
expect(resolveIpInfoLookupPriority('risk.geoip_city')).toBe('standard');
|
||||
expect(resolveIpInfoLookupPriority('risk.geoip_asn')).toBe('standard');
|
||||
expect(resolveIpInfoLookupPriority(undefined)).toBe('standard');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,110 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {afterEach, beforeEach, describe, expect, it} from 'vitest';
|
||||
import {createCurrentBehaviorTestAccountPolicyEvaluator} from '../../test/AccountPolicyTestEvaluator';
|
||||
import {setInjectedAccountPolicyEvaluator} from '../AccountPolicyService';
|
||||
import {
|
||||
type IpInfoPrescreenOptions,
|
||||
ipInfoPrescreenOptionsFromEnv,
|
||||
type LocalIpIntel,
|
||||
prescreenIpInfoLookup,
|
||||
} from '../RegistrationIpPrescreen';
|
||||
|
||||
const CLEAN_LOCAL: LocalIpIntel = {countryIso: 'SE', asn: 64500, asnOrg: 'Example Broadband ISP'};
|
||||
|
||||
function options(overrides: Partial<IpInfoPrescreenOptions> = {}): IpInfoPrescreenOptions {
|
||||
return {enabled: true, allowedAsns: new Set([64500]), ...overrides};
|
||||
}
|
||||
|
||||
function local(overrides: Partial<LocalIpIntel> = {}): LocalIpIntel {
|
||||
return {...CLEAN_LOCAL, ...overrides};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
setInjectedAccountPolicyEvaluator(createCurrentBehaviorTestAccountPolicyEvaluator());
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
setInjectedAccountPolicyEvaluator(undefined);
|
||||
delete process.env.FLUXER_RISK_IPINFO_PRESCREEN_ENABLED;
|
||||
delete process.env.FLUXER_RISK_IPINFO_PRESCREEN_ALLOW_ASNS;
|
||||
});
|
||||
|
||||
describe('prescreenIpInfoLookup', () => {
|
||||
it('skips the lookup only when every local signal is clean and allowlisted', () => {
|
||||
expect(prescreenIpInfoLookup(local(), options())).toBe('skip');
|
||||
});
|
||||
|
||||
it('consults when the pre-screen is disabled', () => {
|
||||
expect(prescreenIpInfoLookup(local(), options({enabled: false}))).toBe('consult');
|
||||
});
|
||||
|
||||
it('consults when the allowlist is empty', () => {
|
||||
expect(prescreenIpInfoLookup(local(), options({allowedAsns: new Set()}))).toBe('consult');
|
||||
});
|
||||
|
||||
it('consults when the local city database has no country', () => {
|
||||
expect(prescreenIpInfoLookup(local({countryIso: null}), options())).toBe('consult');
|
||||
});
|
||||
|
||||
it('consults when the local ASN database has no ASN', () => {
|
||||
expect(prescreenIpInfoLookup(local({asn: null}), options())).toBe('consult');
|
||||
});
|
||||
|
||||
it('consults when the ASN is not on the allowlist', () => {
|
||||
expect(prescreenIpInfoLookup(local({asn: 64501}), options())).toBe('consult');
|
||||
});
|
||||
|
||||
it('consults for a trusted commercial privacy provider even on the allowlist', () => {
|
||||
expect(prescreenIpInfoLookup(local({asnOrg: 'Example Privacy Relay LLC'}), options())).toBe('consult');
|
||||
});
|
||||
|
||||
it('consults for an education organization even on the allowlist', () => {
|
||||
expect(prescreenIpInfoLookup(local({asnOrg: 'North Example Academy'}), options())).toBe('consult');
|
||||
});
|
||||
|
||||
it('consults for a cellular organization even on the allowlist', () => {
|
||||
expect(prescreenIpInfoLookup(local({asnOrg: 'Example cell-net Wireless'}), options())).toBe('consult');
|
||||
});
|
||||
|
||||
it('skips when the allowlisted ASN carries no organization name to veto', () => {
|
||||
expect(prescreenIpInfoLookup(local({asnOrg: null}), options())).toBe('skip');
|
||||
});
|
||||
});
|
||||
|
||||
describe('ipInfoPrescreenOptionsFromEnv', () => {
|
||||
it('is disabled with an empty allowlist by default', () => {
|
||||
const parsed = ipInfoPrescreenOptionsFromEnv();
|
||||
expect(parsed.enabled).toBe(false);
|
||||
expect(parsed.allowedAsns.size).toBe(0);
|
||||
});
|
||||
|
||||
it('treats only 1 and true as enabled', () => {
|
||||
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ENABLED = '1';
|
||||
expect(ipInfoPrescreenOptionsFromEnv().enabled).toBe(true);
|
||||
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ENABLED = 'TRUE';
|
||||
expect(ipInfoPrescreenOptionsFromEnv().enabled).toBe(true);
|
||||
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ENABLED = 'yes';
|
||||
expect(ipInfoPrescreenOptionsFromEnv().enabled).toBe(false);
|
||||
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ENABLED = '0';
|
||||
expect(ipInfoPrescreenOptionsFromEnv().enabled).toBe(false);
|
||||
});
|
||||
|
||||
it('parses a comma separated allowlist and drops non numeric entries', () => {
|
||||
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ALLOW_ASNS = ' 64500, 64501 ,,notanasn,64502x,-3,64503 ';
|
||||
expect([...ipInfoPrescreenOptionsFromEnv().allowedAsns]).toEqual([64500, 64501, 64503]);
|
||||
});
|
||||
|
||||
it('parses an empty allowlist from an empty or whitespace value', () => {
|
||||
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ALLOW_ASNS = '';
|
||||
expect(ipInfoPrescreenOptionsFromEnv().allowedAsns.size).toBe(0);
|
||||
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ALLOW_ASNS = ' , ,';
|
||||
expect(ipInfoPrescreenOptionsFromEnv().allowedAsns.size).toBe(0);
|
||||
});
|
||||
|
||||
it('yields a consult verdict for every input with the shipped defaults', () => {
|
||||
const shipped = ipInfoPrescreenOptionsFromEnv();
|
||||
expect(prescreenIpInfoLookup(local(), shipped)).toBe('consult');
|
||||
expect(prescreenIpInfoLookup(local({asnOrg: null}), shipped)).toBe('consult');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,130 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {IpInfoCache} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {createTieredIpInfoCache} from '@pkgs/geoip/src/TieredIpInfoCache';
|
||||
import {describe, expect, it} from 'vitest';
|
||||
|
||||
interface RecordedSet {
|
||||
key: string;
|
||||
value: unknown;
|
||||
ttlSeconds: number | undefined;
|
||||
}
|
||||
|
||||
interface RecordingCache {
|
||||
cache: IpInfoCache;
|
||||
store: Map<string, unknown>;
|
||||
sets: Array<RecordedSet>;
|
||||
}
|
||||
|
||||
function createRecordingCache(): RecordingCache {
|
||||
const store = new Map<string, unknown>();
|
||||
const sets: Array<RecordedSet> = [];
|
||||
return {
|
||||
store,
|
||||
sets,
|
||||
cache: {
|
||||
async get<T>(key: string): Promise<T | null> {
|
||||
return (store.get(key) as T | undefined) ?? null;
|
||||
},
|
||||
async set<T>(key: string, value: T, ttlSeconds?: number): Promise<void> {
|
||||
store.set(key, value);
|
||||
sets.push({key, value, ttlSeconds});
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('TieredIpInfoCache', () => {
|
||||
it('clamps the hot TTL to the requested TTL and passes the raw TTL to the cold tier', async () => {
|
||||
const hot = createRecordingCache();
|
||||
const cold = createRecordingCache();
|
||||
const tiered = createTieredIpInfoCache({hot: hot.cache, cold: cold.cache});
|
||||
|
||||
await tiered.set('a', {available: false}, 60);
|
||||
|
||||
expect(hot.sets).toEqual([{key: 'a', value: {available: false}, ttlSeconds: 60}]);
|
||||
expect(cold.sets).toEqual([{key: 'a', value: {available: false}, ttlSeconds: 60}]);
|
||||
});
|
||||
|
||||
it('caps the hot TTL at the configured hot window', async () => {
|
||||
const hot = createRecordingCache();
|
||||
const cold = createRecordingCache();
|
||||
const tiered = createTieredIpInfoCache({hot: hot.cache, cold: cold.cache});
|
||||
|
||||
await tiered.set('a', {available: true}, 100000);
|
||||
|
||||
expect(hot.sets[0]?.ttlSeconds).toBe(600);
|
||||
expect(cold.sets[0]?.ttlSeconds).toBe(100000);
|
||||
});
|
||||
|
||||
it('uses the hot window when no TTL is supplied', async () => {
|
||||
const hot = createRecordingCache();
|
||||
const cold = createRecordingCache();
|
||||
const tiered = createTieredIpInfoCache({hot: hot.cache, cold: cold.cache});
|
||||
|
||||
await tiered.set('a', {available: true});
|
||||
|
||||
expect(hot.sets[0]?.ttlSeconds).toBe(600);
|
||||
expect(cold.sets[0]?.ttlSeconds).toBeUndefined();
|
||||
});
|
||||
|
||||
it('skips the cold write when skipColdWrite matches', async () => {
|
||||
const hot = createRecordingCache();
|
||||
const cold = createRecordingCache();
|
||||
const tiered = createTieredIpInfoCache({
|
||||
hot: hot.cache,
|
||||
cold: cold.cache,
|
||||
skipColdWrite: (value) => (value as {available?: unknown}).available === false,
|
||||
});
|
||||
|
||||
await tiered.set('a', {available: false}, 60);
|
||||
await tiered.set('b', {available: true}, 60);
|
||||
|
||||
expect(hot.sets.map((entry) => entry.key)).toEqual(['a', 'b']);
|
||||
expect(cold.sets.map((entry) => entry.key)).toEqual(['b']);
|
||||
});
|
||||
|
||||
it('promotes a cold hit into the hot tier', async () => {
|
||||
const hot = createRecordingCache();
|
||||
const cold = createRecordingCache();
|
||||
cold.store.set('a', {available: true});
|
||||
const tiered = createTieredIpInfoCache({hot: hot.cache, cold: cold.cache});
|
||||
|
||||
const hit = await tiered.get('a');
|
||||
|
||||
expect(hit).toEqual({available: true});
|
||||
expect(hot.sets).toEqual([{key: 'a', value: {available: true}, ttlSeconds: 600}]);
|
||||
});
|
||||
|
||||
it('never promotes a cold hit that skipColdWrite matches', async () => {
|
||||
const hot = createRecordingCache();
|
||||
const cold = createRecordingCache();
|
||||
cold.store.set('a', {available: false});
|
||||
const tiered = createTieredIpInfoCache({
|
||||
hot: hot.cache,
|
||||
cold: cold.cache,
|
||||
skipColdWrite: (value) => (value as {available?: unknown}).available === false,
|
||||
});
|
||||
|
||||
const hit = await tiered.get('a');
|
||||
|
||||
expect(hit).toEqual({available: false});
|
||||
expect(hot.sets).toEqual([]);
|
||||
});
|
||||
|
||||
it('never writes a zero TTL', async () => {
|
||||
const hot = createRecordingCache();
|
||||
const cold = createRecordingCache();
|
||||
const tiered = createTieredIpInfoCache({hot: hot.cache, cold: cold.cache});
|
||||
|
||||
await tiered.set('a', {available: false}, 60);
|
||||
await tiered.set('b', {available: true}, 100000);
|
||||
await tiered.set('c', {available: true});
|
||||
cold.store.set('d', {available: true});
|
||||
await tiered.get('d');
|
||||
|
||||
for (const entry of [...hot.sets, ...cold.sets]) {
|
||||
expect(entry.ttlSeconds === undefined || entry.ttlSeconds > 0).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,15 +1,18 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {parseIpAddress} from '@fluxer/ip_utils/src/IpAddress';
|
||||
import type {GeoipAsnResult, GeoipResult} from '@pkgs/geoip/src/GeoipLookup';
|
||||
import type {IpInfoService} from '@pkgs/geoip/src/IpInfoService';
|
||||
import type {GeoIpAsnResult, GeoIpCityResult} from '../RiskTypes';
|
||||
|
||||
interface GeoIpCityContext {
|
||||
ipInfoService: IpInfoService;
|
||||
lookupLocalCity?: (ip: string) => Promise<GeoipResult>;
|
||||
}
|
||||
|
||||
interface GeoIpAsnContext {
|
||||
ipInfoService: IpInfoService;
|
||||
lookupLocalAsn?: (ip: string) => Promise<GeoipAsnResult>;
|
||||
}
|
||||
|
||||
export function createGeoIpCityAdapter(ctx: GeoIpCityContext) {
|
||||
@@ -19,6 +22,22 @@ export function createGeoIpCityAdapter(ctx: GeoIpCityContext) {
|
||||
if (!parsed) {
|
||||
return notFound(ip, true);
|
||||
}
|
||||
const local = ctx.lookupLocalCity ? await ctx.lookupLocalCity(parsed.normalized) : null;
|
||||
if (local && local.countryCode !== null) {
|
||||
return {
|
||||
ip,
|
||||
available: true,
|
||||
found: true,
|
||||
countryIso: local.countryCode,
|
||||
country: local.countryName,
|
||||
region: local.region,
|
||||
city: local.city,
|
||||
latitude: local.latitude ?? null,
|
||||
longitude: local.longitude ?? null,
|
||||
accuracyRadiusKm: local.accuracyRadiusKm ?? null,
|
||||
timeZone: local.timeZone ?? null,
|
||||
};
|
||||
}
|
||||
const info = await ctx.ipInfoService.lookup(parsed.normalized, {
|
||||
source: 'risk.geoip_city',
|
||||
reason: 'registration_risk',
|
||||
@@ -63,6 +82,10 @@ export function createGeoIpAsnAdapter(ctx: GeoIpAsnContext) {
|
||||
if (!parsed) {
|
||||
return {ip, available: true, found: false, asn: null, asnOrg: null};
|
||||
}
|
||||
const local = ctx.lookupLocalAsn ? await ctx.lookupLocalAsn(parsed.normalized) : null;
|
||||
if (local && local.asn !== null) {
|
||||
return {ip, available: true, found: true, asn: local.asn, asnOrg: local.asnOrg};
|
||||
}
|
||||
const info = await ctx.ipInfoService.lookup(parsed.normalized, {
|
||||
source: 'risk.geoip_asn',
|
||||
reason: 'registration_risk',
|
||||
|
||||
@@ -2,14 +2,40 @@
|
||||
|
||||
import type {IpInfoService} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {isAccountPolicyEducationOrganizationName} from '../AccountPolicyService';
|
||||
import type {IpInfoPrescreenVerdict} from '../RegistrationIpPrescreen';
|
||||
import type {IpConnectionType, IpInfoAnonymousResult} from '../RiskTypes';
|
||||
|
||||
interface IpInfoCheckerContext {
|
||||
ipInfoService: IpInfoService;
|
||||
prescreen?: (ip: string) => Promise<IpInfoPrescreenVerdict>;
|
||||
}
|
||||
|
||||
export function unavailableIpInfoAnonymousResult(ip: string, riskNote: string): IpInfoAnonymousResult {
|
||||
return {
|
||||
ip,
|
||||
available: false,
|
||||
isAnonymous: false,
|
||||
providerName: null,
|
||||
isVpn: false,
|
||||
isProxy: false,
|
||||
isResidentialProxy: false,
|
||||
isTor: false,
|
||||
isRelay: false,
|
||||
isHosting: false,
|
||||
isMobile: false,
|
||||
asnType: null,
|
||||
asnOrg: null,
|
||||
connectionType: 'unknown',
|
||||
percentDaysSeen: null,
|
||||
riskNote,
|
||||
};
|
||||
}
|
||||
|
||||
export function createIpInfoChecker(ctx: IpInfoCheckerContext) {
|
||||
return async function checkIpInfo(ip: string): Promise<IpInfoAnonymousResult> {
|
||||
if (ctx.prescreen && (await ctx.prescreen(ip)) === 'skip') {
|
||||
return unavailableIpInfoAnonymousResult(ip, 'IPInfo skipped (local pre-screen)');
|
||||
}
|
||||
const result = await ctx.ipInfoService.lookup(ip, {
|
||||
source: 'risk.ipinfo_checker',
|
||||
reason: 'registration_risk',
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {resetGeoipReadersForTesting} from '@pkgs/geoip/src/GeoipLookup';
|
||||
import {resetSudoModeServiceForTesting} from '../auth/services/SudoModeService';
|
||||
import {resetSsoRequestUrlPolicyForTesting} from '../instance/SsoConfigValidation';
|
||||
import {resetGlobalLimitConfigServiceForTesting} from '../limits/LimitConfigService';
|
||||
@@ -32,6 +33,7 @@ export function resetServiceStateForTesting(): void {
|
||||
torExitListCache.shutdown();
|
||||
torExitListCache.clearForTesting();
|
||||
urlBlocklistCache.resetForTesting();
|
||||
resetGeoipReadersForTesting();
|
||||
fileShaCache.resetForTesting();
|
||||
phraseBlocklistCache.resetForTesting();
|
||||
bannedAvatarHashCache.resetForTesting();
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {HttpResponse, http} from 'msw';
|
||||
|
||||
export function createIpInfoLookupHandler() {
|
||||
return http.get('https://api.ipinfo.io/lookup/:ip', ({params}) => {
|
||||
const ip = typeof params.ip === 'string' ? params.ip : '198.51.100.1';
|
||||
return HttpResponse.json({
|
||||
ip,
|
||||
geo: {
|
||||
city: 'Ashburn',
|
||||
region: 'Virginia',
|
||||
region_code: 'VA',
|
||||
country: 'United States',
|
||||
country_code: 'US',
|
||||
continent: 'North America',
|
||||
continent_code: 'NA',
|
||||
},
|
||||
as: {
|
||||
asn: 'AS64500',
|
||||
name: 'Test ISP',
|
||||
domain: 'example.com',
|
||||
type: 'isp',
|
||||
},
|
||||
anonymous: {},
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import {setupServer} from 'msw/node';
|
||||
import {createBunnyEdgeHandlers} from './handlers/BunnyEdgeHandlers';
|
||||
import {createIpInfoLookupHandler} from './handlers/IpInfoHandlers';
|
||||
import {createNcmecHandlers} from './handlers/NcmecHandlers';
|
||||
import {createOnionooDetailsHandler} from './handlers/OnionooHandlers';
|
||||
import {createOpenNsfwHandlers} from './handlers/OpenNsfwHandlers';
|
||||
@@ -11,6 +12,7 @@ export const server = setupServer(
|
||||
...createBunnyEdgeHandlers(),
|
||||
...createNcmecHandlers(),
|
||||
...createOpenNsfwHandlers(),
|
||||
createIpInfoLookupHandler(),
|
||||
createOnionooDetailsHandler(),
|
||||
createPwnedPasswordsRangeHandler(),
|
||||
);
|
||||
Reference in new issue
Block a user