pepepper/fluxer
1
0
Fork 0
mirror of https://github.com/fluxerapp/fluxer synced 2026-10-07 19:22:14 +09:00
Code Issues Packages Projects Releases Wiki Activity

refactor(api): drop Bunny for pluggable cache purge adapters (#2742)

Browse Source
This commit is contained in:
Hampus
2026-09-13 20:28:56 +02:00
committed by GitHub
parent 84dcf6b8a8
commit ca7ddd9272
45 changed files with 929 additions and 977 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fluxer_api/src/api/Config.ts
+7 -4
View File
@@ -409,10 +409,13 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
: undefined,
legacyPrices: master.integrations.stripe.legacy_prices,
},
bunny: {
purgeEnabled: master.integrations.bunny.purge_enabled,
apiKey: master.integrations.bunny.api_key,
pullZoneId: master.integrations.bunny.pull_zone_id,
cachePurge: {
adapter: master.integrations.cache_purge.adapter,
http: {
endpoint: master.integrations.cache_purge.http.endpoint,
token: master.integrations.cache_purge.http.token,
timeoutMs: master.integrations.cache_purge.http.timeout_ms,
},
},
clamav: {
enabled: master.integrations.clamav.enabled,
fluxer_api/src/api/admin/tests/AdminSyntheticUserServiceGuards.test.ts
+1 -1
View File
@@ -5,7 +5,7 @@ import type {IChannelRepository} from '@app/api/channel/IChannelRepository';
import {UserMessageDeletionService} from '@app/api/channel/services/message/UserMessageDeletionService';
import type {IGuildRepositoryAggregate} from '@app/api/guild/repositories/IGuildRepositoryAggregate';
import {GuildMemberOperationsService} from '@app/api/guild/services/member/GuildMemberOperationsService';
import type {IPurgeQueue} from '@app/api/infrastructure/BunnyPurgeQueue';
import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import type {IGatewayService} from '@app/api/infrastructure/IGatewayService';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import {DELETED_USER_ID} from '@fluxer/constants/src/UserConstants';
fluxer_api/src/api/channel/services/AttachmentUploadService.ts
+3 -5
View File
@@ -21,7 +21,7 @@ import {
} from '@app/api/channel/services/message/MessageHelpers';
import {applyUploadRelayDecision, resolveUploadRelayDecision} from '@app/api/channel/services/UploadRelay';
import {SYSTEM_USER_ID} from '@app/api/constants/Core';
import type {IPurgeQueue} from '@app/api/infrastructure/BunnyPurgeQueue';
import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import type {IGatewayService} from '@app/api/infrastructure/IGatewayService';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import type {LimitConfigService} from '@app/api/limits/LimitConfigService';
@@ -387,10 +387,8 @@ export class AttachmentUploadService {
}
const cdnKey = makeAttachmentCdnKey(message.channelId, attachment.id, attachment.filename);
await this.storageService.deleteObject(Config.s3.buckets.cdn, cdnKey);
if (Config.bunny.purgeEnabled) {
const cdnUrl = makeAttachmentCdnUrl(message.channelId, attachment.id, attachment.filename);
await this.purgeQueue.addUrls([cdnUrl]);
}
const cdnUrl = makeAttachmentCdnUrl(message.channelId, attachment.id, attachment.filename);
await this.purgeQueue.addUrls([cdnUrl]);
const updatedAttachments = message.attachments.filter((a: Attachment) => a.id !== attachmentId);
const updatedRowData = {
...message.toRow(),
fluxer_api/src/api/channel/services/ChannelDataService.ts
+1 -1
View File
@@ -12,7 +12,7 @@ import type {MessagePersistenceService} from '@app/api/channel/services/message/
import type {GuildAuditLogService} from '@app/api/guild/GuildAuditLogService';
import type {IGuildRepositoryAggregate} from '@app/api/guild/repositories/IGuildRepositoryAggregate';
import type {AvatarService} from '@app/api/infrastructure/AvatarService';
import type {IPurgeQueue} from '@app/api/infrastructure/BunnyPurgeQueue';
import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import type {IGatewayService} from '@app/api/infrastructure/IGatewayService';
import type {ILiveKitService} from '@app/api/infrastructure/ILiveKitService';
import type {ISnowflakeService} from '@app/api/infrastructure/ISnowflakeService';
fluxer_api/src/api/channel/services/ChannelService.ts
+1 -1
View File
@@ -16,7 +16,7 @@ import type {IFavoriteMemeRepository} from '@app/api/favorite_meme/IFavoriteMeme
import type {GuildAuditLogService} from '@app/api/guild/GuildAuditLogService';
import type {IGuildRepositoryAggregate} from '@app/api/guild/repositories/IGuildRepositoryAggregate';
import type {AvatarService} from '@app/api/infrastructure/AvatarService';
import type {IPurgeQueue} from '@app/api/infrastructure/BunnyPurgeQueue';
import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import type {EmbedService} from '@app/api/infrastructure/EmbedService';
import type {ILiveKitService} from '@app/api/infrastructure/ILiveKitService';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
fluxer_api/src/api/channel/services/MessageService.ts
+1 -1
View File
@@ -20,7 +20,7 @@ import {MessageValidationService} from '@app/api/channel/services/message/Messag
import type {IFavoriteMemeRepository} from '@app/api/favorite_meme/IFavoriteMemeRepository';
import type {GuildAuditLogService} from '@app/api/guild/GuildAuditLogService';
import type {IGuildRepositoryAggregate} from '@app/api/guild/repositories/IGuildRepositoryAggregate';
import type {IPurgeQueue} from '@app/api/infrastructure/BunnyPurgeQueue';
import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import type {IGatewayService} from '@app/api/infrastructure/IGatewayService';
import type {IMediaService} from '@app/api/infrastructure/IMediaService';
import type {ISnowflakeService} from '@app/api/infrastructure/ISnowflakeService';
fluxer_api/src/api/channel/services/channel_data/ChannelUtilsService.ts
+1 -1
View File
@@ -6,7 +6,7 @@ import type {IChannelRepositoryAggregate} from '@app/api/channel/repositories/IC
import {dispatchChannelEvent} from '@app/api/channel/services/ChannelGatewayDispatch';
import {dispatchMessageCreateBroadcast} from '@app/api/channel/services/message/MessageGatewayDispatch';
import {purgeMessageAttachments} from '@app/api/channel/services/message/MessageHelpers';
import type {IPurgeQueue} from '@app/api/infrastructure/BunnyPurgeQueue';
import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import type {IGatewayService} from '@app/api/infrastructure/IGatewayService';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import type {UserCacheService} from '@app/api/infrastructure/UserCacheService';
fluxer_api/src/api/channel/services/message/MessageDeleteService.ts
+1 -1
View File
@@ -9,7 +9,7 @@ import {isOperationDisabled, purgeMessageAttachments} from '@app/api/channel/ser
import type {MessageSearchService} from '@app/api/channel/services/message/MessageSearchService';
import type {MessageValidationService} from '@app/api/channel/services/message/MessageValidationService';
import type {GuildAuditLogService} from '@app/api/guild/GuildAuditLogService';
import type {IPurgeQueue} from '@app/api/infrastructure/BunnyPurgeQueue';
import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import type {IGatewayService} from '@app/api/infrastructure/IGatewayService';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import type {RequestCache} from '@app/api/middleware/RequestCacheMiddleware';
fluxer_api/src/api/channel/services/message/MessageHelpers.test.ts
+112
View File
@@ -0,0 +1,112 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createAttachmentID, createChannelID, createMessageID, createUserID} from '@app/api/BrandedTypes';
import {Config} from '@app/api/Config';
import {purgeMessageAttachments} from '@app/api/channel/services/message/MessageHelpers';
import type {MessageEmbed} from '@app/api/database/types/MessageTypes';
import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import {Message} from '@app/api/models/Message';
import {MessageTypes} from '@fluxer/constants/src/ChannelConstants';
import {describe, expect, it} from 'vitest';
const CHANNEL_ID = createChannelID(10n);
const ATTACHMENT_KEY = 'attachments/10/200/ação.png';
const OTHER_MESSAGE_ATTACHMENT_KEY = 'attachments/11/300/photo.jpg';
function imageEmbed(key: string): MessageEmbed {
return {
type: 'image',
title: null,
description: null,
url: null,
timestamp: null,
color: null,
author: null,
provider: null,
thumbnail: null,
image: {
url: `${Config.endpoints.media}/${key}`,
width: 16,
height: 16,
description: null,
content_type: 'image/png',
content_hash: null,
placeholder: null,
flags: 0,
duration: null,
},
video: null,
footer: null,
fields: null,
nsfw: null,
};
}
function makeMessageWithMedia(): Message {
return new Message({
channel_id: CHANNEL_ID,
bucket: 0,
message_id: createMessageID(100n),
author_id: createUserID(3n),
type: MessageTypes.DEFAULT,
webhook_id: null,
webhook_name: null,
webhook_avatar_hash: null,
content: '',
edited_timestamp: null,
pinned_timestamp: null,
flags: 0,
mention_everyone: false,
mention_users: null,
mention_roles: null,
mention_channels: null,
attachments: [
{
attachment_id: createAttachmentID(200n),
filename: 'ação.png',
size: 1024n,
title: null,
description: null,
width: 16,
height: 16,
content_type: 'image/png',
content_hash: null,
placeholder: null,
flags: 0,
duration: null,
nsfw: null,
waveform: null,
},
],
embeds: [imageEmbed(ATTACHMENT_KEY), imageEmbed(OTHER_MESSAGE_ATTACHMENT_KEY)],
sticker_items: null,
message_reference: null,
message_snapshots: null,
call: null,
has_reaction: null,
version: 1,
});
}
describe('purgeMessageAttachments', () => {
it('queues each stored media URL of a deleted message once and leaves media of other messages alone', async () => {
const deletedObjects: Array<string> = [];
const queuedUrls: Array<string> = [];
const storageService = {
deleteObject: async (bucket: string, key: string) => {
deletedObjects.push(`${bucket}/${key}`);
},
} as unknown as IStorageService;
const purgeQueue: IPurgeQueue = {
addUrls: async (urls) => {
queuedUrls.push(...urls);
},
};
await purgeMessageAttachments(makeMessageWithMedia(), storageService, purgeQueue);
expect(deletedObjects).toEqual([`${Config.s3.buckets.cdn}/${ATTACHMENT_KEY}`]);
expect(queuedUrls).toEqual([`${Config.endpoints.media}/${ATTACHMENT_KEY}`]);
});
});
fluxer_api/src/api/channel/services/message/MessageHelpers.ts
+4 -8
View File
@@ -7,7 +7,7 @@ import type {
MessageSnapshot as CassandraMessageSnapshot,
MessageAttachment,
} from '@app/api/database/types/MessageTypes';
import type {IPurgeQueue} from '@app/api/infrastructure/BunnyPurgeQueue';
import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import type {ISnowflakeService} from '@app/api/infrastructure/ISnowflakeService';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import {Logger} from '@app/api/Logger';
@@ -353,21 +353,17 @@ export async function purgeMessageAttachments(
continue;
}
cdnKeys.add(cdnKey);
if (Config.bunny.purgeEnabled) {
cdnUrls.push(makeAttachmentCdnUrl(message.channelId, attachment.id, attachment.filename));
}
cdnUrls.push(makeAttachmentCdnUrl(message.channelId, attachment.id, attachment.filename));
}
for (const embedKey of collectEmbedReferencedAttachmentCdnKeys(message, ownedCdnKeys)) {
if (cdnKeys.has(embedKey)) {
continue;
}
cdnKeys.add(embedKey);
if (Config.bunny.purgeEnabled) {
cdnUrls.push(`${Config.endpoints.media}/${embedKey}`);
}
cdnUrls.push(`${Config.endpoints.media}/${embedKey}`);
}
await Promise.all([...cdnKeys].map((cdnKey) => storageService.deleteObject(Config.s3.buckets.cdn, cdnKey)));
if (Config.bunny.purgeEnabled && cdnUrls.length > 0) {
if (cdnUrls.length > 0) {
await purgeQueue.addUrls(cdnUrls);
}
}
fluxer_api/src/api/channel/services/message/UserMessageDeletionService.ts
+1 -1
View File
@@ -11,7 +11,7 @@ import {
type SelfMessageFilter,
} from '@app/api/channel/services/message/SelfMessageFilter';
import {assertMutableUserId} from '@app/api/constants/Core';
import type {IPurgeQueue} from '@app/api/infrastructure/BunnyPurgeQueue';
import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import type {IGatewayService} from '@app/api/infrastructure/IGatewayService';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import {Logger} from '@app/api/Logger';
fluxer_api/src/api/config/APIConfig.ts
+11 -5
View File
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig';
import type {CachePurgeAdapterName} from '@fluxer/config/src/MasterConfig';
import type {ResolvedDownloadsProvider} from '@fluxer/config/src/S3DownloadsProvider';
export type APIWorkerMode = 'all_lanes' | 'single_lane' | 'single_task';
@@ -14,6 +15,15 @@ export interface PushProviderAppConfig {
projectId?: string;
}
export interface APICachePurgeConfig {
adapter: CachePurgeAdapterName;
http: {
endpoint: string;
token: string;
timeoutMs: number;
};
}
interface APIGeoipFilesystemConfig {
mode: 'filesystem';
maxmindDbPath?: string;
@@ -262,11 +272,7 @@ export interface APIConfig {
};
legacyPrices?: Record<string, Array<string> | undefined>;
};
bunny: {
purgeEnabled: boolean;
apiKey?: string;
pullZoneId?: number;
};
cachePurge: APICachePurgeConfig;
clamav: {
enabled: boolean;
host: string;
fluxer_api/src/api/csam/NcmecSubmissionService.ts
+1 -1
View File
@@ -29,7 +29,7 @@ import type {NcmecRepository} from '@app/api/csam/NcmecRepository';
import type {AttachmentUploadTraceByAttachmentRow} from '@app/api/database/types/AttachmentUploadTypes';
import type {NcmecAttachmentSubmissionRow, NcmecUserWorkflowRow} from '@app/api/database/types/CsamTypes';
import type {IGuildRepositoryAggregate} from '@app/api/guild/repositories/IGuildRepositoryAggregate';
import type {IPurgeQueue} from '@app/api/infrastructure/BunnyPurgeQueue';
import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import type {IGatewayService} from '@app/api/infrastructure/IGatewayService';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import type {KVAccountDeletionQueueService} from '@app/api/infrastructure/KVAccountDeletionQueueService';
fluxer_api/src/api/infrastructure/BunnyPurgeQueue.ts
-124
View File
@@ -1,124 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Logger} from '@app/api/Logger';
import type {IKVProvider, KVPurgeBatchResult} from '@pkgs/kv_client/src/IKVProvider';
export interface IPurgeQueue {
addUrls(urls: Array<string>): Promise<void>;
getQueueSize(): Promise<number>;
clear(): Promise<void>;
}
const EXACT_QUEUE_KEY = 'bunny:purge:exact';
const PREFIX_QUEUE_KEY = 'bunny:purge:prefix';
const EXACT_BUCKET_KEY = 'bunny:purge:budget:exact';
const PREFIX_BUCKET_KEY = 'bunny:purge:budget:prefix';
const EXACT_MAX_TOKENS = 120;
const EXACT_REFILL_RATE = 5;
const EXACT_REFILL_INTERVAL_MS = 1000;
const PREFIX_MAX_TOKENS = 20;
const PREFIX_REFILL_RATE = 1;
const PREFIX_REFILL_INTERVAL_MS = 2000;
function isPrefix(url: string): boolean {
return url.endsWith('*') || url.endsWith('/');
}
export class BunnyPurgeQueue implements IPurgeQueue {
private readonly kvClient: IKVProvider;
constructor(kvClient: IKVProvider) {
this.kvClient = kvClient;
}
async addUrls(urls: Array<string>): Promise<void> {
if (urls.length === 0) {
return;
}
const exactUrls: Array<string> = [];
const prefixUrls: Array<string> = [];
for (const url of urls) {
const trimmed = url.trim();
if (trimmed === '') {
continue;
}
if (isPrefix(trimmed)) {
prefixUrls.push(trimmed);
} else {
exactUrls.push(trimmed);
}
}
try {
const ops: Array<Promise<number>> = [];
if (exactUrls.length > 0) {
ops.push(this.kvClient.sadd(EXACT_QUEUE_KEY, ...exactUrls));
}
if (prefixUrls.length > 0) {
ops.push(this.kvClient.sadd(PREFIX_QUEUE_KEY, ...prefixUrls));
}
await Promise.all(ops);
Logger.debug({exact: exactUrls.length, prefix: prefixUrls.length}, 'Added URLs to CDN purge queue');
} catch (error) {
Logger.error(
{error, exact: exactUrls.length, prefix: prefixUrls.length},
'Failed to add URLs to CDN purge queue',
);
throw error;
}
}
async getQueueSize(): Promise<number> {
try {
const [exactSize, prefixSize] = await Promise.all([
this.kvClient.scard(EXACT_QUEUE_KEY),
this.kvClient.scard(PREFIX_QUEUE_KEY),
]);
return exactSize + prefixSize;
} catch (error) {
Logger.error({error}, 'Failed to get CDN purge queue size');
throw error;
}
}
async clear(): Promise<void> {
try {
await this.kvClient.del(EXACT_QUEUE_KEY, PREFIX_QUEUE_KEY);
Logger.debug('Cleared CDN purge queue');
} catch (error) {
Logger.error({error}, 'Failed to clear CDN purge queue');
throw error;
}
}
async dequeueExactBatch(maxItems: number): Promise<KVPurgeBatchResult> {
return this.kvClient.dequeuePurgeBatch(
EXACT_QUEUE_KEY,
EXACT_BUCKET_KEY,
maxItems,
EXACT_MAX_TOKENS,
EXACT_REFILL_RATE,
EXACT_REFILL_INTERVAL_MS,
);
}
async dequeuePrefixBatch(maxItems: number): Promise<KVPurgeBatchResult> {
return this.kvClient.dequeuePurgeBatch(
PREFIX_QUEUE_KEY,
PREFIX_BUCKET_KEY,
maxItems,
PREFIX_MAX_TOKENS,
PREFIX_REFILL_RATE,
PREFIX_REFILL_INTERVAL_MS,
);
}
}
export class NoopPurgeQueue implements IPurgeQueue {
async addUrls(_urls: Array<string>): Promise<void> {}
async getQueueSize(): Promise<number> {
return 0;
}
async clear(): Promise<void> {}
}
fluxer_api/src/api/infrastructure/CachePurgeAdapter.ts
+29
View File
@@ -0,0 +1,29 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {APICachePurgeConfig} from '@app/api/config/APIConfig';
import {createHttpCachePurgeAdapter} from '@app/api/infrastructure/HttpCachePurgeAdapter';
import {createNoneCachePurgeAdapter} from '@app/api/infrastructure/NoneCachePurgeAdapter';
import type {CachePurgeAdapterName} from '@fluxer/config/src/MasterConfig';
export interface CachePurgeBatch {
readonly exact: ReadonlyArray<string>;
readonly prefix: ReadonlyArray<string>;
}
export type CachePurgeOutcome =
| {readonly kind: 'purged'}
| {readonly kind: 'invalid_entries'; readonly status: number}
| {readonly kind: 'failed'; readonly status: number | null; readonly error: unknown};
export interface CachePurgeAdapter {
purge(batch: CachePurgeBatch): Promise<CachePurgeOutcome>;
}
const CACHE_PURGE_ADAPTERS = {
none: createNoneCachePurgeAdapter,
http: createHttpCachePurgeAdapter,
} satisfies Record<CachePurgeAdapterName, (config: APICachePurgeConfig) => CachePurgeAdapter>;
export function createCachePurgeAdapter(config: APICachePurgeConfig): CachePurgeAdapter {
return CACHE_PURGE_ADAPTERS[config.adapter](config);
}
fluxer_api/src/api/infrastructure/CachePurgeQueue.ts
+122
View File
@@ -0,0 +1,122 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {CachePurgeBatch} from '@app/api/infrastructure/CachePurgeAdapter';
import {Logger} from '@app/api/Logger';
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
export interface IPurgeQueue {
addUrls(urls: Array<string>): Promise<void>;
}
const EXACT_QUEUE_KEY = 'cache_purge:exact';
const PREFIX_QUEUE_KEY = 'cache_purge:prefix';
const EXACT_BUCKET_KEY = 'cache_purge:budget:exact';
const PREFIX_BUCKET_KEY = 'cache_purge:budget:prefix';
const REJECTED_KEY = 'cache_purge:rejected';
const EXACT_MAX_TOKENS = 120;
const EXACT_REFILL_RATE = 5;
const EXACT_REFILL_INTERVAL_MS = 1000;
const PREFIX_MAX_TOKENS = 20;
const PREFIX_REFILL_RATE = 1;
const PREFIX_REFILL_INTERVAL_MS = 2000;
function isPrefix(url: string): boolean {
return url.endsWith('*') || url.endsWith('/');
}
export class CachePurgeQueue implements IPurgeQueue {
private readonly kvClient: IKVProvider;
constructor(kvClient: IKVProvider) {
this.kvClient = kvClient;
}
async addUrls(urls: Array<string>): Promise<void> {
if (urls.length === 0) {
return;
}
const exactUrls: Array<string> = [];
const prefixUrls: Array<string> = [];
for (const url of urls) {
const trimmed = url.trim();
if (trimmed === '') {
continue;
}
if (isPrefix(trimmed)) {
prefixUrls.push(trimmed);
} else {
exactUrls.push(trimmed);
}
}
try {
await this.addToSets({exact: exactUrls, prefix: prefixUrls});
Logger.debug({exact: exactUrls.length, prefix: prefixUrls.length}, 'Added URLs to cache purge queue');
} catch (error) {
Logger.error(
{error, exact: exactUrls.length, prefix: prefixUrls.length},
'Failed to add URLs to cache purge queue',
);
throw error;
}
}
async dequeueBatch(): Promise<CachePurgeBatch> {
const exact = await this.kvClient.dequeuePurgeBatch(
EXACT_QUEUE_KEY,
EXACT_BUCKET_KEY,
EXACT_MAX_TOKENS,
EXACT_MAX_TOKENS,
EXACT_REFILL_RATE,
EXACT_REFILL_INTERVAL_MS,
);
try {
const prefix = await this.kvClient.dequeuePurgeBatch(
PREFIX_QUEUE_KEY,
PREFIX_BUCKET_KEY,
PREFIX_MAX_TOKENS,
PREFIX_MAX_TOKENS,
PREFIX_REFILL_RATE,
PREFIX_REFILL_INTERVAL_MS,
);
return {exact: exact.urls, prefix: prefix.urls};
} catch (error) {
await this.requeue({exact: exact.urls, prefix: []});
throw error;
}
}
async requeue(batch: CachePurgeBatch): Promise<void> {
try {
await this.addToSets(batch);
} catch (error) {
Logger.error(
{error, exact: batch.exact.length, prefix: batch.prefix.length},
'Failed to requeue cache purge entries',
);
throw error;
}
}
async reject(batch: CachePurgeBatch): Promise<void> {
await this.kvClient.sadd(REJECTED_KEY, ...batch.exact, ...batch.prefix);
Logger.error(
{exact: batch.exact.length, prefix: batch.prefix.length, key: REJECTED_KEY},
'Set aside cache purge entries the endpoint rejected',
);
}
private async addToSets(batch: CachePurgeBatch): Promise<void> {
const ops: Array<Promise<number>> = [];
if (batch.exact.length > 0) {
ops.push(this.kvClient.sadd(EXACT_QUEUE_KEY, ...batch.exact));
}
if (batch.prefix.length > 0) {
ops.push(this.kvClient.sadd(PREFIX_QUEUE_KEY, ...batch.prefix));
}
await Promise.all(ops);
}
}
export class NoopPurgeQueue implements IPurgeQueue {
async addUrls(_urls: Array<string>): Promise<void> {}
}
fluxer_api/src/api/infrastructure/HttpCachePurgeAdapter.ts
+40
View File
@@ -0,0 +1,40 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {APICachePurgeConfig} from '@app/api/config/APIConfig';
import type {CachePurgeAdapter, CachePurgeBatch, CachePurgeOutcome} from '@app/api/infrastructure/CachePurgeAdapter';
import * as FetchUtils from '@app/api/utils/FetchUtils';
function stripOneTrailingAsterisk(entry: string): string {
return entry.endsWith('*') ? entry.slice(0, -1) : entry;
}
export function createHttpCachePurgeAdapter(config: APICachePurgeConfig): CachePurgeAdapter {
return {
async purge(batch: CachePurgeBatch): Promise<CachePurgeOutcome> {
const headers: Record<string, string> = {'Content-Type': 'application/json'};
if (config.http.token !== '') {
headers.Authorization = `Bearer ${config.http.token}`;
}
let response: Response;
try {
response = await fetch(config.http.endpoint, {
method: 'POST',
headers,
body: JSON.stringify({exact: batch.exact, prefix: batch.prefix.map(stripOneTrailingAsterisk)}),
redirect: 'manual',
signal: AbortSignal.timeout(config.http.timeoutMs),
});
} catch (error) {
return {kind: 'failed', status: null, error};
}
FetchUtils.discardResponseBody(response.body, response.status);
if (response.ok) {
return {kind: 'purged'};
}
if (response.status === 400 || response.status === 422) {
return {kind: 'invalid_entries', status: response.status};
}
return {kind: 'failed', status: response.status, error: null};
},
};
}
fluxer_api/src/api/infrastructure/NoneCachePurgeAdapter.ts
+9
View File
@@ -0,0 +1,9 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {CachePurgeAdapter} from '@app/api/infrastructure/CachePurgeAdapter';
export function createNoneCachePurgeAdapter(): CachePurgeAdapter {
return {
purge: async () => ({kind: 'purged'}),
};
}
fluxer_api/src/api/middleware/GuildStackServiceFactory.ts
+1 -1
View File
@@ -9,7 +9,7 @@ import type {GuildAuditLogService} from '@app/api/guild/GuildAuditLogService';
import type {IGuildRepositoryAggregate} from '@app/api/guild/repositories/IGuildRepositoryAggregate';
import {GuildService} from '@app/api/guild/services/GuildService';
import type {AvatarService} from '@app/api/infrastructure/AvatarService';
import type {IPurgeQueue} from '@app/api/infrastructure/BunnyPurgeQueue';
import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import type {EmbedService} from '@app/api/infrastructure/EmbedService';
import type {EntityAssetService} from '@app/api/infrastructure/EntityAssetService';
import type {IAssetDeletionQueue} from '@app/api/infrastructure/IAssetDeletionQueue';
fluxer_api/src/api/middleware/ServiceSingletons.ts
+2 -2
View File
@@ -31,7 +31,7 @@ import {GuildRepository} from '@app/api/guild/repositories/GuildRepository';
import {GuildDiscoveryService} from '@app/api/guild/services/GuildDiscoveryService';
import {AssetDeletionQueue} from '@app/api/infrastructure/AssetDeletionQueue';
import {AvatarService} from '@app/api/infrastructure/AvatarService';
import {BunnyPurgeQueue, type IPurgeQueue, NoopPurgeQueue} from '@app/api/infrastructure/BunnyPurgeQueue';
import {CachePurgeQueue, type IPurgeQueue, NoopPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import {DisabledVirusScanService} from '@app/api/infrastructure/DisabledVirusScanService';
import {DiscriminatorService} from '@app/api/infrastructure/DiscriminatorService';
import {EmailDnsValidationService} from '@app/api/infrastructure/EmailDnsValidationService';
@@ -240,7 +240,7 @@ export const getLimitConfigService = singleton(
},
);
export const getPurgeQueue: () => IPurgeQueue = singleton(() =>
Config.bunny.purgeEnabled ? new BunnyPurgeQueue(getKVClient()) : new NoopPurgeQueue(),
Config.cachePurge.adapter === 'none' ? new NoopPurgeQueue() : new CachePurgeQueue(getKVClient()),
);
export const getAssetDeletionQueue: () => IAssetDeletionQueue = singleton(() => new AssetDeletionQueue(getKVClient()));
fluxer_api/src/api/test/msw/handlers/BunnyEdgeHandlers.ts
-27
View File
@@ -1,27 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {HttpResponse, http} from 'msw';
const BUNNY_EDGE_IPV4_LIST = ['198.51.100.10', '198.51.100.11'].join('\n');
const BUNNY_EDGE_IPV6_LIST = ['2001:db8::10', '2001:db8::11'].join('\n');
export function createBunnyEdgeHandlers() {
return [
http.get(
'https://bunnycdn.com/api/system/edgeserverlist/plain',
() =>
new HttpResponse(BUNNY_EDGE_IPV4_LIST, {
status: 200,
headers: {'content-type': 'text/plain; charset=utf-8'},
}),
),
http.get(
'https://bunnycdn.com/api/system/edgeserverlist/IPv6/plain',
() =>
new HttpResponse(BUNNY_EDGE_IPV6_LIST, {
status: 200,
headers: {'content-type': 'text/plain; charset=utf-8'},
}),
),
];
}
fluxer_api/src/api/test/msw/server.ts
-2
View File
@@ -1,6 +1,5 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createBunnyEdgeHandlers} from '@app/api/test/msw/handlers/BunnyEdgeHandlers';
import {createIpInfoLookupHandler} from '@app/api/test/msw/handlers/IpInfoHandlers';
import {createNcmecHandlers} from '@app/api/test/msw/handlers/NcmecHandlers';
import {createOnionooDetailsHandler} from '@app/api/test/msw/handlers/OnionooHandlers';
@@ -9,7 +8,6 @@ import {createPwnedPasswordsRangeHandler} from '@app/api/test/msw/handlers/Pwned
import {setupServer} from 'msw/node';
export const server = setupServer(
...createBunnyEdgeHandlers(),
...createNcmecHandlers(),
...createOpenNsfwHandlers(),
createIpInfoLookupHandler(),
fluxer_api/src/api/user/services/UserDeletionService.ts
+1 -1
View File
@@ -8,7 +8,7 @@ import type {ChannelRepository} from '@app/api/channel/ChannelRepository';
import type {IConnectionRepository} from '@app/api/connection/IConnectionRepository';
import type {FavoriteMemeRepository} from '@app/api/favorite_meme/FavoriteMemeRepository';
import type {GuildRepository} from '@app/api/guild/repositories/GuildRepository';
import type {IPurgeQueue} from '@app/api/infrastructure/BunnyPurgeQueue';
import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import type {DiscriminatorService} from '@app/api/infrastructure/DiscriminatorService';
import type {IGatewayService} from '@app/api/infrastructure/IGatewayService';
import type {ISnowflakeService} from '@app/api/infrastructure/ISnowflakeService';
fluxer_api/src/api/utils/ExternalResponseLimits.ts
-1
View File
@@ -21,7 +21,6 @@ export const EXTERNAL_RESPONSE_LIMITS = {
pwnedPasswordsBytes: 1024 * 1024,
rdapBytes: 512 * 1024,
externalTemplateBytes: 512 * 1024,
bunnyErrorBytes: 16 * 1024,
ncmecResponseBytes: 64 * 1024,
fileBlocklistBytes: 25 * 1024 * 1024,
urlBlocklistBytes: 25 * 1024 * 1024,
fluxer_api/src/api/worker/WorkerDependencies.ts
+1 -1
View File
@@ -17,7 +17,7 @@ import type {GuildAuditLogService} from '@app/api/guild/GuildAuditLogService';
import type {GuildRepository} from '@app/api/guild/repositories/GuildRepository';
import type {GuildService} from '@app/api/guild/services/GuildService';
import type {AvatarService} from '@app/api/infrastructure/AvatarService';
import type {IPurgeQueue} from '@app/api/infrastructure/BunnyPurgeQueue';
import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import type {DiscriminatorService} from '@app/api/infrastructure/DiscriminatorService';
import type {EmbedService} from '@app/api/infrastructure/EmbedService';
import type {IAssetDeletionQueue} from '@app/api/infrastructure/IAssetDeletionQueue';
fluxer_api/src/api/worker/WorkerLaneConfig.ts
+1 -1
View File
@@ -70,7 +70,7 @@ const LANE_CONFIG = {
'indexChannelMessages',
'indexGuildMembers',
'processAssetDeletionQueue',
'processBunnyPurgeQueue',
'processCachePurgeQueue',
'processExpiredPremiumSweep',
'processInactivityDeletions',
'processPendingBulkMessageDeletions',
fluxer_api/src/api/worker/WorkerMain.ts
+3 -3
View File
@@ -54,8 +54,8 @@ const SEARCH_REQUIRED_TASKS = new Set<string>([
function registerCronJobs(cron: CronScheduler): void {
cron.upsert('processAssetDeletionQueue', 'processAssetDeletionQueue', {}, '0 */5 * * * *', {ledger: false});
if (Config.bunny.purgeEnabled) {
cron.upsert('processBunnyPurgeQueue', 'processBunnyPurgeQueue', {}, '*/10 * * * * *', {ledger: false});
if (Config.cachePurge.adapter !== 'none') {
cron.upsert('processCachePurgeQueue', 'processCachePurgeQueue', {}, '*/10 * * * * *', {ledger: false});
}
cron.upsert('processPendingBulkMessageDeletions', 'processPendingBulkMessageDeletions', {}, '0 */10 * * * *', {
ledger: false,
@@ -80,7 +80,7 @@ function registerCronJobs(cron: CronScheduler): void {
Logger.info(
{
blocklistFeeds: Config.blocklistFeeds.enabled,
bunnyPurge: Config.bunny.purgeEnabled,
cachePurgeAdapter: Config.cachePurge.adapter,
selfHosted: Config.instance.selfHosted,
},
'Cron jobs registered successfully',
fluxer_api/src/api/worker/WorkerTaskRegistry.ts
+2 -2
View File
@@ -25,7 +25,7 @@ import indexChannelMessages from '@app/api/worker/tasks/IndexChannelMessages';
import indexGuildMembers from '@app/api/worker/tasks/IndexGuildMembers';
import messageShred from '@app/api/worker/tasks/MessageShred';
import processAssetDeletionQueue from '@app/api/worker/tasks/ProcessAssetDeletionQueue';
import processBunnyPurgeQueue from '@app/api/worker/tasks/ProcessBunnyPurgeQueue';
import processCachePurgeQueue from '@app/api/worker/tasks/ProcessCachePurgeQueue';
import processExpiredPremiumSweep from '@app/api/worker/tasks/ProcessExpiredPremiumSweep';
import processInactivityDeletions from '@app/api/worker/tasks/ProcessInactivityDeletions';
import processPendingBulkMessageDeletions from '@app/api/worker/tasks/ProcessPendingBulkMessageDeletions';
@@ -69,7 +69,7 @@ export const workerTasks: Record<WorkerTaskName, WorkerTaskHandler> = {
indexGuildMembers,
messageShred,
processAssetDeletionQueue,
processBunnyPurgeQueue,
processCachePurgeQueue,
processStripeWebhook,
processExpiredPremiumSweep,
processInactivityDeletions,
fluxer_api/src/api/worker/tasks/ProcessAssetDeletionQueue.ts
+1 -1
View File
@@ -3,7 +3,7 @@
import {createGuildID, createUserID} from '@app/api/BrandedTypes';
import {Config} from '@app/api/Config';
import type {GuildRepository} from '@app/api/guild/repositories/GuildRepository';
import type {IPurgeQueue} from '@app/api/infrastructure/BunnyPurgeQueue';
import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import type {
IAssetDeletionQueue,
QueuedAssetDeletion,
fluxer_api/src/api/worker/tasks/ProcessBunnyPurgeQueue.ts
-152
View File
@@ -1,152 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Config} from '@app/api/Config';
import type {BunnyPurgeQueue} from '@app/api/infrastructure/BunnyPurgeQueue';
import {Logger} from '@app/api/Logger';
import {EXTERNAL_RESPONSE_LIMITS} from '@app/api/utils/ExternalResponseLimits';
import * as FetchUtils from '@app/api/utils/FetchUtils';
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
import {formatUrlForDiagnostics} from '@pkgs/http_client/src/HttpClientDiagnostics';
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
import {ms} from 'itty-time';
const EXACT_BATCH_SIZE = 120;
const PREFIX_BATCH_SIZE = 20;
type PurgeLabel = 'exact' | 'prefix';
interface PurgeStats {
attempted: number;
purged: number;
requeued: number;
failures: number;
rateLimited: number;
}
function createEmptyPurgeStats(): PurgeStats {
return {
attempted: 0,
purged: 0,
requeued: 0,
failures: 0,
rateLimited: 0,
};
}
function mergePurgeStats(target: PurgeStats, source: PurgeStats): void {
target.attempted += source.attempted;
target.purged += source.purged;
target.requeued += source.requeued;
target.failures += source.failures;
target.rateLimited += source.rateLimited;
}
async function purgeUrls(
urls: Array<string>,
apiKey: string,
queue: BunnyPurgeQueue,
label: PurgeLabel,
): Promise<PurgeStats> {
const stats = createEmptyPurgeStats();
for (let index = 0; index < urls.length; index++) {
const url = urls[index]!;
stats.attempted++;
try {
const response = await fetch(`https://api.bunny.net/purge?url=${encodeURIComponent(url)}&async=true`, {
method: 'POST',
headers: {
AccessKey: apiKey,
},
signal: AbortSignal.timeout(ms('30 seconds')),
});
if (response.status === 429) {
FetchUtils.discardResponseBody(response.body, response.status);
const remaining = urls.slice(index);
await queue.addUrls(remaining);
stats.requeued += remaining.length;
stats.rateLimited += remaining.length;
Logger.warn({label, rateLimitedUrls: remaining.length}, 'Rate limited by Bunny CDN, re-queued remaining URLs');
break;
}
if (!response.ok) {
stats.failures++;
const errorBody = await FetchUtils.streamToBufferWithLimit(response.body, {
maxBytes: EXTERNAL_RESPONSE_LIMITS.bunnyErrorBytes,
headers: response.headers,
url: response.url,
description: 'Bunny CDN purge response',
});
Logger.error(
{
status: response.status,
responseBodyBytes: errorBody.byteLength,
url: formatUrlForDiagnostics(url),
label,
},
'Failed to purge URL via Bunny CDN',
);
await queue.addUrls([url]);
stats.requeued++;
continue;
}
FetchUtils.discardResponseBody(response.body, response.status);
stats.purged++;
} catch (error) {
stats.failures++;
Logger.error({error, url: formatUrlForDiagnostics(url), label}, 'Error purging URL via Bunny CDN');
await queue.addUrls([url]);
stats.requeued++;
}
}
return stats;
}
const processBunnyPurgeQueue: WorkerTaskHandler = async (_payload, _helpers) => {
if (!Config.bunny.purgeEnabled) {
Logger.debug('Bunny CDN cache purge is disabled, skipping queue processing');
return;
}
if (!Config.bunny.apiKey || !Config.bunny.pullZoneId) {
Logger.error('Bunny CDN cache purge is enabled but credentials are missing');
return;
}
const deps = getWorkerDependencies();
const queue = deps.purgeQueue as BunnyPurgeQueue;
const apiKey = Config.bunny.apiKey;
const totalStats = createEmptyPurgeStats();
try {
const queueSizeBefore = await queue.getQueueSize();
if (queueSizeBefore === 0) {
Logger.debug('CDN purge queue is empty');
return;
}
Logger.debug({queueSize: queueSizeBefore}, 'Processing CDN purge queue');
const exactBatch = await queue.dequeueExactBatch(EXACT_BATCH_SIZE);
if (exactBatch.urls.length > 0) {
const exactStats = await purgeUrls(exactBatch.urls, apiKey, queue, 'exact');
mergePurgeStats(totalStats, exactStats);
}
const prefixBatch = await queue.dequeuePrefixBatch(PREFIX_BATCH_SIZE);
if (prefixBatch.urls.length > 0) {
const prefixStats = await purgeUrls(prefixBatch.urls, apiKey, queue, 'prefix');
mergePurgeStats(totalStats, prefixStats);
}
const remainingQueueSize = await queue.getQueueSize();
Logger.debug(
{
totalAttempted: totalStats.attempted,
totalPurged: totalStats.purged,
totalRequeued: totalStats.requeued,
totalFailures: totalStats.failures,
totalRateLimited: totalStats.rateLimited,
remainingQueueSize,
},
'Finished processing CDN purge queue',
);
} catch (error) {
Logger.error({error}, 'Error processing CDN purge queue');
throw error;
}
};
export default processBunnyPurgeQueue;
fluxer_api/src/api/worker/tasks/ProcessCachePurgeQueue.ts
+104
View File
@@ -0,0 +1,104 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Config} from '@app/api/Config';
import {
type CachePurgeBatch,
type CachePurgeOutcome,
createCachePurgeAdapter,
} from '@app/api/infrastructure/CachePurgeAdapter';
import {CachePurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import {Logger} from '@app/api/Logger';
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
import type {CachePurgeAdapterName} from '@fluxer/config/src/MasterConfig';
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
const RUN_DEADLINE_MS = 10_000;
type CachePurgeFailure = Extract<CachePurgeOutcome, {kind: 'failed'}>;
function describeBatch(adapter: CachePurgeAdapterName, batch: CachePurgeBatch) {
return {adapter, exact: batch.exact.length, prefix: batch.prefix.length};
}
function logFailure(adapter: CachePurgeAdapterName, requeued: CachePurgeBatch, failure: CachePurgeFailure): void {
const {status, error} = failure;
const context = {...describeBatch(adapter, requeued), status, error};
if (status === null || status === 408 || status === 429 || status >= 500) {
Logger.warn(context, 'Cache purge request failed, requeued its entries');
return;
}
Logger.error(context, 'Cache purge endpoint refused the request, requeued its entries');
}
function untriedFrom(singles: ReadonlyArray<CachePurgeBatch>, index: number): CachePurgeBatch {
const untried = singles.slice(index);
return {exact: untried.flatMap((single) => single.exact), prefix: untried.flatMap((single) => single.prefix)};
}
const processCachePurgeQueue: WorkerTaskHandler = async (_payload, _helpers) => {
const adapterName = Config.cachePurge.adapter;
if (adapterName === 'none') {
Logger.warn('Skipped a cache purge run because the adapter is none');
return;
}
const adapter = createCachePurgeAdapter(Config.cachePurge);
const queue = new CachePurgeQueue(getWorkerDependencies().kvClient);
const startedAt = Date.now();
const batch = await queue.dequeueBatch();
if (batch.exact.length === 0 && batch.prefix.length === 0) {
return;
}
let outcome: CachePurgeOutcome;
try {
outcome = await adapter.purge(batch);
} catch (error) {
await queue.requeue(batch);
throw error;
}
if (outcome.kind === 'purged') {
Logger.debug(describeBatch(adapterName, batch), 'Purged a batch of cache entries');
return;
}
if (outcome.kind === 'failed') {
await queue.requeue(batch);
logFailure(adapterName, batch, outcome);
return;
}
Logger.warn(
{...describeBatch(adapterName, batch), status: outcome.status},
'Cache purge endpoint rejected the batch, retrying each entry on its own',
);
const singles: Array<CachePurgeBatch> = [
...batch.exact.map((entry) => ({exact: [entry], prefix: []})),
...batch.prefix.map((entry) => ({exact: [], prefix: [entry]})),
];
for (const [index, single] of singles.entries()) {
if (Date.now() - startedAt >= RUN_DEADLINE_MS) {
const untried = untriedFrom(singles, index);
await queue.requeue(untried);
Logger.warn(
describeBatch(adapterName, untried),
'Cache purge run reached its deadline, requeued untried entries',
);
return;
}
let singleOutcome: CachePurgeOutcome;
try {
singleOutcome = await adapter.purge(single);
if (singleOutcome.kind === 'invalid_entries') {
await queue.reject(single);
}
} catch (error) {
await queue.requeue(untriedFrom(singles, index));
throw error;
}
if (singleOutcome.kind === 'failed') {
const untried = untriedFrom(singles, index);
await queue.requeue(untried);
logFailure(adapterName, untried, singleOutcome);
return;
}
}
};
export default processCachePurgeQueue;
fluxer_api/src/api/worker/tests/ProcessAssetDeletionQueue.test.ts
+1 -1
View File
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {AssetDeletionQueue} from '@app/api/infrastructure/AssetDeletionQueue';
import {NoopPurgeQueue} from '@app/api/infrastructure/BunnyPurgeQueue';
import {NoopPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import {MockKVProvider} from '@app/api/test/mocks/MockKVProvider';
import {NoopLogger} from '@app/api/test/mocks/NoopLogger';
fluxer_api/src/api/worker/tests/ProcessCachePurgeQueue.test.ts
+315
View File
@@ -0,0 +1,315 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Config} from '@app/api/Config';
import type {APICachePurgeConfig} from '@app/api/config/APIConfig';
import {CachePurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import {MockKVProvider} from '@app/api/test/mocks/MockKVProvider';
import {NoopLogger} from '@app/api/test/mocks/NoopLogger';
import {server} from '@app/api/test/msw/server';
import processCachePurgeQueue from '@app/api/worker/tasks/ProcessCachePurgeQueue';
import {clearWorkerDependencies, setWorkerDependenciesForTest} from '@app/api/worker/WorkerContext';
import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
import {delay, HttpResponse, http} from 'msw';
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
const HELPERS = {logger: new NoopLogger()} as unknown as WorkerTaskHelpers;
const ENDPOINT = 'https://cache-purge.test/purge';
const MEDIA = 'https://media.test';
const EXACT_KEY = 'cache_purge:exact';
const PREFIX_KEY = 'cache_purge:prefix';
const REJECTED_KEY = 'cache_purge:rejected';
interface PurgeBody {
exact: Array<string>;
prefix: Array<string>;
}
interface RecordedRequest {
authorization: string | null;
contentType: string | null;
body: PurgeBody;
}
function createHarness() {
const kvClient = new MockKVProvider();
setWorkerDependenciesForTest({kvClient});
return {kvClient, queue: new CachePurgeQueue(kvClient)};
}
function recordPurges(respond: (body: PurgeBody) => Response | Promise<Response>): Array<RecordedRequest> {
const requests: Array<RecordedRequest> = [];
server.use(
http.post(ENDPOINT, async ({request}) => {
const body = (await request.json()) as PurgeBody;
requests.push({
authorization: request.headers.get('authorization'),
contentType: request.headers.get('content-type'),
body,
});
return respond(body);
}),
);
return requests;
}
function entryCount(body: PurgeBody): number {
return body.exact.length + body.prefix.length;
}
async function members(kvClient: MockKVProvider, key: string): Promise<Array<string>> {
return (await kvClient.smembers(key)).sort();
}
function sorted(values: Array<string>): Array<string> {
return [...values].sort();
}
describe('processCachePurgeQueue', () => {
let previousCachePurge: APICachePurgeConfig;
beforeEach(() => {
previousCachePurge = {adapter: Config.cachePurge.adapter, http: Config.cachePurge.http};
Config.cachePurge.adapter = 'http';
Config.cachePurge.http = {endpoint: ENDPOINT, token: 'test-token', timeoutMs: 50};
});
afterEach(() => {
Config.cachePurge.adapter = previousCachePurge.adapter;
Config.cachePurge.http = previousCachePurge.http;
clearWorkerDependencies();
vi.useRealTimers();
});
it('sends queued exact and prefix entries in one request and drains the queue', async () => {
const {kvClient, queue} = createHarness();
const exact = [`${MEDIA}/avatars/1/a_abc`, `${MEDIA}/emojis/9.webp`, `${MEDIA}/attachments/1/2/ação.png`];
const prefix = [`${MEDIA}/some/dir/`];
await queue.addUrls([...exact, ...prefix]);
const requests = recordPurges(() => new HttpResponse(null, {status: 204}));
await processCachePurgeQueue({}, HELPERS);
expect(requests).toHaveLength(1);
expect(requests[0]!.authorization).toBe('Bearer test-token');
expect(requests[0]!.contentType).toBe('application/json');
expect(sorted(requests[0]!.body.exact)).toEqual(sorted(exact));
expect(requests[0]!.body.prefix).toEqual(prefix);
expect(await members(kvClient, EXACT_KEY)).toEqual([]);
expect(await members(kvClient, PREFIX_KEY)).toEqual([]);
});
it('strips a trailing asterisk from a prefix entry and keeps a trailing slash', async () => {
const {kvClient, queue} = createHarness();
await queue.addUrls([`${MEDIA}/stickers/1**`, `${MEDIA}/emojis/`]);
const requests = recordPurges(() => new HttpResponse(null, {status: 204}));
await processCachePurgeQueue({}, HELPERS);
expect(requests).toHaveLength(1);
expect(requests[0]!.body.exact).toEqual([]);
expect(sorted(requests[0]!.body.prefix)).toEqual([`${MEDIA}/emojis/`, `${MEDIA}/stickers/1*`]);
expect(await members(kvClient, PREFIX_KEY)).toEqual([]);
});
it('sends no authorization header when no token is configured', async () => {
Config.cachePurge.http.token = '';
const {queue} = createHarness();
await queue.addUrls([`${MEDIA}/avatars/1/abc`]);
const requests = recordPurges(() => new HttpResponse(null, {status: 204}));
await processCachePurgeQueue({}, HELPERS);
expect(requests).toHaveLength(1);
expect(requests[0]!.authorization).toBeNull();
});
it('requeues the whole batch after a server error', async () => {
const {kvClient, queue} = createHarness();
const exact = [`${MEDIA}/avatars/1/abc`, `${MEDIA}/banners/1/def`];
const prefix = [`${MEDIA}/some/dir/`];
await queue.addUrls([...exact, ...prefix]);
const requests = recordPurges(() => new HttpResponse(null, {status: 503}));
await processCachePurgeQueue({}, HELPERS);
expect(requests).toHaveLength(1);
expect(await members(kvClient, EXACT_KEY)).toEqual(sorted(exact));
expect(await members(kvClient, PREFIX_KEY)).toEqual(prefix);
});
it('requeues the batch when the endpoint does not answer in time', async () => {
const {kvClient, queue} = createHarness();
const exact = [`${MEDIA}/avatars/1/abc`, `${MEDIA}/banners/1/def`];
await queue.addUrls(exact);
const requests = recordPurges(async () => {
await delay('infinite');
return new HttpResponse(null, {status: 204});
});
await processCachePurgeQueue({}, HELPERS);
expect(requests).toHaveLength(1);
expect(await members(kvClient, EXACT_KEY)).toEqual(sorted(exact));
});
it('requeues the batch when the endpoint is unreachable', async () => {
const {kvClient, queue} = createHarness();
const exact = [`${MEDIA}/avatars/1/abc`, `${MEDIA}/banners/1/def`];
await queue.addUrls(exact);
const requests = recordPurges(() => HttpResponse.error());
await processCachePurgeQueue({}, HELPERS);
expect(requests).toHaveLength(1);
expect(await members(kvClient, EXACT_KEY)).toEqual(sorted(exact));
});
it('requeues the batch on a redirect without following it', async () => {
const {kvClient, queue} = createHarness();
const exact = [`${MEDIA}/avatars/1/abc`, `${MEDIA}/banners/1/def`];
await queue.addUrls(exact);
const redirectTarget = 'https://cache-purge.test/elsewhere';
const redirectedRequests: Array<string> = [];
server.use(
http.all(redirectTarget, ({request}) => {
redirectedRequests.push(request.method);
return new HttpResponse(null, {status: 204});
}),
);
const requests = recordPurges(() => new HttpResponse(null, {status: 302, headers: {Location: redirectTarget}}));
await processCachePurgeQueue({}, HELPERS);
expect(requests).toHaveLength(1);
expect(redirectedRequests).toEqual([]);
expect(await members(kvClient, EXACT_KEY)).toEqual(sorted(exact));
});
it('requeues the batch on an authorisation failure without splitting it', async () => {
const {kvClient, queue} = createHarness();
const exact = [`${MEDIA}/avatars/1/abc`, `${MEDIA}/banners/1/def`, `${MEDIA}/emojis/9.webp`];
await queue.addUrls(exact);
const requests = recordPurges(() => new HttpResponse(null, {status: 401}));
await processCachePurgeQueue({}, HELPERS);
expect(requests).toHaveLength(1);
expect(await members(kvClient, EXACT_KEY)).toEqual(sorted(exact));
expect(await members(kvClient, REJECTED_KEY)).toEqual([]);
});
it('sets aside only the entry the endpoint rejects on its own', async () => {
const {kvClient, queue} = createHarness();
const poison = `${MEDIA}/attachments/1/2/poison.png`;
await queue.addUrls([`${MEDIA}/avatars/1/abc`, poison, `${MEDIA}/emojis/9.webp`]);
const requests = recordPurges((body) =>
body.exact.includes(poison) ? new HttpResponse(null, {status: 422}) : new HttpResponse(null, {status: 204}),
);
await processCachePurgeQueue({}, HELPERS);
expect(requests.map((request) => entryCount(request.body))).toEqual([3, 1, 1, 1]);
expect(await members(kvClient, REJECTED_KEY)).toEqual([poison]);
expect(await members(kvClient, EXACT_KEY)).toEqual([]);
});
it('splits the batch when the endpoint answers 400', async () => {
const {kvClient, queue} = createHarness();
const poison = `${MEDIA}/attachments/1/2/poison.png`;
await queue.addUrls([`${MEDIA}/avatars/1/abc`, poison, `${MEDIA}/emojis/9.webp`]);
const requests = recordPurges((body) =>
body.exact.includes(poison) ? new HttpResponse(null, {status: 400}) : new HttpResponse(null, {status: 204}),
);
await processCachePurgeQueue({}, HELPERS);
expect(requests.map((request) => entryCount(request.body))).toEqual([3, 1, 1, 1]);
expect(await members(kvClient, REJECTED_KEY)).toEqual([poison]);
expect(await members(kvClient, EXACT_KEY)).toEqual([]);
});
it('keeps untried exact and prefix entries queued when a single-entry retry fails', async () => {
const {kvClient, queue} = createHarness();
const first = `${MEDIA}/avatars/1/abc`;
const second = `${MEDIA}/banners/1/def`;
const third = `${MEDIA}/emojis/9.webp`;
const directory = `${MEDIA}/some/dir/`;
await queue.addUrls([first, second, third, directory]);
let singleRequests = 0;
const requests = recordPurges((body) => {
if (entryCount(body) > 1) {
return new HttpResponse(null, {status: 422});
}
singleRequests++;
return new HttpResponse(null, {status: singleRequests === 1 ? 204 : 503});
});
await processCachePurgeQueue({}, HELPERS);
expect(requests.map((request) => request.body.exact)).toEqual([[first, second, third], [first], [second]]);
expect(await members(kvClient, EXACT_KEY)).toEqual(sorted([second, third]));
expect(await members(kvClient, PREFIX_KEY)).toEqual([directory]);
expect(await members(kvClient, REJECTED_KEY)).toEqual([]);
});
it('requeues untried entries when the fallback runs out of time', async () => {
vi.useFakeTimers({toFake: ['Date']});
vi.setSystemTime(new Date('2026-09-13T00:00:00.000Z'));
const {kvClient, queue} = createHarness();
const entries = [1, 2, 3, 4, 5].map((index) => `${MEDIA}/avatars/${index}/abc`);
await queue.addUrls(entries);
const requests = recordPurges((body) => {
if (entryCount(body) > 1) {
return new HttpResponse(null, {status: 422});
}
vi.setSystemTime(Date.now() + 4_000);
return new HttpResponse(null, {status: 204});
});
await processCachePurgeQueue({}, HELPERS);
expect(requests.map((request) => request.body.exact)).toEqual([entries, [entries[0]], [entries[1]], [entries[2]]]);
expect(await members(kvClient, EXACT_KEY)).toEqual(sorted([entries[3]!, entries[4]!]));
expect(await members(kvClient, REJECTED_KEY)).toEqual([]);
});
it('leaves the queue untouched when the adapter is none', async () => {
Config.cachePurge.adapter = 'none';
const {kvClient, queue} = createHarness();
const exact = [`${MEDIA}/avatars/1/abc`];
const prefix = [`${MEDIA}/some/dir/`];
await queue.addUrls([...exact, ...prefix]);
const requests = recordPurges(() => new HttpResponse(null, {status: 204}));
await processCachePurgeQueue({}, HELPERS);
expect(requests).toEqual([]);
expect(await members(kvClient, EXACT_KEY)).toEqual(exact);
expect(await members(kvClient, PREFIX_KEY)).toEqual(prefix);
expect(await kvClient.get('cache_purge:budget:exact')).toBeNull();
expect(await kvClient.get('cache_purge:budget:prefix')).toBeNull();
});
it('sends no more than the token bucket allows and refills at the configured rate', async () => {
vi.useFakeTimers({toFake: ['Date']});
vi.setSystemTime(new Date('2026-09-13T00:00:00.000Z'));
const {kvClient, queue} = createHarness();
await queue.addUrls([
...Array.from({length: 200}, (_, index) => `${MEDIA}/avatars/${index}/abc`),
...Array.from({length: 30}, (_, index) => `${MEDIA}/dirs/${index}/`),
]);
const requests = recordPurges(() => new HttpResponse(null, {status: 204}));
await processCachePurgeQueue({}, HELPERS);
await processCachePurgeQueue({}, HELPERS);
vi.setSystemTime(Date.now() + 10_000);
await processCachePurgeQueue({}, HELPERS);
expect(requests.map((request) => [request.body.exact.length, request.body.prefix.length])).toEqual([
[120, 20],
[50, 5],
]);
expect(await kvClient.scard(EXACT_KEY)).toBe(30);
expect(await kvClient.scard(PREFIX_KEY)).toBe(5);
});
});
fluxer_docs/src/content/docs/operator/configuration.mdx
+11 -19
View File
@@ -1187,17 +1187,21 @@ Default empty. The Klipy GIF provider key. GIF search reports itself unavailable
Default empty. The YouTube Data API key. Also read by `unfurl`, which accepts `YOUTUBE_API_KEY` as a fallback.
#### `FLUXER_BUNNY_PURGE_ENABLED`
#### `FLUXER_CACHE_PURGE_ADAPTER`
Default `false`. Bunny CDN cache purging. Needs `FLUXER_BUNNY_API_KEY` and `FLUXER_BUNNY_PULL_ZONE_ID`.
Default `none`. How cached copies of deleted or replaced media are purged. `none` or `http`. Anything else fails startup. With `none`, nothing is queued or sent.
#### `FLUXER_BUNNY_API_KEY`
#### `FLUXER_CACHE_PURGE_HTTP_ENDPOINT`
Default empty. The Bunny API key. Paired with the pull zone.
Default empty. Required when the adapter is `http`. Must be an absolute `http` or `https` URL without credentials, or startup fails. While purges are queued, the worker posts JSON `{"exact": [...], "prefix": [...]}` here every 10 seconds. Redirects are not followed. Any 2xx marks the batch done. The endpoint purges every proxy that caches media. A `400` or `422` makes the worker resend each URL alone, and a URL refused alone moves to the `cache_purge:rejected` key. Any other failure is retried on a later run.
#### `FLUXER_BUNNY_PULL_ZONE_ID`
#### `FLUXER_CACHE_PURGE_HTTP_TOKEN`
Default `0`. The Bunny pull zone. Integer.
Default empty. The bearer token sent in the `Authorization` header of each purge request. Empty sends no header. When the adapter is `http`, a token with spaces, line breaks or non-ASCII characters fails startup.
#### `FLUXER_CACHE_PURGE_HTTP_TIMEOUT_MS`
Default `10000`. Purge request timeout. Accepts 1000 to 10000. The range is checked only when the adapter is `http`.
#### `FLUXER_GEOIP_DB_PATH`
@@ -1397,18 +1401,6 @@ Default empty. An external NSFW classifier. No such service ships with the stack
Default `0.85`. The media-side NSFW cutoff. Accepts 0.0 to 1.0 and finite. Distinct from the API threshold.
#### `FLUXER_MEDIA_PROXY_BUNNY_IP_GATE_ENABLED`
Default `false`. Restricts reads to Bunny edge IPs.
#### `FLUXER_MEDIA_PROXY_BUNNY_IP_GATE_TRUSTED_PROXIES`
Default empty. Extra trusted addresses for that gate. Every entry must parse as an IP or the process exits.
#### `FLUXER_MEDIA_PROXY_BUNNY_IP_GATE_REFRESH_SECS`
Default `3600`. How often the edge IP list refreshes. Accepts 60 to 86400.
`media-proxy` range-checks these values at startup and then reads them nowhere: `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_BYTES` and `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_TOTAL_BYTES`. A bad value still fails the boot.
## Gateway settings
@@ -2004,7 +1996,7 @@ CORS origins are exactly the app and marketing endpoints. Serving the client fro
| edge-config | The edge's own state | No |
| meilisearch-data | The search index, rebuildable | No |
Losing `valkey-data` can delay scheduled account and bulk-message deletions while their queues are rebuilt. Pending asset deletions and CDN purges can be lost, so do not treat this volume as disposable cache.
Losing `valkey-data` can delay scheduled account and bulk-message deletions while their queues are rebuilt. Pending asset deletions and cache purges can be lost, so do not treat this volume as disposable cache.
`nats-data` retains pending jobs in `JOBS` for up to 7 days and failed jobs in `JOBS_DLQ` for up to 30 days. A full jobs stream rejects new work. A full dead-letter stream drops its oldest entries, so investigate failures promptly. If dead-letter storage is unavailable, failed jobs remain in `JOBS` only until they expire. Losing this volume loses queued work, which is not automatically recovered from the database.
fluxer_media_proxy/src/bunny_ip_gate.rs
-469
View File
@@ -1,469 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use axum::{
body::Body,
extract::{ConnectInfo, State},
http::{Request, StatusCode},
middleware::Next,
response::{IntoResponse, Response},
};
use parking_lot::RwLock;
use std::{
collections::HashSet,
net::{IpAddr, SocketAddr},
sync::Arc,
time::Duration,
};
use tracing::{info, warn};
pub const BUNNY_IPV4_URL: &str = "https://api.bunny.net/system/edgeserverlist";
pub const BUNNY_IPV6_URL: &str = "https://api.bunny.net/system/edgeserverlist/IPv6";
#[derive(Clone, Debug, Default)]
pub struct Allowlist {
ips: Arc<HashSet<IpAddr>>,
}
impl Allowlist {
fn is_empty(&self) -> bool {
self.ips.is_empty()
}
fn len(&self) -> usize {
self.ips.len()
}
pub fn contains(&self, ip: &IpAddr) -> bool {
self.ips.contains(ip)
}
#[cfg(test)]
pub fn from_ips<I: IntoIterator<Item = IpAddr>>(iter: I) -> Self {
Self {
ips: Arc::new(iter.into_iter().collect()),
}
}
}
pub struct BunnyIpGate {
inner: RwLock<Allowlist>,
trusted_proxies: HashSet<IpAddr>,
client: reqwest::Client,
ipv4_url: String,
ipv6_url: String,
}
impl BunnyIpGate {
pub fn new(client: reqwest::Client, trusted_proxies: Vec<IpAddr>) -> Self {
Self::with_urls(
client,
trusted_proxies,
BUNNY_IPV4_URL.to_owned(),
BUNNY_IPV6_URL.to_owned(),
)
}
pub fn with_urls(
client: reqwest::Client,
trusted_proxies: Vec<IpAddr>,
ipv4_url: String,
ipv6_url: String,
) -> Self {
Self {
inner: RwLock::new(Allowlist::default()),
trusted_proxies: trusted_proxies.into_iter().collect(),
client,
ipv4_url,
ipv6_url,
}
}
pub fn snapshot(&self) -> Allowlist {
self.inner.read().clone()
}
pub async fn refresh_once(&self) -> anyhow::Result<usize> {
let allow = fetch_allowlist(&self.client, &self.ipv4_url, &self.ipv6_url).await?;
let len = allow.len();
*self.inner.write() = allow;
Ok(len)
}
pub fn spawn_background_refresher(self: Arc<Self>, interval: Duration) {
tokio::spawn(async move {
let mut ticker = tokio::time::interval(interval);
ticker.tick().await;
loop {
ticker.tick().await;
match self.refresh_once().await {
Ok(count) => info!(count, "bunny ip allowlist refreshed"),
Err(err) => warn!(
error = %err,
"bunny ip allowlist refresh failed; serving previous list"
),
}
}
});
}
fn is_trusted_proxy(&self, ip: &IpAddr) -> bool {
self.trusted_proxies.contains(ip)
}
#[cfg(test)]
pub fn install_for_test(&self, allow: Allowlist) {
*self.inner.write() = allow;
}
}
async fn fetch_allowlist(
client: &reqwest::Client,
ipv4_url: &str,
ipv6_url: &str,
) -> anyhow::Result<Allowlist> {
let (v4, v6) = tokio::join!(
fetch_ip_list(client, ipv4_url),
fetch_ip_list(client, ipv6_url)
);
let v4 = v4?;
let v6 = v6?;
let mut set: HashSet<IpAddr> = HashSet::with_capacity(v4.len() + v6.len());
set.extend(v4);
set.extend(v6);
anyhow::ensure!(
!set.is_empty(),
"bunny returned an empty edge ip list from {ipv4_url} + {ipv6_url}"
);
Ok(Allowlist { ips: Arc::new(set) })
}
async fn fetch_ip_list(client: &reqwest::Client, url: &str) -> anyhow::Result<Vec<IpAddr>> {
let raw: Vec<String> = client
.get(url)
.send()
.await?
.error_for_status()?
.json()
.await?;
Ok(parse_ip_list(&raw, url))
}
fn parse_ip_list(raw: &[String], url: &str) -> Vec<IpAddr> {
let mut out = Vec::with_capacity(raw.len());
for entry in raw {
match entry.trim().parse::<IpAddr>() {
Ok(ip) => out.push(ip),
Err(_) => warn!(value = %entry, url, "skipping unparseable bunny ip"),
}
}
out
}
pub async fn gate_middleware(
State(gate): State<Arc<BunnyIpGate>>,
ConnectInfo(peer): ConnectInfo<SocketAddr>,
request: Request<Body>,
next: Next,
) -> Response {
if is_exempt_path(request.uri().path()) {
return next.run(request).await;
}
let allow = gate.snapshot();
if allow.is_empty() {
warn!("bunny ip allowlist is empty; refusing public request");
return (
StatusCode::SERVICE_UNAVAILABLE,
"bunny allowlist not loaded",
)
.into_response();
}
let client_ip = resolve_client_ip(&gate, &peer, request.headers());
if !allow.contains(&client_ip) {
warn!(%client_ip, path = request.uri().path(), "rejecting non-bunny origin");
return (StatusCode::FORBIDDEN, "origin not in bunny allowlist").into_response();
}
next.run(request).await
}
fn is_exempt_path(path: &str) -> bool {
matches!(
path,
"/_health" | "/_metrics" | "/_metadata" | "/_thumbnail" | "/_frames"
) || path.starts_with("/v1/relay/")
}
fn resolve_client_ip(
gate: &BunnyIpGate,
peer: &SocketAddr,
headers: &axum::http::HeaderMap,
) -> IpAddr {
let peer_ip = peer.ip();
if !gate.is_trusted_proxy(&peer_ip) {
return peer_ip;
}
if let Some(xff) = headers.get("x-forwarded-for").and_then(|v| v.to_str().ok()) {
for hop in xff.split(',').rev() {
if let Ok(ip) = hop.trim().parse::<IpAddr>()
&& !gate.is_trusted_proxy(&ip)
{
return ip;
}
}
}
peer_ip
}
pub fn build_refresh_client() -> reqwest::Result<reqwest::Client> {
reqwest::Client::builder()
.connect_timeout(Duration::from_secs(5))
.timeout(Duration::from_secs(20))
.user_agent(crate::constants::OUTBOUND_USER_AGENT)
.build()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::test_fixtures::ADVERSARIAL_TEXT_INPUTS;
use axum::{Router, middleware, routing::get};
use std::net::{Ipv4Addr, Ipv6Addr};
use tower::ServiceExt;
fn ip4(a: u8, b: u8, c: u8, d: u8) -> IpAddr {
IpAddr::V4(Ipv4Addr::new(a, b, c, d))
}
fn sock(ip: IpAddr) -> SocketAddr {
SocketAddr::new(ip, 0)
}
#[test]
fn parse_ip_list_skips_garbage_keeps_good() {
let raw = vec![
"1.2.3.4".to_owned(),
"not-an-ip".to_owned(),
" 5.6.7.8 ".to_owned(),
"2a01:4f8::1".to_owned(),
"".to_owned(),
];
let parsed = parse_ip_list(&raw, "test");
assert_eq!(parsed.len(), 3);
assert!(parsed.contains(&ip4(1, 2, 3, 4)));
assert!(parsed.contains(&ip4(5, 6, 7, 8)));
assert!(parsed.contains(&IpAddr::V6("2a01:4f8::1".parse::<Ipv6Addr>().unwrap())));
}
#[test]
fn untrusted_peer_xff_is_ignored() {
let gate = BunnyIpGate::new(reqwest::Client::new(), vec![ip4(10, 0, 0, 1)]);
let mut headers = axum::http::HeaderMap::new();
headers.insert("x-forwarded-for", "9.9.9.9".parse().unwrap());
let resolved = resolve_client_ip(&gate, &sock(ip4(8, 8, 8, 8)), &headers);
assert_eq!(resolved, ip4(8, 8, 8, 8));
}
#[test]
fn trusted_peer_uses_rightmost_untrusted_xff() {
let gate = BunnyIpGate::new(
reqwest::Client::new(),
vec![ip4(10, 0, 0, 1), ip4(10, 0, 0, 2)],
);
let mut headers = axum::http::HeaderMap::new();
headers.insert(
"x-forwarded-for",
"89.187.188.227, 10.0.0.2, 10.0.0.1".parse().unwrap(),
);
let resolved = resolve_client_ip(&gate, &sock(ip4(10, 0, 0, 1)), &headers);
assert_eq!(resolved, ip4(89, 187, 188, 227));
}
#[test]
fn trusted_peer_falls_back_to_peer_when_no_xff() {
let gate = BunnyIpGate::new(reqwest::Client::new(), vec![ip4(10, 0, 0, 1)]);
let headers = axum::http::HeaderMap::new();
let resolved = resolve_client_ip(&gate, &sock(ip4(10, 0, 0, 1)), &headers);
assert_eq!(resolved, ip4(10, 0, 0, 1));
}
#[test]
fn exempt_paths_cover_internal_routes() {
assert!(is_exempt_path("/_health"));
assert!(is_exempt_path("/_metrics"));
assert!(is_exempt_path("/_metadata"));
assert!(is_exempt_path("/_thumbnail"));
assert!(is_exempt_path("/_frames"));
assert!(is_exempt_path("/v1/relay/abc/def"));
assert!(!is_exempt_path("/external/some/url"));
assert!(!is_exempt_path("/some/asset.png"));
assert!(!is_exempt_path("/"));
}
#[test]
fn snapshot_lookup_matches_loaded_ips() {
let gate = BunnyIpGate::new(reqwest::Client::new(), vec![]);
gate.install_for_test(Allowlist::from_ips([ip4(89, 187, 188, 227)]));
let snap = gate.snapshot();
assert!(snap.contains(&ip4(89, 187, 188, 227)));
assert!(!snap.contains(&ip4(1, 1, 1, 1)));
assert_eq!(snap.len(), 1);
}
fn ip6(text: &str) -> IpAddr {
IpAddr::V6(text.parse::<Ipv6Addr>().unwrap())
}
fn forwarded_for(gate: &BunnyIpGate, peer: IpAddr, xff: &str) -> IpAddr {
let mut headers = axum::http::HeaderMap::new();
headers.insert("x-forwarded-for", xff.parse().unwrap());
resolve_client_ip(gate, &sock(peer), &headers)
}
#[test]
fn a_long_forwarded_chain_still_resolves_the_rightmost_untrusted_hop() {
let trusted: Vec<IpAddr> = (1..=24).map(|last| ip4(10, 0, 0, last)).collect();
let gate = BunnyIpGate::new(reqwest::Client::new(), trusted.clone());
let mut chain = vec!["89.187.188.227".to_owned()];
chain.extend(trusted.iter().map(ToString::to_string));
assert_eq!(25, chain.len());
assert_eq!(
ip4(89, 187, 188, 227),
forwarded_for(&gate, ip4(10, 0, 0, 24), &chain.join(", "))
);
assert_eq!(
ip4(10, 0, 0, 24),
forwarded_for(&gate, ip4(10, 0, 0, 24), &chain[1..].join(", "))
);
}
#[test]
fn unparseable_hops_are_skipped_instead_of_ending_the_scan() {
let gate = BunnyIpGate::new(
reqwest::Client::new(),
vec![ip4(10, 0, 0, 1), ip4(10, 0, 0, 2)],
);
assert_eq!(
ip4(89, 187, 188, 227),
forwarded_for(
&gate,
ip4(10, 0, 0, 1),
"89.187.188.227, unknown, 10.0.0.2, 10.0.0.1"
)
);
assert_eq!(
ip4(89, 187, 188, 227),
forwarded_for(
&gate,
ip4(10, 0, 0, 1),
"89.187.188.227, 10.0.0.1, , not-an-ip"
)
);
assert_eq!(
ip4(10, 0, 0, 1),
forwarded_for(&gate, ip4(10, 0, 0, 1), "unknown, also-unknown")
);
}
#[test]
fn forwarded_ipv6_hops_are_only_accepted_in_their_bare_form() {
let gate = BunnyIpGate::new(reqwest::Client::new(), vec![ip4(10, 0, 0, 1)]);
assert_eq!(
ip6("2a01:4f8::1"),
forwarded_for(&gate, ip4(10, 0, 0, 1), "2a01:4f8::1, 10.0.0.1")
);
assert_eq!(
ip4(10, 0, 0, 1),
forwarded_for(&gate, ip4(10, 0, 0, 1), "[2a01:4f8::1]:443, 10.0.0.1")
);
assert_eq!(
ip6("2a01:4f8::2"),
forwarded_for(
&gate,
ip4(10, 0, 0, 1),
"2a01:4f8::2, [2a01:4f8::1]:443, 10.0.0.1"
)
);
}
#[test]
fn a_trusted_ipv6_proxy_is_skipped_like_any_other_hop() {
let gate = BunnyIpGate::new(reqwest::Client::new(), vec![ip6("2a01:4f8::1")]);
assert_eq!(
ip4(89, 187, 188, 227),
forwarded_for(&gate, ip6("2a01:4f8::1"), "89.187.188.227, 2a01:4f8::1")
);
}
#[test]
fn adversarial_paths_are_never_exempt_from_the_bunny_gate() {
for text in ADVERSARIAL_TEXT_INPUTS {
for path in [
(*text).to_owned(),
format!("/{text}"),
format!("/avatars/1/{text}.png"),
format!("/external/{text}"),
format!("/_health{text}"),
format!("/v1/relay{text}"),
format!("/v1/relay/{text}"),
] {
if !is_exempt_path(&path) {
continue;
}
assert!(
path == "/_health" || path.starts_with("/v1/relay/"),
"{path:?} is exempt from the bunny gate"
);
}
}
}
#[tokio::test]
async fn gate_after_successful_refresh_never_returns_503() {
let gate = Arc::new(BunnyIpGate::new(reqwest::Client::new(), vec![]));
gate.install_for_test(Allowlist::from_ips([ip4(89, 187, 188, 227)]));
let app = Router::new()
.route("/asset.png", get(|| async { "ok" }))
.layer(middleware::from_fn_with_state(gate, gate_middleware));
let allowed = app
.clone()
.oneshot(
Request::builder()
.uri("/asset.png")
.extension(ConnectInfo(sock(ip4(89, 187, 188, 227))))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(StatusCode::OK, allowed.status());
let foreign = app
.oneshot(
Request::builder()
.uri("/asset.png")
.extension(ConnectInfo(sock(ip4(1, 1, 1, 1))))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(StatusCode::FORBIDDEN, foreign.status());
}
#[tokio::test]
async fn an_empty_allowlist_refuses_a_public_request() {
let gate = Arc::new(BunnyIpGate::new(reqwest::Client::new(), vec![]));
let app = Router::new()
.route("/asset.png", get(|| async { "ok" }))
.layer(middleware::from_fn_with_state(gate, gate_middleware));
let refused = app
.oneshot(
Request::builder()
.uri("/asset.png")
.extension(ConnectInfo(sock(ip4(89, 187, 188, 227))))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(StatusCode::SERVICE_UNAVAILABLE, refused.status());
}
}
fluxer_media_proxy/src/config/mod.rs
+3 -22
View File
@@ -8,10 +8,10 @@ use crate::constants;
use crate::secret::{SecretBytes, SecretString};
use parse::{
EnvMap, decode_upload_relay_secret, default_native_transform_concurrency, non_empty,
parse_bool, parse_bucket_style, parse_f32, parse_ip_list_env, parse_mode_env,
parse_storage_backend, parse_u16, parse_u64, parse_usize, validate_read_endpoint,
parse_bool, parse_bucket_style, parse_f32, parse_mode_env, parse_storage_backend, parse_u16,
parse_u64, parse_usize, validate_read_endpoint,
};
use std::{env, net::IpAddr, path::PathBuf};
use std::{env, path::PathBuf};
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum StorageBackend {
@@ -92,9 +92,6 @@ pub struct Config {
pub storage: StorageConfig,
pub media: MediaServingConfig,
pub upload_relay: UploadRelayConfig,
pub bunny_ip_gate_enabled: bool,
pub bunny_ip_gate_trusted_proxies: Vec<IpAddr>,
pub bunny_ip_gate_refresh_secs: u64,
}
impl Config {
@@ -170,22 +167,6 @@ impl Config {
storage: StorageConfig::load(&env)?,
media: MediaServingConfig::load(&env)?,
upload_relay: UploadRelayConfig::load(&env, mode)?,
bunny_ip_gate_enabled: parse_bool(
"FLUXER_MEDIA_PROXY_BUNNY_IP_GATE_ENABLED",
env.get("FLUXER_MEDIA_PROXY_BUNNY_IP_GATE_ENABLED"),
)?
.unwrap_or(false),
bunny_ip_gate_trusted_proxies: parse_ip_list_env(
"FLUXER_MEDIA_PROXY_BUNNY_IP_GATE_TRUSTED_PROXIES",
env.get("FLUXER_MEDIA_PROXY_BUNNY_IP_GATE_TRUSTED_PROXIES"),
)?,
bunny_ip_gate_refresh_secs: parse_u64(
"FLUXER_MEDIA_PROXY_BUNNY_IP_GATE_REFRESH_SECS",
env.get("FLUXER_MEDIA_PROXY_BUNNY_IP_GATE_REFRESH_SECS"),
3_600,
60,
24 * 60 * 60,
)?,
})
}
}
fluxer_media_proxy/src/config/parse.rs
-19
View File
@@ -3,7 +3,6 @@
use super::{BucketStyle, DeploymentMode, StorageBackend};
use crate::secret::SecretBytes;
use base64::{Engine as _, engine::general_purpose};
use std::net::IpAddr;
#[derive(Debug, Default)]
pub(super) struct EnvMap(Vec<(String, String)>);
@@ -213,24 +212,6 @@ where
Ok(parsed)
}
pub(super) fn parse_ip_list_env(var_name: &str, raw: Option<&str>) -> anyhow::Result<Vec<IpAddr>> {
let Some(raw) = raw.map(str::trim).filter(|s| !s.is_empty()) else {
return Ok(Vec::new());
};
let mut out = Vec::new();
for entry in raw.split(',') {
let entry = entry.trim();
if entry.is_empty() {
continue;
}
let ip = entry
.parse::<IpAddr>()
.map_err(|_| anyhow::anyhow!("{var_name} contains invalid IP: {entry}"))?;
out.push(ip);
}
Ok(out)
}
pub(super) fn default_native_transform_concurrency() -> usize {
std::thread::available_parallelism()
.map(usize::from)
fluxer_media_proxy/src/config/tests/mod.rs
+1 -32
View File
@@ -410,7 +410,7 @@ fn every_deployment_mode_and_storage_backend_variant_parses() {
}
#[test]
fn upload_relay_spool_and_bunny_ip_gate_keys_apply() {
fn upload_relay_spool_keys_apply() {
let cfg = Config::load_from_iter([
("FLUXER_MEDIA_PROXY_SECRET_KEY", "secret"),
(
@@ -425,12 +425,6 @@ fn upload_relay_spool_and_bunny_ip_gate_keys_apply() {
"FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_MAX_TOTAL_BYTES",
"1073741824",
),
("FLUXER_MEDIA_PROXY_BUNNY_IP_GATE_ENABLED", "yes"),
(
"FLUXER_MEDIA_PROXY_BUNNY_IP_GATE_TRUSTED_PROXIES",
"10.0.0.1, 2001:db8::1 ,",
),
("FLUXER_MEDIA_PROXY_BUNNY_IP_GATE_REFRESH_SECS", "900"),
])
.unwrap();
@@ -440,31 +434,6 @@ fn upload_relay_spool_and_bunny_ip_gate_keys_apply() {
);
assert_eq!(2 << 20, cfg.upload_relay.spool_chunk_bytes);
assert_eq!(1 << 30, cfg.upload_relay.spool_max_total_bytes);
assert!(cfg.bunny_ip_gate_enabled);
assert_eq!(
vec![
"10.0.0.1".parse::<IpAddr>().unwrap(),
"2001:db8::1".parse::<IpAddr>().unwrap(),
],
cfg.bunny_ip_gate_trusted_proxies
);
assert_eq!(900, cfg.bunny_ip_gate_refresh_secs);
}
#[test]
fn rejects_invalid_bunny_ip_gate_trusted_proxies() {
let err = Config::load_from_iter([
("FLUXER_MEDIA_PROXY_SECRET_KEY", "secret"),
(
"FLUXER_MEDIA_PROXY_BUNNY_IP_GATE_TRUSTED_PROXIES",
"10.0.0.1,not-an-ip",
),
])
.unwrap_err();
assert!(
err.to_string()
.contains("FLUXER_MEDIA_PROXY_BUNNY_IP_GATE_TRUSTED_PROXIES")
);
}
#[test]
fluxer_media_proxy/src/lib.rs
-1
View File
@@ -4,7 +4,6 @@ mod aggregate_error;
pub mod asset_hash;
mod asset_size;
pub mod aws_sigv4;
pub mod bunny_ip_gate;
mod byte_budget;
mod byte_cache;
pub mod cli;
fluxer_media_proxy/src/server/runtime.rs
+6 -51
View File
@@ -7,13 +7,7 @@ use super::{
routes,
state::AppState,
};
use crate::{
aggregate_error::aggregate_results,
bunny_ip_gate::{self, BunnyIpGate},
config::Config,
media_process, request_log,
};
use anyhow::Context as _;
use crate::{aggregate_error::aggregate_results, config::Config, media_process, request_log};
use axum::{
Router, middleware,
routing::{any, get, post, put},
@@ -26,7 +20,6 @@ pub async fn run(cfg: Config) -> anyhow::Result<()> {
media_process::warmup_vips()?;
let addr: SocketAddr = format!("{}:{}", cfg.bind_host, cfg.port).parse()?;
let state = Arc::new(AppState::try_new(cfg)?);
let bunny_gate = start_bunny_ip_gate(&state).await?;
if let Some(read_endpoint) = state.cfg.storage.s3_read_endpoint.as_deref() {
info!(
endpoint = read_endpoint,
@@ -37,7 +30,7 @@ pub async fn run(cfg: Config) -> anyhow::Result<()> {
);
}
let drain = HttpRequestDrain::new();
let app = build_router(Arc::clone(&state), bunny_gate, drain.clone());
let app = build_router(Arc::clone(&state), drain.clone());
let listener = TcpListener::bind(addr).await?;
info!(%addr, "media proxy listening");
axum::serve(
@@ -49,12 +42,8 @@ pub async fn run(cfg: Config) -> anyhow::Result<()> {
drain_and_shutdown(&state, &drain).await
}
fn build_router(
state: Arc<AppState>,
bunny_gate: Option<Arc<BunnyIpGate>>,
drain: HttpRequestDrain,
) -> Router {
let mut router = Router::new()
fn build_router(state: Arc<AppState>, drain: HttpRequestDrain) -> Router {
Router::new()
.route("/_health", get(routes::ops::health))
.route("/_metrics", get(routes::ops::metrics_handler))
.route("/_metadata", post(routes::internal::metadata_handler))
@@ -69,14 +58,7 @@ fn build_router(
.layer(middleware::from_fn_with_state(
state.metrics.request(),
request_log::trace,
));
if let Some(gate) = bunny_gate {
router = router.layer(middleware::from_fn_with_state(
gate,
bunny_ip_gate::gate_middleware,
));
}
router
))
.layer(middleware::from_fn_with_state(
state.cfg.mode,
add_security_header_middleware,
@@ -85,29 +67,6 @@ fn build_router(
.with_state(state)
}
async fn start_bunny_ip_gate(state: &Arc<AppState>) -> anyhow::Result<Option<Arc<BunnyIpGate>>> {
if !state.cfg.bunny_ip_gate_enabled {
return Ok(None);
}
let gate = Arc::new(BunnyIpGate::new(
bunny_ip_gate::build_refresh_client()?,
state.cfg.bunny_ip_gate_trusted_proxies.clone(),
));
let count = gate
.refresh_once()
.await
.context("initial bunny ip allowlist fetch failed")?;
info!(
count,
trusted_proxies = state.cfg.bunny_ip_gate_trusted_proxies.len(),
refresh_secs = state.cfg.bunny_ip_gate_refresh_secs,
"bunny ip gate enabled"
);
Arc::clone(&gate)
.spawn_background_refresher(Duration::from_secs(state.cfg.bunny_ip_gate_refresh_secs));
Ok(Some(gate))
}
async fn drain_and_shutdown(state: &Arc<AppState>, drain: &HttpRequestDrain) -> anyhow::Result<()> {
let grace_ms = state.cfg.shutdown_grace_ms;
let deadline = tokio::time::Instant::now() + Duration::from_millis(grace_ms);
@@ -177,11 +136,7 @@ mod tests {
fn test_router() -> Router {
let cfg = Config::load_from_iter([("FLUXER_MEDIA_PROXY_SECRET_KEY", "secret")])
.expect("test config");
build_router(
Arc::new(AppState::for_tests(cfg)),
None,
HttpRequestDrain::new(),
)
build_router(Arc::new(AppState::for_tests(cfg)), HttpRequestDrain::new())
}
async fn probe(path: &str, method: Method) -> axum::response::Response {
fluxer_media_proxy/src/storage/tests/mod.rs
-3
View File
@@ -75,9 +75,6 @@ fn test_config(root: &Path) -> Config {
spool_chunk_bytes: 64 * 1024,
spool_max_total_bytes: 1 << 30,
},
bunny_ip_gate_enabled: false,
bunny_ip_gate_trusted_proxies: Vec::new(),
bunny_ip_gate_refresh_secs: 3_600,
}
}
packages/config/src/ConfigLoader.ts
+31 -5
View File
@@ -11,7 +11,7 @@ import {
parsePublicOrigin,
parseWebOrigin,
} from '@fluxer/config/src/EndpointDerivation';
import type {MasterConfig} from '@fluxer/config/src/MasterConfig';
import {CACHE_PURGE_ADAPTER_NAMES, type MasterConfig} from '@fluxer/config/src/MasterConfig';
let cachedConfig: MasterConfig | null = null;
@@ -235,10 +235,13 @@ function defaultConfig(): MasterConfig {
youtube: {
api_key: '',
},
bunny: {
purge_enabled: false,
api_key: '',
pull_zone_id: 0,
cache_purge: {
adapter: 'none',
http: {
endpoint: '',
token: '',
timeout_ms: 10_000,
},
},
blocklist_feeds: {},
risk_integration: {
@@ -449,6 +452,27 @@ function validateApiWorkerConfig(config: MasterConfig): void {
}
}
function validateCachePurgeConfig(config: MasterConfig): void {
const cachePurge = config.integrations.cache_purge;
if (cachePurge.adapter !== 'http') {
return;
}
requireString(cachePurge.http.endpoint, 'FLUXER_CACHE_PURGE_HTTP_ENDPOINT');
const endpoint = URL.parse(cachePurge.http.endpoint);
if (
endpoint === null ||
(endpoint.protocol !== 'http:' && endpoint.protocol !== 'https:') ||
endpoint.username !== '' ||
endpoint.password !== ''
) {
throw new Error('FLUXER_CACHE_PURGE_HTTP_ENDPOINT must be an absolute http or https URL without credentials');
}
if (!/^[\x21-\x7e]*$/u.test(cachePurge.http.token)) {
throw new Error('FLUXER_CACHE_PURGE_HTTP_TOKEN must contain only visible ASCII characters');
}
assertIntegerInRange(cachePurge.http.timeout_ms, 'FLUXER_CACHE_PURGE_HTTP_TIMEOUT_MS', 1_000, 10_000);
}
function validateDomain(value: string, envName: string): void {
if (value === '') return;
const parsed = URL.parse(`http://${value}/`);
@@ -507,6 +531,7 @@ function normalizeConfig(config: MasterConfig): MasterConfig {
assertOneOf(config.integrations.email.provider, ['smtp', 'none'], 'FLUXER_EMAIL_PROVIDER');
assertOneOf(config.integrations.captcha.provider, ['hcaptcha', 'turnstile', 'none'], 'FLUXER_CAPTCHA_PROVIDER');
assertOneOf(config.integrations.search.engine, ['elasticsearch', 'meilisearch'], 'FLUXER_SEARCH_ENGINE');
assertOneOf(config.integrations.cache_purge.adapter, CACHE_PURGE_ADAPTER_NAMES, 'FLUXER_CACHE_PURGE_ADAPTER');
assertOneOf(
config.instance.abuse_policy.direct_contact_spam.action,
['flag_spammer', 'suppress_delivery'],
@@ -515,6 +540,7 @@ function normalizeConfig(config: MasterConfig): MasterConfig {
validatePostgresConfig(config);
validateCaptchaConfig(config);
validateApiWorkerConfig(config);
validateCachePurgeConfig(config);
assertIntegerInRange(config.services.api.max_inflight_requests, 'FLUXER_API_MAX_INFLIGHT_REQUESTS', 1, 100_000);
assertIntegerInRange(config.services.api.headers_timeout_ms, 'FLUXER_API_HEADERS_TIMEOUT_MS', 1_000, 3_600_000);
assertIntegerInRange(config.services.api.request_timeout_ms, 'FLUXER_API_REQUEST_TIMEOUT_MS', 1_000, 3_600_000);
packages/config/src/MasterConfig.ts
+9 -4
View File
@@ -5,6 +5,8 @@ import type {DerivedEndpoints} from '@fluxer/config/src/EndpointDerivation';
export type RuntimeEnv = 'development' | 'production' | 'test';
export type DatabaseBackend = 'postgres' | 'cassandra';
export type PublicScheme = 'http' | 'https';
export const CACHE_PURGE_ADAPTER_NAMES = ['none', 'http'] as const;
export type CachePurgeAdapterName = (typeof CACHE_PURGE_ADAPTER_NAMES)[number];
export interface InstanceBrandingConfig {
product_name: string;
@@ -274,10 +276,13 @@ export interface MasterConfig {
youtube: {
api_key: string;
};
bunny: {
purge_enabled: boolean;
api_key: string;
pull_zone_id: number;
cache_purge: {
adapter: CachePurgeAdapterName;
http: {
endpoint: string;
token: string;
timeout_ms: number;
};
};
blocklist_feeds: {
enabled?: boolean;
packages/config/src/__tests__/ConfigLoader.test.ts
+84
View File
@@ -513,6 +513,90 @@ describe('ConfigLoader', () => {
expect((await loadConfig()).integrations.captcha.enabled).toBe(false);
});
test('defaults the cache purge adapter to none', async () => {
stubMinimalEnv();
expect((await loadConfig()).integrations.cache_purge).toEqual({
adapter: 'none',
http: {endpoint: '', token: '', timeout_ms: 10_000},
});
});
test('reads the http cache purge settings from the environment', async () => {
stubMinimalEnv({
FLUXER_CACHE_PURGE_ADAPTER: 'http',
FLUXER_CACHE_PURGE_HTTP_ENDPOINT: 'https://purge.internal/purge',
FLUXER_CACHE_PURGE_HTTP_TOKEN: 'purge-token',
FLUXER_CACHE_PURGE_HTTP_TIMEOUT_MS: '5000',
});
expect((await loadConfig()).integrations.cache_purge).toEqual({
adapter: 'http',
http: {endpoint: 'https://purge.internal/purge', token: 'purge-token', timeout_ms: 5000},
});
});
test('rejects an unknown cache purge adapter', async () => {
stubMinimalEnv({FLUXER_CACHE_PURGE_ADAPTER: 'varnish'});
await expect(loadConfig()).rejects.toThrow('Invalid FLUXER_CACHE_PURGE_ADAPTER: varnish');
});
test('rejects the http cache purge adapter without an endpoint', async () => {
stubMinimalEnv({FLUXER_CACHE_PURGE_ADAPTER: 'http'});
await expect(loadConfig()).rejects.toThrow('FLUXER_CACHE_PURGE_HTTP_ENDPOINT is required');
});
test('rejects a cache purge endpoint that is not an absolute http URL', async () => {
for (const endpoint of ['/purge', 'purge.internal/purge', 'ftp://purge.internal/purge']) {
stubMinimalEnv({FLUXER_CACHE_PURGE_ADAPTER: 'http', FLUXER_CACHE_PURGE_HTTP_ENDPOINT: endpoint});
await expect(loadConfig()).rejects.toThrow(
'FLUXER_CACHE_PURGE_HTTP_ENDPOINT must be an absolute http or https URL without credentials',
);
}
});
test('rejects a cache purge endpoint that carries credentials', async () => {
stubMinimalEnv({
FLUXER_CACHE_PURGE_ADAPTER: 'http',
FLUXER_CACHE_PURGE_HTTP_ENDPOINT: 'https://purger:[email protected]/purge',
});
await expect(loadConfig()).rejects.toThrow(
'FLUXER_CACHE_PURGE_HTTP_ENDPOINT must be an absolute http or https URL without credentials',
);
});
test('rejects a cache purge timeout outside 1000 to 10000', async () => {
for (const timeout of ['999', '10001']) {
stubMinimalEnv({
FLUXER_CACHE_PURGE_ADAPTER: 'http',
FLUXER_CACHE_PURGE_HTTP_ENDPOINT: 'https://purge.internal/purge',
FLUXER_CACHE_PURGE_HTTP_TIMEOUT_MS: timeout,
});
await expect(loadConfig()).rejects.toThrow(
'FLUXER_CACHE_PURGE_HTTP_TIMEOUT_MS must be an integer between 1000 and 10000',
);
}
});
test('rejects a cache purge token with spaces or control characters without echoing it', async () => {
for (const token of ['secret\nvalue', 'secret\rvalue', 'secret value', 'secret-value\n']) {
stubMinimalEnv({
FLUXER_CACHE_PURGE_ADAPTER: 'http',
FLUXER_CACHE_PURGE_HTTP_ENDPOINT: 'https://purge.internal/purge',
FLUXER_CACHE_PURGE_HTTP_TOKEN: token,
});
await expect(loadConfig()).rejects.toThrow(
/^FLUXER_CACHE_PURGE_HTTP_TOKEN must contain only visible ASCII characters$/,
);
}
});
test('leaves the cache purge settings unvalidated when the adapter is none', async () => {
stubMinimalEnv({
FLUXER_CACHE_PURGE_HTTP_ENDPOINT: 'ftp://purge.internal/purge',
FLUXER_CACHE_PURGE_HTTP_TIMEOUT_MS: '1',
});
expect((await loadConfig()).integrations.cache_purge.adapter).toBe('none');
});
test('leaves Bluesky login off with no legal URLs by default', async () => {
stubMinimalEnv();
packages/config/src/config_loader/EnvironmentOverrides.ts
+7 -3
View File
@@ -318,10 +318,14 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
FLUXER_CLAMAV_FAIL_OPEN: {path: ['integrations', 'clamav', 'fail_open'], parse: parseBoolean},
FLUXER_KLIPY_API_KEY: {path: ['integrations', 'klipy', 'api_key']},
FLUXER_YOUTUBE_API_KEY: {path: ['integrations', 'youtube', 'api_key']},
FLUXER_BUNNY_PURGE_ENABLED: {path: ['integrations', 'bunny', 'purge_enabled'], parse: parseBoolean},
FLUXER_CACHE_PURGE_ADAPTER: {path: ['integrations', 'cache_purge', 'adapter']},
FLUXER_CACHE_PURGE_HTTP_ENDPOINT: {path: ['integrations', 'cache_purge', 'http', 'endpoint']},
FLUXER_CACHE_PURGE_HTTP_TOKEN: {path: ['integrations', 'cache_purge', 'http', 'token']},
FLUXER_CACHE_PURGE_HTTP_TIMEOUT_MS: {
path: ['integrations', 'cache_purge', 'http', 'timeout_ms'],
parse: parseInteger,
},
FLUXER_BLOCKLIST_FEEDS_ENABLED: {path: ['integrations', 'blocklist_feeds', 'enabled'], parse: parseBoolean},
FLUXER_BUNNY_API_KEY: {path: ['integrations', 'bunny', 'api_key']},
FLUXER_BUNNY_PULL_ZONE_ID: {path: ['integrations', 'bunny', 'pull_zone_id'], parse: parseInteger},
FLUXER_RISK_INTEGRATION_ENABLED: {path: ['integrations', 'risk_integration', 'enabled'], parse: parseBoolean},
FLUXER_RISK_IPINFO_API_KEY: {path: ['integrations', 'risk_integration', 'ipinfo_api_key']},
FLUXER_ACCOUNT_POLICY_DSL: {