refactor(api): drop Bunny for pluggable cache purge adapters (#2742)

fluxer_api/src/api/infrastructure/BunnyPurgeQueue.ts

@@ -1,124 +0,0 @@
-// SPDX-License-Identifier: AGPL-3.0-or-later
-
-import {Logger} from '@app/api/Logger';
-import type {IKVProvider, KVPurgeBatchResult} from '@pkgs/kv_client/src/IKVProvider';
-
-export interface IPurgeQueue {
-	addUrls(urls: Array<string>): Promise<void>;
-	getQueueSize(): Promise<number>;
-	clear(): Promise<void>;
-}
-
-const EXACT_QUEUE_KEY = 'bunny:purge:exact';
-const PREFIX_QUEUE_KEY = 'bunny:purge:prefix';
-const EXACT_BUCKET_KEY = 'bunny:purge:budget:exact';
-const PREFIX_BUCKET_KEY = 'bunny:purge:budget:prefix';
-const EXACT_MAX_TOKENS = 120;
-const EXACT_REFILL_RATE = 5;
-const EXACT_REFILL_INTERVAL_MS = 1000;
-const PREFIX_MAX_TOKENS = 20;
-const PREFIX_REFILL_RATE = 1;
-const PREFIX_REFILL_INTERVAL_MS = 2000;
-
-function isPrefix(url: string): boolean {
-	return url.endsWith('*') || url.endsWith('/');
-}
-
-export class BunnyPurgeQueue implements IPurgeQueue {
-	private readonly kvClient: IKVProvider;
-
-	constructor(kvClient: IKVProvider) {
-		this.kvClient = kvClient;
-	}
-
-	async addUrls(urls: Array<string>): Promise<void> {
-		if (urls.length === 0) {
-			return;
-		}
-		const exactUrls: Array<string> = [];
-		const prefixUrls: Array<string> = [];
-		for (const url of urls) {
-			const trimmed = url.trim();
-			if (trimmed === '') {
-				continue;
-			}
-			if (isPrefix(trimmed)) {
-				prefixUrls.push(trimmed);
-			} else {
-				exactUrls.push(trimmed);
-			}
-		}
-		try {
-			const ops: Array<Promise<number>> = [];
-			if (exactUrls.length > 0) {
-				ops.push(this.kvClient.sadd(EXACT_QUEUE_KEY, ...exactUrls));
-			}
-			if (prefixUrls.length > 0) {
-				ops.push(this.kvClient.sadd(PREFIX_QUEUE_KEY, ...prefixUrls));
-			}
-			await Promise.all(ops);
-			Logger.debug({exact: exactUrls.length, prefix: prefixUrls.length}, 'Added URLs to CDN purge queue');
-		} catch (error) {
-			Logger.error(
-				{error, exact: exactUrls.length, prefix: prefixUrls.length},
-				'Failed to add URLs to CDN purge queue',
-			);
-			throw error;
-		}
-	}
-
-	async getQueueSize(): Promise<number> {
-		try {
-			const [exactSize, prefixSize] = await Promise.all([
-				this.kvClient.scard(EXACT_QUEUE_KEY),
-				this.kvClient.scard(PREFIX_QUEUE_KEY),
-			]);
-			return exactSize + prefixSize;
-		} catch (error) {
-			Logger.error({error}, 'Failed to get CDN purge queue size');
-			throw error;
-		}
-	}
-
-	async clear(): Promise<void> {
-		try {
-			await this.kvClient.del(EXACT_QUEUE_KEY, PREFIX_QUEUE_KEY);
-			Logger.debug('Cleared CDN purge queue');
-		} catch (error) {
-			Logger.error({error}, 'Failed to clear CDN purge queue');
-			throw error;
-		}
-	}
-
-	async dequeueExactBatch(maxItems: number): Promise<KVPurgeBatchResult> {
-		return this.kvClient.dequeuePurgeBatch(
-			EXACT_QUEUE_KEY,
-			EXACT_BUCKET_KEY,
-			maxItems,
-			EXACT_MAX_TOKENS,
-			EXACT_REFILL_RATE,
-			EXACT_REFILL_INTERVAL_MS,
-		);
-	}
-
-	async dequeuePrefixBatch(maxItems: number): Promise<KVPurgeBatchResult> {
-		return this.kvClient.dequeuePurgeBatch(
-			PREFIX_QUEUE_KEY,
-			PREFIX_BUCKET_KEY,
-			maxItems,
-			PREFIX_MAX_TOKENS,
-			PREFIX_REFILL_RATE,
-			PREFIX_REFILL_INTERVAL_MS,
-		);
-	}
-}
-
-export class NoopPurgeQueue implements IPurgeQueue {
-	async addUrls(_urls: Array<string>): Promise<void> {}
-
-	async getQueueSize(): Promise<number> {
-		return 0;
-	}
-
-	async clear(): Promise<void> {}
-}

fluxer_api/src/api/infrastructure/CachePurgeAdapter.ts

@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import type {APICachePurgeConfig} from '@app/api/config/APIConfig';
+import {createHttpCachePurgeAdapter} from '@app/api/infrastructure/HttpCachePurgeAdapter';
+import {createNoneCachePurgeAdapter} from '@app/api/infrastructure/NoneCachePurgeAdapter';
+import type {CachePurgeAdapterName} from '@fluxer/config/src/MasterConfig';
+
+export interface CachePurgeBatch {
+	readonly exact: ReadonlyArray<string>;
+	readonly prefix: ReadonlyArray<string>;
+}
+
+export type CachePurgeOutcome =
+	| {readonly kind: 'purged'}
+	| {readonly kind: 'invalid_entries'; readonly status: number}
+	| {readonly kind: 'failed'; readonly status: number | null; readonly error: unknown};
+
+export interface CachePurgeAdapter {
+	purge(batch: CachePurgeBatch): Promise<CachePurgeOutcome>;
+}
+
+const CACHE_PURGE_ADAPTERS = {
+	none: createNoneCachePurgeAdapter,
+	http: createHttpCachePurgeAdapter,
+} satisfies Record<CachePurgeAdapterName, (config: APICachePurgeConfig) => CachePurgeAdapter>;
+
+export function createCachePurgeAdapter(config: APICachePurgeConfig): CachePurgeAdapter {
+	return CACHE_PURGE_ADAPTERS[config.adapter](config);
+}

fluxer_api/src/api/infrastructure/CachePurgeQueue.ts

@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import type {CachePurgeBatch} from '@app/api/infrastructure/CachePurgeAdapter';
+import {Logger} from '@app/api/Logger';
+import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
+
+export interface IPurgeQueue {
+	addUrls(urls: Array<string>): Promise<void>;
+}
+
+const EXACT_QUEUE_KEY = 'cache_purge:exact';
+const PREFIX_QUEUE_KEY = 'cache_purge:prefix';
+const EXACT_BUCKET_KEY = 'cache_purge:budget:exact';
+const PREFIX_BUCKET_KEY = 'cache_purge:budget:prefix';
+const REJECTED_KEY = 'cache_purge:rejected';
+const EXACT_MAX_TOKENS = 120;
+const EXACT_REFILL_RATE = 5;
+const EXACT_REFILL_INTERVAL_MS = 1000;
+const PREFIX_MAX_TOKENS = 20;
+const PREFIX_REFILL_RATE = 1;
+const PREFIX_REFILL_INTERVAL_MS = 2000;
+
+function isPrefix(url: string): boolean {
+	return url.endsWith('*') || url.endsWith('/');
+}
+
+export class CachePurgeQueue implements IPurgeQueue {
+	private readonly kvClient: IKVProvider;
+
+	constructor(kvClient: IKVProvider) {
+		this.kvClient = kvClient;
+	}
+
+	async addUrls(urls: Array<string>): Promise<void> {
+		if (urls.length === 0) {
+			return;
+		}
+		const exactUrls: Array<string> = [];
+		const prefixUrls: Array<string> = [];
+		for (const url of urls) {
+			const trimmed = url.trim();
+			if (trimmed === '') {
+				continue;
+			}
+			if (isPrefix(trimmed)) {
+				prefixUrls.push(trimmed);
+			} else {
+				exactUrls.push(trimmed);
+			}
+		}
+		try {
+			await this.addToSets({exact: exactUrls, prefix: prefixUrls});
+			Logger.debug({exact: exactUrls.length, prefix: prefixUrls.length}, 'Added URLs to cache purge queue');
+		} catch (error) {
+			Logger.error(
+				{error, exact: exactUrls.length, prefix: prefixUrls.length},
+				'Failed to add URLs to cache purge queue',
+			);
+			throw error;
+		}
+	}
+
+	async dequeueBatch(): Promise<CachePurgeBatch> {
+		const exact = await this.kvClient.dequeuePurgeBatch(
+			EXACT_QUEUE_KEY,
+			EXACT_BUCKET_KEY,
+			EXACT_MAX_TOKENS,
+			EXACT_MAX_TOKENS,
+			EXACT_REFILL_RATE,
+			EXACT_REFILL_INTERVAL_MS,
+		);
+		try {
+			const prefix = await this.kvClient.dequeuePurgeBatch(
+				PREFIX_QUEUE_KEY,
+				PREFIX_BUCKET_KEY,
+				PREFIX_MAX_TOKENS,
+				PREFIX_MAX_TOKENS,
+				PREFIX_REFILL_RATE,
+				PREFIX_REFILL_INTERVAL_MS,
+			);
+			return {exact: exact.urls, prefix: prefix.urls};
+		} catch (error) {
+			await this.requeue({exact: exact.urls, prefix: []});
+			throw error;
+		}
+	}
+
+	async requeue(batch: CachePurgeBatch): Promise<void> {
+		try {
+			await this.addToSets(batch);
+		} catch (error) {
+			Logger.error(
+				{error, exact: batch.exact.length, prefix: batch.prefix.length},
+				'Failed to requeue cache purge entries',
+			);
+			throw error;
+		}
+	}
+
+	async reject(batch: CachePurgeBatch): Promise<void> {
+		await this.kvClient.sadd(REJECTED_KEY, ...batch.exact, ...batch.prefix);
+		Logger.error(
+			{exact: batch.exact.length, prefix: batch.prefix.length, key: REJECTED_KEY},
+			'Set aside cache purge entries the endpoint rejected',
+		);
+	}
+
+	private async addToSets(batch: CachePurgeBatch): Promise<void> {
+		const ops: Array<Promise<number>> = [];
+		if (batch.exact.length > 0) {
+			ops.push(this.kvClient.sadd(EXACT_QUEUE_KEY, ...batch.exact));
+		}
+		if (batch.prefix.length > 0) {
+			ops.push(this.kvClient.sadd(PREFIX_QUEUE_KEY, ...batch.prefix));
+		}
+		await Promise.all(ops);
+	}
+}
+
+export class NoopPurgeQueue implements IPurgeQueue {
+	async addUrls(_urls: Array<string>): Promise<void> {}
+}

fluxer_api/src/api/infrastructure/HttpCachePurgeAdapter.ts

@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import type {APICachePurgeConfig} from '@app/api/config/APIConfig';
+import type {CachePurgeAdapter, CachePurgeBatch, CachePurgeOutcome} from '@app/api/infrastructure/CachePurgeAdapter';
+import * as FetchUtils from '@app/api/utils/FetchUtils';
+
+function stripOneTrailingAsterisk(entry: string): string {
+	return entry.endsWith('*') ? entry.slice(0, -1) : entry;
+}
+
+export function createHttpCachePurgeAdapter(config: APICachePurgeConfig): CachePurgeAdapter {
+	return {
+		async purge(batch: CachePurgeBatch): Promise<CachePurgeOutcome> {
+			const headers: Record<string, string> = {'Content-Type': 'application/json'};
+			if (config.http.token !== '') {
+				headers.Authorization = `Bearer ${config.http.token}`;
+			}
+			let response: Response;
+			try {
+				response = await fetch(config.http.endpoint, {
+					method: 'POST',
+					headers,
+					body: JSON.stringify({exact: batch.exact, prefix: batch.prefix.map(stripOneTrailingAsterisk)}),
+					redirect: 'manual',
+					signal: AbortSignal.timeout(config.http.timeoutMs),
+				});
+			} catch (error) {
+				return {kind: 'failed', status: null, error};
+			}
+			FetchUtils.discardResponseBody(response.body, response.status);
+			if (response.ok) {
+				return {kind: 'purged'};
+			}
+			if (response.status === 400 || response.status === 422) {
+				return {kind: 'invalid_entries', status: response.status};
+			}
+			return {kind: 'failed', status: response.status, error: null};
+		},
+	};
+}

fluxer_api/src/api/infrastructure/NoneCachePurgeAdapter.ts

@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import type {CachePurgeAdapter} from '@app/api/infrastructure/CachePurgeAdapter';
+
+export function createNoneCachePurgeAdapter(): CachePurgeAdapter {
+	return {
+		purge: async () => ({kind: 'purged'}),
+	};
+}