fix: migrate gifs to klipy service (#1220)

This commit is contained in:
Hampus authored and GitHub committed 2026-06-30 01:19:57 +02:00
1 parent 99eedf6d1e
commit 4a741d8396
79 files changed
+3032 -1653

No files matched your search

-6
View File
@@ -360,15 +360,9 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
bluesky: master.auth.bluesky as BlueskyOAuthConfig,
},
cookie: master.cookie,
gif: {
provider: master.integrations.gif.provider,
},
klipy: {
apiKey: master.integrations.klipy.api_key,
},
tenor: {
apiKey: master.integrations.tenor.api_key,
},
youtube: {
apiKey: master.integrations.youtube.api_key,
},
@@ -290,8 +290,6 @@ export function InstanceConfigAdminController(app: HonoApp) {
await instanceConfigRepository.setInstanceIntegrationsConfig({
gif: data.integrations.gif
? omitUndefinedFields({
provider: readOptionalField(data.integrations.gif, 'provider'),
tenor_api_key: readOptionalField(data.integrations.gif, 'tenor_api_key'),
klipy_api_key: readOptionalField(data.integrations.gif, 'klipy_api_key'),
})
: undefined,
+1 -1
View File
@@ -68,7 +68,7 @@ export function configureMiddleware(routes: HonoApp, options: MiddlewarePipeline
enabled: true,
logger,
trustClientIpHeader,
clientIpHeaderName: clientIpHeaderName ?? 'x-forwarded-for',
clientIpHeaderName: resolvedHeader,
}),
);
}
-6
View File
@@ -271,15 +271,9 @@ export interface APIConfig {
domain: string;
secure: boolean;
};
gif: {
provider: 'klipy' | 'tenor';
};
klipy: {
apiKey?: string;
};
tenor: {
apiKey?: string;
};
youtube: {
apiKey?: string;
};
@@ -2,6 +2,7 @@
import {ResolveGifUrlsBodySchema, ResolveGifUrlsResponse} from '@fluxer/schema/src/domains/gif/FavoriteGifSchemas';
import type {Context} from 'hono';
import {resolveGifRequestCountry} from '../gif/GifRequestCountry';
import {DefaultUserOnly, LoginRequired} from '../middleware/AuthMiddleware';
import {RateLimitMiddleware} from '../middleware/RateLimitMiddleware';
import {OpenAPI} from '../middleware/ResponseTypeMiddleware';
@@ -11,8 +12,8 @@ import type {HonoApp, HonoEnv} from '../types/HonoEnv';
import {Validator} from '../Validator';
import {resolveFavoriteGifEntry} from './FavoriteGifResolver';
function getCountry(ctx: Context<HonoEnv>): string {
return ctx.req.header('CF-IPCountry') || 'US';
async function getCountry(ctx: Context<HonoEnv>): Promise<string> {
return resolveGifRequestCountry(ctx.req.raw);
}
function getLocale(ctx: Context<HonoEnv>): string {
@@ -42,7 +43,7 @@ export function FavoriteGifController(app: HonoApp) {
const gifService = ctx.get('gifService');
const unfurlerService = getUnfurlerService();
const locale = getLocale(ctx);
const country = getCountry(ctx);
const country = await getCountry(ctx);
const entries = await Promise.all(
urls.map((url) => resolveFavoriteGifEntry({url, locale, country, gifService, mediaService, unfurlerService})),
);
@@ -0,0 +1,87 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
import {describe, expect, it, vi} from 'vitest';
import {GifService} from '../gif/GifService';
import type {IGifProvider} from '../gif/IGifProvider';
import type {IMediaService} from '../infrastructure/IMediaService';
import type {IUnfurlerService} from '../infrastructure/IUnfurlerService';
import {resolveFavoriteGifEntry} from './FavoriteGifResolver';
function createProvider(gif: GifResponse): IGifProvider {
return {
meta: {
name: 'klipy',
displayName: 'KLIPY',
attributionRequired: true,
},
isAvailable: async () => true,
search: async () => [],
registerShare: async () => undefined,
getFeatured: async () => ({gifs: [], categories: []}),
getTrendingGifs: async () => [],
suggest: async () => [],
resolveByUrl: async () => gif,
buildShareUrl: (slug) => `https://klipy.com/gifs/${slug}`,
extractSlugFromUrl: () => gif.slug,
};
}
describe('resolveFavoriteGifEntry', () => {
it('returns provider GIF media without probing generic media fallbacks', async () => {
const gif: GifResponse = {
id: 'goatplaybanjo-chat-4',
slug: 'goatplaybanjo-chat-4',
provider: 'klipy',
title: 'Goatplaybanjo Chat',
url: 'https://klipy.com/gifs/goatplaybanjo-chat-4',
src: 'https://static.klipy.example/fallback.gif',
proxy_src: 'https://media.example/fallback.gif',
width: 120,
height: 100,
media: {
tinygif: {
src: 'https://static.klipy.example/tiny.gif',
proxy_src: 'https://media.example/tiny.gif',
width: 80,
height: 60,
},
webm: {
src: 'https://static.klipy.example/full.webm',
proxy_src: 'https://media.example/full.webm',
width: 220,
height: 229,
},
},
placeholder: 'thumbhash',
};
const mediaService = {
getMetadata: vi.fn(),
getExternalMediaProxyURL: vi.fn(),
} as unknown as IMediaService;
const unfurlerService = {
unfurl: vi.fn(),
} as unknown as IUnfurlerService;
await expect(
resolveFavoriteGifEntry({
url: gif.url,
locale: 'en-US',
country: 'US',
gifService: new GifService(createProvider(gif)),
mediaService,
unfurlerService,
}),
).resolves.toEqual({
url: gif.url,
proxy_url: 'https://media.example/full.webm',
width: 220,
height: 229,
media: gif.media,
content_type: 'video/webm',
placeholder: 'thumbhash',
});
expect(mediaService.getMetadata).not.toHaveBeenCalled();
expect(unfurlerService.unfurl).not.toHaveBeenCalled();
});
});
@@ -4,6 +4,7 @@ import {Logger} from '@fluxer/logger/src/Logger';
import type {ResolvedGifEntrySchema} from '@fluxer/schema/src/domains/gif/FavoriteGifSchemas';
import type {GifMediaFormat, GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
import type {EmbedMediaResponse} from '@fluxer/schema/src/domains/message/EmbedSchemas';
import {tryExtractGifProviderSlug} from '../gif/GifProviderUtils';
import type {GifService} from '../gif/GifService';
import type {IGifProvider} from '../gif/IGifProvider';
import type {IMediaService, MediaProxyMetadataResponse} from '../infrastructure/IMediaService';
@@ -91,12 +92,9 @@ async function resolveProviderGifUrl({
country: string;
gifService: GifService;
}): Promise<GifResponse | null> {
for (const provider of gifService.listProviders()) {
if (!(await provider.isAvailable()) || !provider.extractSlugFromUrl(url)) continue;
const gif = await resolveProviderUrl(provider, {url, locale, country});
if (gif) return gif;
}
return null;
const provider = gifService.getProvider();
if (!(await tryExtractGifProviderSlug(provider, url))) return null;
return resolveProviderUrl(provider, {url, locale, country});
}
async function resolveProviderUrl(
@@ -18,8 +18,12 @@ import {createAttachmentID, createMemeID, userIdToChannelId} from '../BrandedTyp
import {Config} from '../Config';
import type {ChannelService} from '../channel/services/ChannelService';
import {makeAttachmentCdnKey, makeAttachmentCdnUrl} from '../channel/services/message/MessageHelpers';
import {
isOptionalGifProviderError,
type ResolvedGifProviderSlug,
tryExtractGifProviderSlug,
} from '../gif/GifProviderUtils';
import type {GifService} from '../gif/GifService';
import type {IGifProvider} from '../gif/IGifProvider';
import type {MediaProxyMetadataResponse} from '../infrastructure/IMediaService';
import type {IStorageService} from '../infrastructure/IStorageService';
import type {IUnfurlerService} from '../infrastructure/IUnfurlerService';
@@ -122,7 +126,7 @@ export class FavoriteMemeService {
if (!message) {
throw new UnknownMessageError();
}
const media = this.findMediaInMessage(message, attachmentId, embedIndex);
const media = await this.findMediaInMessage(message, attachmentId, embedIndex);
if (!media) {
throw InputValidationError.fromCode('media', ValidationErrorCodes.NO_VALID_MEDIA_IN_MESSAGE);
}
@@ -319,18 +323,9 @@ export class FavoriteMemeService {
throw new MediaMetadataError('URL');
}
let contentHash = metadata.content_hash;
const fileData = Buffer.from(metadata.base64 ?? '', 'base64');
const resolvedGif = this.resolveGifFromInputs({slug: gifSlug, providerName: gifProvider, url});
const resolvedGif = await this.resolveGifFromInputs({slug: gifSlug, providerName: gifProvider, url});
if (resolvedGif) {
const canonicalUrl = resolvedGif.provider.buildShareUrl(resolvedGif.slug);
const unfurled = await this.unfurlerService.unfurl(canonicalUrl, 'allow');
if (unfurled.length > 0 && unfurled[0].video?.content_hash) {
contentHash = unfurled[0].video.content_hash;
Logger.debug(
{provider: resolvedGif.provider.meta.name, gifSlug: resolvedGif.slug, contentHash},
'Using unfurled video content_hash for provider GIF',
);
}
contentHash = await this.resolveProviderGifContentHash(resolvedGif, contentHash);
}
const existingMemes = await this.favoriteMemeRepository.findByUserId(user.id);
const duplicate = existingMemes.find((meme) => meme.contentHash === contentHash);
@@ -343,6 +338,7 @@ export class FavoriteMemeService {
const userChannelId = userIdToChannelId(user.id);
const newAttachmentId = createAttachmentID(await this.apiContext.services.snowflake.generate());
const storageKey = makeAttachmentCdnKey(userChannelId, newAttachmentId, filename);
const fileData = Buffer.from(metadata.base64 ?? '', 'base64');
await this.storageService.uploadObject({
bucket: Config.s3.buckets.cdn,
key: storageKey,
@@ -470,7 +466,7 @@ export class FavoriteMemeService {
}
}
private resolveGifFromInputs({
private async resolveGifFromInputs({
slug,
providerName,
url,
@@ -478,34 +474,51 @@ export class FavoriteMemeService {
slug?: string | null;
providerName?: string | null;
url: string;
}): {
provider: IGifProvider;
slug: string;
} | null {
}): Promise<ResolvedGifProviderSlug | null> {
if (slug && providerName) {
const provider = this.gifService.getByName(providerName);
const trimmed = slug.trim();
if (provider && trimmed) {
const normalized = provider.extractSlugFromUrl(trimmed) ?? trimmed;
const normalized = (await tryExtractGifProviderSlug(provider, trimmed)) ?? trimmed;
return {provider, slug: normalized};
}
}
for (const provider of this.gifService.listProviders()) {
const extracted = provider.extractSlugFromUrl(url);
if (extracted) {
return {provider, slug: extracted};
}
}
return null;
const provider = this.gifService.getProvider();
const extracted = await tryExtractGifProviderSlug(provider, url);
return extracted ? {provider, slug: extracted} : null;
}
private detectGifFromUrl(url: string): {
provider: IGifProvider;
slug: string;
} | null {
private async detectGifFromUrl(url: string): Promise<ResolvedGifProviderSlug | null> {
return this.resolveGifFromInputs({url});
}
private async resolveProviderGifContentHash(
resolvedGif: ResolvedGifProviderSlug,
fallbackContentHash: string,
): Promise<string> {
try {
const canonicalUrl = resolvedGif.provider.buildShareUrl(resolvedGif.slug);
const unfurled = await this.unfurlerService.unfurl(canonicalUrl, 'allow');
if (unfurled.length > 0 && unfurled[0].video?.content_hash) {
Logger.debug(
{
provider: resolvedGif.provider.meta.name,
gifSlug: resolvedGif.slug,
contentHash: unfurled[0].video.content_hash,
},
'Using unfurled video content_hash for provider GIF',
);
return unfurled[0].video.content_hash;
}
} catch (error) {
if (!isOptionalGifProviderError(error)) {
throw error;
}
Logger.debug({error, provider: resolvedGif.provider.meta.name}, 'Skipping unavailable GIF provider enrichment');
}
return fallbackContentHash;
}
private resolveFavoriteMemeName(name: string | undefined | null, fallbackFilename: string): string {
const normalizedInput = typeof name === 'string' ? name.trim() : '';
const fallbackName = fallbackFilename.trim() || 'favorite meme';
@@ -517,11 +530,11 @@ export class FavoriteMemeService {
return finalName;
}
private findMediaInMessage(
private async findMediaInMessage(
message: Message,
preferredAttachmentId?: string,
preferredEmbedIndex?: number,
): FavoriteMemeMedia | null {
): Promise<FavoriteMemeMedia | null> {
const attachments = this.getMessageAttachmentCandidates(message);
const embeds = this.getMessageEmbedCandidates(message);
if (preferredEmbedIndex !== undefined) {
@@ -553,7 +566,7 @@ export class FavoriteMemeService {
}
}
for (const embed of embeds) {
const media = this.mediaFromEmbed(embed, 'media');
const media = await this.mediaFromEmbed(embed, 'media');
if (media) return media;
}
return null;
@@ -591,7 +604,10 @@ export class FavoriteMemeService {
};
}
private mediaFromEmbed(embed: MessageEmbedCandidate, fallbackFilename: string): FavoriteMemeMedia | null {
private async mediaFromEmbed(
embed: MessageEmbedCandidate,
fallbackFilename: string,
): Promise<FavoriteMemeMedia | null> {
const media = embed.image || embed.video || embed.thumbnail;
if (!media?.url) {
return null;
@@ -603,7 +619,7 @@ export class FavoriteMemeService {
}
const isExternal = !this.isInternalCDNUrl(media.url);
const isGifv = embed.type === 'gifv' || isAnimatedEmbedMedia(media.contentType, media.flags);
const detectedGif = embed.type === 'gifv' ? this.detectGifFromUrl(media.url) : null;
const detectedGif = embed.type === 'gifv' ? await this.detectGifFromUrl(media.url) : null;
return {
isExternal,
url: media.url,
@@ -0,0 +1,71 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {GIF_PROVIDER_HEADER, type GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
import {Hono} from 'hono';
import {describe, expect, it, vi} from 'vitest';
import type {HonoEnv} from '../types/HonoEnv';
import {GifController} from './GifController';
import {GifService} from './GifService';
import type {IGifProvider} from './IGifProvider';
function createProvider(gifs: Array<GifResponse>): IGifProvider {
return {
meta: {
name: 'klipy',
displayName: 'KLIPY',
attributionRequired: true,
},
isAvailable: async () => true,
search: vi.fn(async () => gifs),
registerShare: async () => undefined,
getFeatured: async () => ({gifs: [], categories: []}),
getTrendingGifs: async () => [],
suggest: async () => [],
resolveByUrl: async () => null,
buildShareUrl: (slug) => `https://klipy.com/gifs/${slug}`,
extractSlugFromUrl: () => null,
};
}
function createApp(gifService: GifService): Hono<HonoEnv> {
const app = new Hono<HonoEnv>({strict: true});
app.use('*', async (ctx, next) => {
ctx.set('gifService', gifService);
ctx.set('user', {
isBot: false,
suspiciousActivityFlags: 0,
} as HonoEnv['Variables']['user']);
ctx.set('authTokenType', 'session');
await next();
});
GifController(app);
return app;
}
describe('GifController', () => {
it('serves deprecated Tenor routes from the configured KLIPY provider', async () => {
const gifs: Array<GifResponse> = [
{
id: 'goatplaybanjo-chat-4',
slug: 'goatplaybanjo-chat-4',
provider: 'klipy',
title: 'Goatplaybanjo Chat',
url: 'https://klipy.com/gifs/goatplaybanjo-chat-4',
src: 'https://static.klipy.example/full.webm',
proxy_src: 'https://media.example/full.webm',
width: 220,
height: 229,
media: {},
placeholder: null,
},
];
const provider = createProvider(gifs);
const response = await createApp(new GifService(provider)).request('/tenor/search?q=cat&locale=en-US');
expect(response.status).toBe(200);
expect(response.headers.get('Deprecation')).toBe('true');
expect(response.headers.get(GIF_PROVIDER_HEADER)).toBe('klipy');
await expect(response.json()).resolves.toEqual(gifs);
expect(provider.search).toHaveBeenCalledWith({q: 'cat', locale: 'en_US', country: 'US'});
});
});
+11 -6
View File
@@ -17,6 +17,7 @@ import {RateLimitConfigs} from '../RateLimitConfig';
import type {HonoApp, HonoEnv} from '../types/HonoEnv';
import {Validator} from '../Validator';
import {GifProviderHeaderMiddleware} from './GifProviderHeaderMiddleware';
import {resolveGifRequestCountry} from './GifRequestCountry';
const TAGS = ['GIFs'];
@@ -34,8 +35,8 @@ const PREFIXES: ReadonlyArray<PrefixConfig> = [
];
const DEPRECATION_NOTICE = 'Use /gifs/* instead - these vendor-specific paths are deprecated and will be removed.';
function getCountry(ctx: Context<HonoEnv>): string {
return ctx.req.header('CF-IPCountry') || 'US';
async function getCountry(ctx: Context<HonoEnv>): Promise<string> {
return resolveGifRequestCountry(ctx.req.raw);
}
function deprecationMiddleware(deprecated: boolean): MiddlewareHandler<HonoEnv> {
@@ -72,7 +73,8 @@ function registerRoutes(app: HonoApp, cfg: PrefixConfig) {
async (ctx) => {
const {q, locale} = ctx.req.valid('query');
const provider = await ctx.get('gifService').getActive();
return ctx.json(await provider.search({q, locale, country: getCountry(ctx)}));
const country = await getCountry(ctx);
return ctx.json(await provider.search({q, locale, country}));
},
);
app.get(
@@ -94,7 +96,8 @@ function registerRoutes(app: HonoApp, cfg: PrefixConfig) {
Validator('query', GifLocaleQuery),
async (ctx) => {
const provider = await ctx.get('gifService').getActive();
return ctx.json(await provider.getFeatured({locale: ctx.req.valid('query').locale, country: getCountry(ctx)}));
const country = await getCountry(ctx);
return ctx.json(await provider.getFeatured({locale: ctx.req.valid('query').locale, country}));
},
);
const trendingPath = prefix === '/gifs' ? `${prefix}/trending` : `${prefix}/trending-gifs`;
@@ -117,10 +120,11 @@ function registerRoutes(app: HonoApp, cfg: PrefixConfig) {
Validator('query', GifLocaleQuery),
async (ctx) => {
const provider = await ctx.get('gifService').getActive();
const country = await getCountry(ctx);
return ctx.json(
await provider.getTrendingGifs({
locale: ctx.req.valid('query').locale,
country: getCountry(ctx),
country,
}),
);
},
@@ -145,7 +149,8 @@ function registerRoutes(app: HonoApp, cfg: PrefixConfig) {
async (ctx) => {
const {id, q, locale} = ctx.req.valid('json');
const provider = await ctx.get('gifService').getActive();
await provider.registerShare({id, q: q ?? '', locale, country: getCountry(ctx)});
const country = await getCountry(ctx);
await provider.registerShare({id, q: q ?? '', locale, country});
return ctx.body(null, 204);
},
);
@@ -1,99 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {GifCategoryTagResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
import type {ICacheService} from '@pkgs/cache/src/ICacheService';
import {ms} from 'itty-time';
const ENRICHED_CATEGORIES_FRESH_MS = ms('30 minutes');
const ENRICHED_CATEGORIES_TTL_SECONDS = Math.floor(ms('6 hours') / 1000);
const SEEN_LOCALES_TTL_SECONDS = Math.floor(ms('7 days') / 1000);
export const REFRESH_LOCK_TTL_SECONDS = Math.floor(ms('5 minutes') / 1000);
interface EnrichedCategoriesEntry {
data: Array<GifCategoryTagResponse>;
timestamp: number;
}
function enrichedCategoriesCacheKey(provider: string, locale: string, country: string): string {
return `gif:featured_categories_enriched:${provider}:${locale}:${country}`;
}
function seenLocalesSetKey(provider: string): string {
return `gif:featured_categories_seen_locales:${provider}`;
}
function encodeLocaleMember(locale: string, country: string): string {
return `${locale}|${country}`;
}
function decodeLocaleMember(member: string): {
locale: string;
country: string;
} | null {
const idx = member.indexOf('|');
if (idx <= 0) return null;
const locale = member.slice(0, idx);
const country = member.slice(idx + 1);
if (!locale || !country) return null;
return {locale, country};
}
export function refreshLockKey(provider: string, locale: string, country: string): string {
return `gif:featured_categories_refresh_lock:${provider}:${locale}:${country}`;
}
export async function readEnrichedCategoriesCache(
cache: ICacheService,
provider: string,
locale: string,
country: string,
): Promise<{
data: Array<GifCategoryTagResponse>;
isStale: boolean;
} | null> {
const entry = await cache.get<EnrichedCategoriesEntry>(enrichedCategoriesCacheKey(provider, locale, country));
if (!entry) return null;
const isStale = Date.now() - entry.timestamp > ENRICHED_CATEGORIES_FRESH_MS;
return {data: entry.data, isStale};
}
export async function writeEnrichedCategoriesCache(
cache: ICacheService,
provider: string,
locale: string,
country: string,
data: Array<GifCategoryTagResponse>,
): Promise<void> {
const entry: EnrichedCategoriesEntry = {data, timestamp: Date.now()};
await cache.set(enrichedCategoriesCacheKey(provider, locale, country), entry, ENRICHED_CATEGORIES_TTL_SECONDS);
}
export async function trackSeenLocale(
cache: ICacheService,
provider: string,
locale: string,
country: string,
): Promise<void> {
await cache.sadd(seenLocalesSetKey(provider), encodeLocaleMember(locale, country), SEEN_LOCALES_TTL_SECONDS);
}
export async function listSeenLocales(
cache: ICacheService,
provider: string,
): Promise<
Array<{
locale: string;
country: string;
}>
> {
const members = await cache.smembers(seenLocalesSetKey(provider));
const out: Array<{
locale: string;
country: string;
}> = [];
for (const member of members) {
const decoded = decodeLocaleMember(member);
if (decoded) out.push(decoded);
}
return out;
}
@@ -25,7 +25,6 @@ function createProvider(): IGifProvider {
getFeatured: async () => ({gifs: [], categories: []}),
getTrendingGifs: async () => [],
suggest: async () => [],
refreshFeaturedCategories: async () => undefined,
resolveByUrl: async () => null,
buildShareUrl: (slug) => `https://klipy.example/${slug}`,
extractSlugFromUrl: () => null,
@@ -47,7 +46,7 @@ function createApp(gifService?: GifService): Hono<HonoEnv> {
describe('GifProviderHeaderMiddleware', () => {
it('emits active GIF provider metadata headers', async () => {
const gifService = new GifService({providers: [createProvider()], activeName: 'klipy'});
const gifService = new GifService(createProvider());
const response = await createApp(gifService).request('/probe');
expect(response.headers.get(GIF_PROVIDER_HEADER)).toBe('klipy');
@@ -13,8 +13,8 @@ export const GifProviderHeaderMiddleware = createMiddleware<HonoEnv>(async (ctx,
await next();
const gifService = ctx.get('gifService') as GifService | undefined;
if (!gifService) return;
const provider = gifService.getByName(await gifService.getActiveName());
if (!provider || !(await provider.isAvailable())) return;
const provider = await gifService.getActive().catch(() => null);
if (!provider) return;
ctx.header(GIF_PROVIDER_HEADER, provider.meta.name);
ctx.header(GIF_PROVIDER_DISPLAY_NAME_HEADER, provider.meta.displayName);
ctx.header(GIF_PROVIDER_ATTRIBUTION_HEADER, provider.meta.attributionRequired ? 'true' : 'false');
@@ -0,0 +1,82 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {ServiceUnavailableError} from '@fluxer/errors/src/domains/core/ServiceUnavailableError';
import {describe, expect, it} from 'vitest';
import {tryExtractGifProviderSlug} from './GifProviderUtils';
import type {IGifProvider} from './IGifProvider';
function createProvider(overrides: Partial<IGifProvider> = {}): IGifProvider {
return {
meta: {
name: 'klipy',
displayName: 'KLIPY',
attributionRequired: true,
},
isAvailable: async () => true,
search: async () => [],
registerShare: async () => undefined,
getFeatured: async () => ({gifs: [], categories: []}),
getTrendingGifs: async () => [],
suggest: async () => [],
resolveByUrl: async () => null,
buildShareUrl: (slug) => `https://klipy.example/${slug}`,
extractSlugFromUrl: () => 'slug',
...overrides,
};
}
describe('GifProviderUtils', () => {
it('returns a trimmed slug from an available provider', async () => {
await expect(
tryExtractGifProviderSlug(
createProvider({
extractSlugFromUrl: () => ' goatplaybanjo-chat-4 ',
}),
'https://klipy.com/gifs/goatplaybanjo-chat-4',
),
).resolves.toBe('goatplaybanjo-chat-4');
});
it('treats unavailable optional provider failures as no match', async () => {
await expect(
tryExtractGifProviderSlug(
createProvider({
extractSlugFromUrl: () => {
throw new ServiceUnavailableError();
},
}),
'https://klipy.com/gifs/goatplaybanjo-chat-4',
),
).resolves.toBeNull();
});
it('does not check availability when a provider cannot extract a slug', async () => {
let availabilityChecks = 0;
await expect(
tryExtractGifProviderSlug(
createProvider({
isAvailable: async () => {
availabilityChecks += 1;
return true;
},
extractSlugFromUrl: () => null,
}),
'https://example.com/media.gif',
),
).resolves.toBeNull();
expect(availabilityChecks).toBe(0);
});
it('returns null when the configured provider is unavailable', async () => {
await expect(
tryExtractGifProviderSlug(
createProvider({
isAvailable: async () => false,
extractSlugFromUrl: () => 'matched',
}),
'https://klipy.com/gifs/matched',
),
).resolves.toBeNull();
});
});
@@ -0,0 +1,30 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';
import {ServiceUnavailableError} from '@fluxer/errors/src/domains/core/ServiceUnavailableError';
import type {IGifProvider} from './IGifProvider';
export type ResolvedGifProviderSlug = {
provider: IGifProvider;
slug: string;
};
export function isOptionalGifProviderError(error: unknown): boolean {
return error instanceof FeatureTemporarilyDisabledError || error instanceof ServiceUnavailableError;
}
export async function tryExtractGifProviderSlug(provider: IGifProvider, value: string): Promise<string | null> {
try {
const slug = provider.extractSlugFromUrl(value);
const trimmed = slug?.trim() ?? '';
if (trimmed.length === 0) {
return null;
}
return (await provider.isAvailable()) ? trimmed : null;
} catch (error) {
if (isOptionalGifProviderError(error)) {
return null;
}
throw error;
}
}
@@ -0,0 +1,49 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {GeoipResult} from '@pkgs/geoip/src/GeoipLookup';
import {beforeEach, describe, expect, it, vi} from 'vitest';
import {resolveGifRequestCountry} from './GifRequestCountry';
const {lookupGeoipMock} = vi.hoisted(() => ({
lookupGeoipMock: vi.fn(),
}));
vi.mock('../utils/IpUtils', () => ({
lookupGeoip: lookupGeoipMock,
}));
function geoip(countryCode: string | null): GeoipResult {
return {
countryCode,
normalizedIp: '203.0.113.10',
city: null,
region: null,
countryName: null,
};
}
describe('resolveGifRequestCountry', () => {
const req = new Request('https://fluxer.test/gifs/search');
beforeEach(() => {
lookupGeoipMock.mockReset();
});
it('uses the GeoIP country code for GIF provider requests', async () => {
lookupGeoipMock.mockResolvedValue(geoip('se'));
await expect(resolveGifRequestCountry(req)).resolves.toBe('SE');
});
it('falls back to US when GeoIP has no country', async () => {
lookupGeoipMock.mockResolvedValue(geoip(null));
await expect(resolveGifRequestCountry(req)).resolves.toBe('US');
});
it('falls back to US when GeoIP lookup fails', async () => {
lookupGeoipMock.mockRejectedValue(new Error('geoip unavailable'));
await expect(resolveGifRequestCountry(req)).resolves.toBe('US');
});
});
@@ -0,0 +1,14 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {lookupGeoip} from '../utils/IpUtils';
const KLIPY_DEFAULT_COUNTRY = 'US';
export async function resolveGifRequestCountry(req: Request): Promise<string> {
try {
const geoip = await lookupGeoip(req);
return geoip.countryCode?.trim().toUpperCase() || KLIPY_DEFAULT_COUNTRY;
} catch {
return KLIPY_DEFAULT_COUNTRY;
}
}
+10 -29
View File
@@ -1,46 +1,27 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';
import {Logger} from '../Logger';
import type {IGifProvider} from './IGifProvider';
export class GifService {
private readonly providersByName: Map<string, IGifProvider>;
private readonly activeName: string | (() => Promise<string>);
private readonly provider: IGifProvider;
constructor(params: {
providers: ReadonlyArray<IGifProvider>;
activeName: string | (() => Promise<string>);
}) {
this.providersByName = new Map(params.providers.map((p) => [p.meta.name, p]));
this.activeName = params.activeName;
if (typeof params.activeName === 'string' && !this.providersByName.has(params.activeName)) {
Logger.warn(
{activeName: params.activeName, registered: Array.from(this.providersByName.keys())},
'Active GIF provider is not registered; /gifs requests will fail until configuration is fixed',
);
}
constructor(provider: IGifProvider) {
this.provider = provider;
}
getProvider(): IGifProvider {
return this.provider;
}
async getActive(): Promise<IGifProvider> {
const activeName = await this.getActiveName();
const provider = this.providersByName.get(activeName);
if (!provider || !(await provider.isAvailable())) {
Logger.debug({activeName}, 'Active GIF provider unavailable');
if (!(await this.provider.isAvailable())) {
throw new FeatureTemporarilyDisabledError();
}
return provider;
}
async getActiveName(): Promise<string> {
return typeof this.activeName === 'string' ? this.activeName : this.activeName();
return this.provider;
}
getByName(name: string): IGifProvider | null {
return this.providersByName.get(name) ?? null;
}
listProviders(): ReadonlyArray<IGifProvider> {
return Array.from(this.providersByName.values());
return name === this.provider.meta.name ? this.provider : null;
}
}
-1
View File
@@ -19,7 +19,6 @@ export interface IGifProvider {
}>;
getTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>>;
suggest(params: {q: string; locale: string}): Promise<Array<string>>;
refreshFeaturedCategories(params: {locale: string; country: string}): Promise<void>;
resolveByUrl(params: {url: string; locale: string; country: string}): Promise<GifResponse | null>;
buildShareUrl(slug: string): string;
extractSlugFromUrl(url: string): string | null;
-563
View File
@@ -1,563 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {FLUXER_USER_AGENT} from '@fluxer/constants/src/Core';
import type {GifCategoryTagResponse, GifMediaFormat, GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
import type {ICacheService} from '@pkgs/cache/src/ICacheService';
import {ms} from 'itty-time';
import {Config} from '../Config';
import type {IMediaService} from '../infrastructure/IMediaService';
import {Logger} from '../Logger';
import {getWorkerService} from '../middleware/ServiceRegistry';
import {EXTERNAL_RESPONSE_LIMITS} from '../utils/ExternalResponseLimits';
import * as FetchUtils from '../utils/FetchUtils';
import {isJsonRecord, parseJsonUnknown} from '../utils/JsonBoundaryUtils';
import {
REFRESH_LOCK_TTL_SECONDS,
readEnrichedCategoriesCache,
refreshLockKey,
trackSeenLocale,
writeEnrichedCategoriesCache,
} from './GifFeaturedCategoriesCache';
import type {GifProviderMeta, IGifProvider} from './IGifProvider';
const KLIPY_BASE_URL = 'https://api.klipy.com/v2';
const DEFAULT_CONTENT_FILTER = 'low';
const CLIENT_KEY = 'fluxer';
const MAX_RETRIES = 3;
const BACKOFF_BASE_DELAY = ms('1 second');
const CACHE_EXPIRATION_TIME = ms('5 minutes');
const KLIPY_PROVIDER_META: GifProviderMeta = {
name: 'klipy',
displayName: 'KLIPY',
attributionRequired: true,
};
interface KlipyFileEntry {
url?: string;
width?: number;
height?: number;
size?: number;
}
type KlipyFormatKey = 'gif' | 'webp' | 'mp4' | 'webm';
type KlipySizeKey = 'hd' | 'md' | 'sm' | 'xs';
type KlipyFileBucket = Partial<Record<KlipyFormatKey, KlipyFileEntry>>;
interface KlipyGif {
id: string;
slug?: string;
title: string;
itemurl: string;
file?: Partial<Record<KlipySizeKey, KlipyFileBucket>>;
media_formats?: {
webm?: {
url: string;
dims: [number, number];
};
};
}
const KLIPY_SIZE_PREFERENCE: ReadonlyArray<KlipySizeKey> = ['hd', 'md', 'sm', 'xs'];
const KLIPY_FORMAT_KEYS: ReadonlyArray<KlipyFormatKey> = ['webm', 'mp4', 'webp', 'gif'];
const KLIPY_PUBLIC_FORMAT_KEYS: Record<KlipySizeKey, Record<KlipyFormatKey, string>> = {
hd: {webm: 'webm', mp4: 'mp4', webp: 'webp', gif: 'gif'},
md: {webm: 'mediumwebm', mp4: 'mediummp4', webp: 'mediumwebp', gif: 'mediumgif'},
sm: {webm: 'tinywebm', mp4: 'tinymp4', webp: 'tinywebp', gif: 'tinygif'},
xs: {webm: 'nanowebm', mp4: 'nanomp4', webp: 'nanowebp', gif: 'nanogif'},
};
interface KlipyCategoryTag {
searchterm: string;
}
function isKlipyFileEntry(value: unknown): value is KlipyFileEntry {
return (
isJsonRecord(value) &&
(value.url === undefined || typeof value.url === 'string') &&
(value.width === undefined || typeof value.width === 'number') &&
(value.height === undefined || typeof value.height === 'number') &&
(value.size === undefined || typeof value.size === 'number')
);
}
function isKlipyFileBucket(value: unknown): value is KlipyFileBucket {
return isJsonRecord(value) && Object.values(value).every(isKlipyFileEntry);
}
function isKlipyFallbackMediaFormat(value: unknown): value is NonNullable<KlipyGif['media_formats']>['webm'] {
if (!isJsonRecord(value) || typeof value.url !== 'string' || !Array.isArray(value.dims)) return false;
return value.dims.length === 2 && value.dims.every((dimension) => typeof dimension === 'number');
}
function isKlipyGif(value: unknown): value is KlipyGif {
if (
!isJsonRecord(value) ||
typeof value.id !== 'string' ||
typeof value.title !== 'string' ||
typeof value.itemurl !== 'string'
) {
return false;
}
return (
(value.slug === undefined || typeof value.slug === 'string') &&
(value.file === undefined || (isJsonRecord(value.file) && Object.values(value.file).every(isKlipyFileBucket))) &&
(value.media_formats === undefined ||
(isJsonRecord(value.media_formats) &&
(value.media_formats.webm === undefined || isKlipyFallbackMediaFormat(value.media_formats.webm))))
);
}
function isKlipyCategoryTag(value: unknown): value is KlipyCategoryTag {
return isJsonRecord(value) && typeof value.searchterm === 'string';
}
function readResultsArray(value: unknown): Array<unknown> {
if (!isJsonRecord(value) || !Array.isArray(value.results)) {
throw new Error('KLIPY API response did not include a results array');
}
return value.results;
}
function readTagsArray(value: unknown): Array<unknown> {
if (!isJsonRecord(value) || !Array.isArray(value.tags)) {
throw new Error('KLIPY API response did not include a tags array');
}
return value.tags;
}
type CacheEntry<T> = {
data: T;
timestamp: number;
};
interface KlipyPath {
type: 'gif' | 'clip';
slug: string;
}
type GifApiKeyResolver = () => Promise<string | null>;
export class KlipyGifProvider implements IGifProvider {
readonly meta = KLIPY_PROVIDER_META;
private readonly FEATURED_CACHE_KEY = 'klipy:featured';
private readonly TRENDING_CACHE_KEY = 'klipy:trending';
private refreshingKeys: Map<string, boolean> = new Map();
constructor(
private cacheService: ICacheService,
private mediaService: IMediaService,
private apiKeyResolver: GifApiKeyResolver = async () => Config.klipy.apiKey || null,
) {}
async isAvailable(): Promise<boolean> {
return Boolean(await this.apiKeyResolver());
}
private async getApiKey(): Promise<string> {
const apiKey = await this.apiKeyResolver();
if (!apiKey) {
throw new Error('KLIPY API key is not configured');
}
return apiKey;
}
private createURL({endpoint, params}: {endpoint: string; params: Record<string, string | number | undefined>}): URL {
const url = new URL(`${KLIPY_BASE_URL}/${endpoint}`);
const defaultParams = {
client_key: CLIENT_KEY,
contentfilter: DEFAULT_CONTENT_FILTER,
...params,
};
for (const [key, value] of Object.entries(defaultParams)) {
if (value !== undefined) {
url.searchParams.append(key, value.toString());
}
}
return url;
}
private async fetchKlipyData(url: URL): Promise<unknown> {
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
try {
const response = await fetch(url.toString(), {
headers: {'User-Agent': FLUXER_USER_AGENT},
signal: AbortSignal.timeout(ms('30 seconds')),
});
if (!response.ok) {
throw new Error(`Failed to fetch KLIPY data: ${response.statusText}`);
}
const responseText = await FetchUtils.streamToStringWithLimit(response.body, {
maxBytes: EXTERNAL_RESPONSE_LIMITS.klipyApiBytes,
headers: response.headers,
url: response.url,
description: 'KLIPY API response',
});
return parseJsonUnknown(responseText);
} catch (error) {
if (attempt < MAX_RETRIES - 1) {
const delay = BACKOFF_BASE_DELAY * 2 ** attempt;
await new Promise((resolve) => setTimeout(resolve, delay));
} else {
throw error;
}
}
}
throw new Error('Exceeded maximum retries');
}
private async fetchAndTransformGifs(url: URL): Promise<Array<GifResponse>> {
const results = readResultsArray(await this.fetchKlipyData(url)).filter(isKlipyGif);
return results.map((gif) => this.transformKlipyGif(gif)).filter((gif): gif is GifResponse => gif !== null);
}
private async getCache<T>(key: string): Promise<{
data: T;
isStale: boolean;
} | null> {
const cached = await this.cacheService.get<CacheEntry<T>>(key);
if (!cached) return null;
const age = Date.now() - cached.timestamp;
const isStale = age > CACHE_EXPIRATION_TIME;
return {data: cached.data, isStale};
}
private async setCache<T>(key: string, data: T): Promise<void> {
const cacheEntry: CacheEntry<T> = {
data,
timestamp: Date.now(),
};
await this.cacheService.set(key, cacheEntry);
}
private triggerBackgroundRefresh<T>(key: string, refreshFn: () => Promise<T>): void {
if (this.refreshingKeys.get(key)) {
return;
}
this.refreshingKeys.set(key, true);
setImmediate(async () => {
try {
const freshData = await refreshFn();
await this.setCache(key, freshData);
} catch (error) {
Logger.debug({key, error}, `Background refresh failed for key ${key}`);
} finally {
this.refreshingKeys.delete(key);
}
});
}
async search(params: {q: string; locale: string; country: string}): Promise<Array<GifResponse>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'search',
params: {
key: apiKey,
q: params.q,
country: params.country,
locale: params.locale,
limit: 50,
},
});
return this.fetchAndTransformGifs(url);
}
async registerShare(params: {id: string; q: string; locale: string; country: string}): Promise<void> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'registershare',
params: {
key: apiKey,
id: params.id,
country: params.country,
locale: params.locale,
q: params.q,
},
});
await fetch(url.toString(), {
headers: {'User-Agent': FLUXER_USER_AGENT},
signal: AbortSignal.timeout(ms('30 seconds')),
});
}
async getFeatured(params: {locale: string; country: string}): Promise<{
gifs: Array<GifResponse>;
categories: Array<GifCategoryTagResponse>;
}> {
const cached = await this.getCache<{
gifs: Array<GifResponse>;
categories: Array<GifCategoryTagResponse>;
}>(this.FEATURED_CACHE_KEY);
if (cached) {
if (cached.isStale) {
this.triggerBackgroundRefresh(this.FEATURED_CACHE_KEY, () => this.fetchFeaturedData(params));
}
return cached.data;
}
const data = await this.fetchFeaturedData(params);
await this.setCache(this.FEATURED_CACHE_KEY, data);
return data;
}
private async fetchFeaturedData(params: {locale: string; country: string}): Promise<{
gifs: Array<GifResponse>;
categories: Array<GifCategoryTagResponse>;
}> {
const [gifs, categories] = await Promise.all([this.getFeaturedGifs(params), this.getFeaturedCategories(params)]);
return {gifs, categories};
}
async getTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
const cached = await this.getCache<Array<GifResponse>>(this.TRENDING_CACHE_KEY);
if (cached) {
if (cached.isStale) {
this.triggerBackgroundRefresh(this.TRENDING_CACHE_KEY, () => this.fetchTrendingGifs(params));
}
return cached.data;
}
const gifs = await this.fetchTrendingGifs(params);
await this.setCache(this.TRENDING_CACHE_KEY, gifs);
return gifs;
}
private async fetchTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'featured',
params: {
key: apiKey,
country: params.country,
locale: params.locale,
limit: 50,
},
});
return this.fetchAndTransformGifs(url);
}
async suggest(params: {q: string; locale: string}): Promise<Array<string>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'autocomplete',
params: {
key: apiKey,
q: params.q,
locale: params.locale,
},
});
return readResultsArray(await this.fetchKlipyData(url)).filter(
(result): result is string => typeof result === 'string',
);
}
async resolveByUrl(params: {url: string; locale: string; country: string}): Promise<GifResponse | null> {
const slug = this.extractSlugFromUrl(params.url);
if (!slug) return null;
const results = await this.search({q: slug, locale: params.locale, country: params.country});
return results.find((gif) => gif.slug === slug || this.extractSlugFromUrl(gif.url) === slug) ?? results[0] ?? null;
}
private async getFeaturedGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'featured',
params: {
key: apiKey,
country: params.country,
locale: params.locale,
limit: 1,
},
});
return this.fetchAndTransformGifs(url);
}
private async getFeaturedCategories(params: {
locale: string;
country: string;
}): Promise<Array<GifCategoryTagResponse>> {
trackSeenLocale(this.cacheService, this.meta.name, params.locale, params.country).catch((error) => {
Logger.debug({err: error, ...params}, 'Failed to track seen GIF locale');
});
const rawTags = await this.fetchRawCategoryTags(params);
const cached = await readEnrichedCategoriesCache(this.cacheService, this.meta.name, params.locale, params.country);
if (cached) {
if (cached.isStale) {
this.scheduleEnrichmentRefresh(params);
}
const byName = new Map(cached.data.map((entry) => [entry.name, entry]));
return rawTags.map((tag) => byName.get(tag.name) ?? tag);
}
this.scheduleEnrichmentRefresh(params);
return rawTags;
}
private async fetchRawCategoryTags(params: {
locale: string;
country: string;
}): Promise<Array<GifCategoryTagResponse>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'categories',
params: {
key: apiKey,
country: params.country,
locale: params.locale,
type: 'featured',
},
});
const tags = readTagsArray(await this.fetchKlipyData(url)).filter(isKlipyCategoryTag);
return tags
.filter((tag) => Boolean(tag.searchterm))
.map((tag) => ({
name: tag.searchterm,
src: '',
proxy_src: '',
gif: null,
}));
}
private scheduleEnrichmentRefresh(params: {locale: string; country: string}): void {
(async () => {
try {
const workerService = getWorkerService();
await workerService.addJob('refreshGifFeaturedCategories', {
provider: this.meta.name,
locale: params.locale,
country: params.country,
});
} catch (error) {
Logger.debug({err: error, ...params}, 'Failed to enqueue GIF featured-categories refresh');
}
})();
}
async refreshFeaturedCategories(params: {locale: string; country: string}): Promise<void> {
const lockKey = refreshLockKey(this.meta.name, params.locale, params.country);
const token = await this.cacheService.acquireLock(lockKey, REFRESH_LOCK_TTL_SECONDS);
if (!token) {
Logger.debug(params, 'Skipping enriched GIF categories refresh; another worker holds the lock');
return;
}
try {
const rawTags = await this.fetchRawCategoryTags(params);
const enriched = await Promise.all(
rawTags.map(async (tag) => {
try {
const [gif] = await this.search({q: tag.name, locale: params.locale, country: params.country});
if (!gif) return tag;
return {
...tag,
src: gif.src,
proxy_src: gif.proxy_src,
gif,
};
} catch (error) {
Logger.debug({err: error, tag: tag.name, ...params}, 'Failed to enrich GIF category');
return tag;
}
}),
);
await writeEnrichedCategoriesCache(this.cacheService, this.meta.name, params.locale, params.country, enriched);
} catch (error) {
Logger.warn({err: error, ...params}, 'Failed to refresh enriched GIF categories');
throw error;
} finally {
await this.cacheService.releaseLock(lockKey, token).catch(() => undefined);
}
}
private parseKlipyPath(url: string): KlipyPath | null {
try {
const parsed = new URL(url);
const hostname = parsed.hostname.toLowerCase();
if (hostname !== 'klipy.com' && hostname !== 'www.klipy.com') return null;
const match = parsed.pathname.match(/^\/(gif|gifs|clip|clips)\/([^/]+)/i);
if (!match?.[1] || !match[2]) return null;
const type = match[1].toLowerCase().startsWith('clip') ? 'clip' : 'gif';
const slug = decodeURIComponent(match[2]).trim();
return slug ? {type, slug} : null;
} catch {
return null;
}
}
private buildKlipyPageUrl(path: KlipyPath): string {
const basePath = path.type === 'clip' ? 'clips' : 'gifs';
return `https://klipy.com/${basePath}/${encodeURIComponent(path.slug)}`;
}
private toMediaFormat(entry: KlipyFileEntry | undefined): GifMediaFormat | null {
if (!entry?.url) return null;
const width = typeof entry.width === 'number' && entry.width > 0 ? entry.width : 0;
const height = typeof entry.height === 'number' && entry.height > 0 ? entry.height : 0;
if (!width || !height) return null;
return {
src: entry.url,
proxy_src: this.mediaService.getExternalMediaProxyURL(entry.url),
width,
height,
};
}
private collectKlipyMedia(input: KlipyGif): {
media: Record<string, GifMediaFormat>;
preferred: GifMediaFormat | null;
} {
const media: Record<string, GifMediaFormat> = {};
let preferred: GifMediaFormat | null = null;
for (const size of KLIPY_SIZE_PREFERENCE) {
const bucket = input.file?.[size];
if (!bucket) continue;
for (const format of KLIPY_FORMAT_KEYS) {
const entry = this.toMediaFormat(bucket[format]);
if (!entry) continue;
const publicKey = KLIPY_PUBLIC_FORMAT_KEYS[size][format];
media[publicKey] = entry;
if (!preferred) preferred = entry;
}
}
if (Object.keys(media).length === 0 && input.media_formats?.webm) {
const webm = input.media_formats.webm;
const fallback: GifMediaFormat = {
src: webm.url,
proxy_src: this.mediaService.getExternalMediaProxyURL(webm.url),
width: webm.dims[0],
height: webm.dims[1],
};
media.webm = fallback;
preferred = fallback;
}
return {media, preferred};
}
private transformKlipyGif(input: KlipyGif): GifResponse | null {
const parsedPath = this.parseKlipyPath(input.itemurl);
const explicitSlug = input.slug?.trim();
const normalizedSlug = explicitSlug || parsedPath?.slug || input.id;
const normalizedType = parsedPath?.type ?? 'gif';
const normalizedUrl =
parsedPath || explicitSlug ? this.buildKlipyPageUrl({type: normalizedType, slug: normalizedSlug}) : input.itemurl;
const {media, preferred} = this.collectKlipyMedia(input);
const top = media.webm ?? preferred;
if (!top) return null;
return {
id: normalizedSlug,
slug: normalizedSlug,
provider: this.meta.name,
title: input.title,
url: normalizedUrl,
src: top.src,
proxy_src: top.proxy_src,
width: top.width,
height: top.height,
media,
};
}
extractSlugFromUrl(url: string): string | null {
return this.parseKlipyPath(url)?.slug ?? null;
}
buildShareUrl(slug: string): string {
const trimmed = slug.trim();
if (!trimmed) return 'https://klipy.com/gifs';
return this.buildKlipyPageUrl({type: 'gif', slug: trimmed});
}
}
@@ -0,0 +1,24 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {describe, expect, it} from 'vitest';
import {buildKlipyShareUrl, extractKlipySlugFromUrl} from './NatsGifProvider';
describe('NatsGifProvider KLIPY URL helpers', () => {
it('extracts GIF and clip slugs from KLIPY share URLs', () => {
expect(extractKlipySlugFromUrl('https://klipy.com/gifs/goatplaybanjo-chat-4')).toBe('goatplaybanjo-chat-4');
expect(extractKlipySlugFromUrl('https://www.klipy.com/gif/funny-123')).toBe('funny-123');
expect(extractKlipySlugFromUrl('https://klipy.com/clips/clip-123')).toBe('clip-123');
});
it('rejects non-KLIPY and unsupported URLs', () => {
expect(extractKlipySlugFromUrl('https://notklipy.com/gifs/funny-123')).toBeNull();
expect(extractKlipySlugFromUrl('https://klipy.com/search/funny-123')).toBeNull();
expect(extractKlipySlugFromUrl('not a url')).toBeNull();
});
it('builds canonical GIF share URLs locally', () => {
expect(buildKlipyShareUrl('goatplaybanjo-chat-4')).toBe('https://klipy.com/gifs/goatplaybanjo-chat-4');
expect(buildKlipyShareUrl(' ')).toBe('https://klipy.com/gifs');
expect(buildKlipyShareUrl('a slug')).toBe('https://klipy.com/gifs/a%20slug');
});
});
+290
View File
@@ -0,0 +1,290 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';
import {ServiceUnavailableError} from '@fluxer/errors/src/domains/core/ServiceUnavailableError';
import {
type GifCategoryTagResponse,
GifCategoryTagResponse as GifCategoryTagResponseSchema,
type GifResponse,
GifResponse as GifResponseSchema,
} from '@fluxer/schema/src/domains/gif/GifSchemas';
import type {INatsConnectionManager} from '@pkgs/nats/src/INatsConnectionManager';
import {NatsConnectionManager} from '@pkgs/nats/src/NatsConnectionManager';
import {StringCodec} from 'nats';
import {Config} from '../Config';
import {Logger} from '../Logger';
import {isJsonRecord, parseJsonUnknown} from '../utils/JsonBoundaryUtils';
import type {GifProviderMeta, IGifProvider} from './IGifProvider';
const GIF_SERVICE_SUBJECT = process.env.FLUXER_GIF_SERVICE_SUBJECT || 'svc.gifs';
const DEFAULT_GIF_SERVICE_TIMEOUT_MS = 12_000;
const DEFAULT_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS = 3_000;
const GIF_PROVIDER_META: GifProviderMeta = {
name: 'klipy',
displayName: 'KLIPY',
attributionRequired: true,
};
const KLIPY_SHARE_ORIGIN = 'https://klipy.com';
const KLIPY_SHARE_HOSTS = new Set(['klipy.com', 'www.klipy.com']);
type GifApiKeyResolver = () => Promise<string | null>;
type NatsGifRequest =
| {op: 'IsAvailable'; api_key: string | null}
| {op: 'Search'; api_key: string; q: string; locale: string; country: string}
| {op: 'GetFeatured'; api_key: string; locale: string; country: string}
| {op: 'GetTrendingGifs'; api_key: string; locale: string; country: string}
| {op: 'Suggest'; api_key: string; q: string; locale: string}
| {op: 'RegisterShare'; api_key: string; id: string; q: string; locale: string; country: string}
| {op: 'ResolveByUrl'; api_key: string; url: string; locale: string; country: string};
export function extractKlipySlugFromUrl(rawUrl: string): string | null {
let parsed: URL;
try {
parsed = new URL(rawUrl);
} catch {
return null;
}
if (!KLIPY_SHARE_HOSTS.has(parsed.hostname.toLowerCase())) {
return null;
}
const segments = parsed.pathname.split('/').filter(Boolean);
const kind = segments[0]?.toLowerCase();
const slug = segments[1]?.trim();
if (!slug) {
return null;
}
switch (kind) {
case 'gif':
case 'gifs':
case 'clip':
case 'clips':
return slug;
default:
return null;
}
}
export function buildKlipyShareUrl(slug: string): string {
const trimmed = slug.trim();
if (!trimmed) {
return `${KLIPY_SHARE_ORIGIN}/gifs`;
}
return `${KLIPY_SHARE_ORIGIN}/gifs/${encodeURIComponent(trimmed)}`;
}
function readPositiveIntegerEnv(name: string, fallback: number): number {
const value = process.env[name];
if (!value) return fallback;
const parsed = Number(value);
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
}
function readFailedMessage(value: unknown): string | null {
if (!isJsonRecord(value)) return null;
if (typeof value.error === 'string') return value.error;
if (!('Failed' in value)) return null;
const failed = value.Failed;
return isJsonRecord(failed) && typeof failed.message === 'string' ? failed.message : 'GIF service failed';
}
function readVariant(value: unknown, variant: string): unknown {
if (!isJsonRecord(value) || !(variant in value)) {
throw new ServiceUnavailableError({message: `GIF service returned an unexpected ${variant} response`});
}
return value[variant];
}
function readGifList(value: unknown, variant: string): Array<GifResponse> {
const parsed = GifResponseSchema.array().safeParse(readVariant(value, variant));
if (!parsed.success) {
throw new ServiceUnavailableError({message: `GIF service returned invalid ${variant} data`});
}
return parsed.data;
}
function readFeatured(value: unknown): {
gifs: Array<GifResponse>;
categories: Array<GifCategoryTagResponse>;
} {
const featured = readVariant(value, 'Featured');
if (!isJsonRecord(featured)) {
throw new ServiceUnavailableError({message: 'GIF service returned invalid featured data'});
}
const gifs = GifResponseSchema.array().safeParse(featured.gifs);
const categories = GifCategoryTagResponseSchema.array().safeParse(featured.categories);
if (!gifs.success || !categories.success) {
throw new ServiceUnavailableError({message: 'GIF service returned invalid featured data'});
}
return {gifs: gifs.data, categories: categories.data};
}
function readSuggestions(value: unknown): Array<string> {
const suggestions = readVariant(value, 'Suggestions');
if (!Array.isArray(suggestions) || !suggestions.every((suggestion) => typeof suggestion === 'string')) {
throw new ServiceUnavailableError({message: 'GIF service returned invalid suggestions'});
}
return suggestions;
}
function readResolved(value: unknown): GifResponse | null {
const resolved = readVariant(value, 'Resolved');
if (!isJsonRecord(resolved)) {
throw new ServiceUnavailableError({message: 'GIF service returned invalid resolved data'});
}
if (resolved.gif === null || resolved.gif === undefined) return null;
const parsed = GifResponseSchema.safeParse(resolved.gif);
if (!parsed.success) {
throw new ServiceUnavailableError({message: 'GIF service returned invalid resolved GIF'});
}
return parsed.data;
}
class NatsGifProvider implements IGifProvider {
readonly meta = GIF_PROVIDER_META;
private readonly codec = StringCodec();
constructor(
private readonly connectionManager: INatsConnectionManager,
private readonly apiKeyResolver: GifApiKeyResolver,
private readonly requestTimeoutMs = readPositiveIntegerEnv(
'FLUXER_GIF_SERVICE_TIMEOUT_MS',
DEFAULT_GIF_SERVICE_TIMEOUT_MS,
),
private readonly registerShareTimeoutMs = readPositiveIntegerEnv(
'FLUXER_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS',
DEFAULT_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS,
),
private readonly subject = GIF_SERVICE_SUBJECT,
) {}
async isAvailable(): Promise<boolean> {
return Boolean((await this.apiKeyResolver())?.trim());
}
async search(params: {q: string; locale: string; country: string}): Promise<Array<GifResponse>> {
const response = await this.request({
op: 'Search',
api_key: await this.getApiKey(),
q: params.q,
locale: params.locale,
country: params.country,
});
return readGifList(response, 'SearchResults');
}
async registerShare(params: {id: string; q: string; locale: string; country: string}): Promise<void> {
const response = await this.request(
{
op: 'RegisterShare',
api_key: await this.getApiKey(),
id: params.id,
q: params.q,
locale: params.locale,
country: params.country,
},
this.registerShareTimeoutMs,
);
if (response !== 'Registered') {
throw new ServiceUnavailableError({message: 'GIF service returned an unexpected register-share response'});
}
}
async getFeatured(params: {locale: string; country: string}): Promise<{
gifs: Array<GifResponse>;
categories: Array<GifCategoryTagResponse>;
}> {
return readFeatured(
await this.request({
op: 'GetFeatured',
api_key: await this.getApiKey(),
locale: params.locale,
country: params.country,
}),
);
}
async getTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
const response = await this.request({
op: 'GetTrendingGifs',
api_key: await this.getApiKey(),
locale: params.locale,
country: params.country,
});
return readGifList(response, 'TrendingResults');
}
async suggest(params: {q: string; locale: string}): Promise<Array<string>> {
return readSuggestions(
await this.request({
op: 'Suggest',
api_key: await this.getApiKey(),
q: params.q,
locale: params.locale,
}),
);
}
async resolveByUrl(params: {url: string; locale: string; country: string}): Promise<GifResponse | null> {
return readResolved(
await this.request({
op: 'ResolveByUrl',
api_key: await this.getApiKey(),
url: params.url,
locale: params.locale,
country: params.country,
}),
);
}
buildShareUrl(slug: string): string {
return buildKlipyShareUrl(slug);
}
extractSlugFromUrl(url: string): string | null {
return extractKlipySlugFromUrl(url);
}
private async getApiKey(): Promise<string> {
const apiKey = (await this.apiKeyResolver())?.trim();
if (!apiKey) {
throw new FeatureTemporarilyDisabledError();
}
return apiKey;
}
private async request(payload: NatsGifRequest, timeout = this.requestTimeoutMs): Promise<unknown> {
try {
if (this.connectionManager.isClosed()) {
await this.connectionManager.connect();
}
const connection = this.connectionManager.getConnection();
const response = await connection.request(this.subject, this.codec.encode(JSON.stringify(payload)), {timeout});
const decoded = this.codec.decode(response.data);
const parsed = parseJsonUnknown(decoded);
const failedMessage = readFailedMessage(parsed);
if (failedMessage) {
throw new ServiceUnavailableError({message: failedMessage});
}
return parsed;
} catch (error) {
Logger.warn({error, op: payload.op}, '[gif-service] request failed');
if (error instanceof FeatureTemporarilyDisabledError || error instanceof ServiceUnavailableError) {
throw error;
}
throw new ServiceUnavailableError({message: 'GIF service is temporarily unavailable'});
}
}
}
export function createNatsGifProvider(apiKeyResolver: GifApiKeyResolver): NatsGifProvider {
const manager = new NatsConnectionManager({
url: Config.nats.coreUrl,
token: Config.nats.authToken || undefined,
name: process.env.FLUXER_GIF_SERVICE_NATS_CLIENT_NAME || 'fluxer-api-gifs',
});
void manager.connect().catch((error) => {
Logger.warn({error}, '[gif-service] Failed to establish NATS connection');
});
return new NatsGifProvider(manager, apiKeyResolver);
}
-523
View File
@@ -1,523 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {FLUXER_USER_AGENT} from '@fluxer/constants/src/Core';
import type {GifCategoryTagResponse, GifMediaFormat, GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
import type {ICacheService} from '@pkgs/cache/src/ICacheService';
import {ms} from 'itty-time';
import {Config} from '../Config';
import type {IMediaService} from '../infrastructure/IMediaService';
import {Logger} from '../Logger';
import {getWorkerService} from '../middleware/ServiceRegistry';
import {EXTERNAL_RESPONSE_LIMITS} from '../utils/ExternalResponseLimits';
import * as FetchUtils from '../utils/FetchUtils';
import {isJsonRecord, parseJsonUnknown} from '../utils/JsonBoundaryUtils';
import {
REFRESH_LOCK_TTL_SECONDS,
readEnrichedCategoriesCache,
refreshLockKey,
trackSeenLocale,
writeEnrichedCategoriesCache,
} from './GifFeaturedCategoriesCache';
import type {GifProviderMeta, IGifProvider} from './IGifProvider';
const TENOR_BASE_URL = 'https://tenor.googleapis.com/v2';
const DEFAULT_MEDIA_FILTER = 'webm,mp4,webp,gif,tinywebm,tinymp4,tinygif,nanogif';
const PUBLIC_MEDIA_FORMATS = ['webm', 'mp4', 'webp', 'gif', 'tinywebm', 'tinymp4', 'tinygif', 'nanogif'] as const;
const DEFAULT_CONTENT_FILTER = 'low';
const CLIENT_KEY = 'fluxer';
const MAX_RETRIES = 3;
const BACKOFF_BASE_DELAY = ms('1 second');
const CACHE_EXPIRATION_TIME = ms('5 minutes');
const TENOR_PROVIDER_META: GifProviderMeta = {
name: 'tenor',
displayName: 'Tenor',
attributionRequired: false,
};
type GifApiKeyResolver = () => Promise<string | null>;
interface TenorMediaFormat {
url: string;
dims: [number, number];
}
interface TenorGif {
id: string;
title?: string;
content_description?: string;
media_formats?: Record<string, TenorMediaFormat>;
itemurl?: string;
url?: string;
}
interface TenorCategoryTag {
searchterm: string;
image: string;
}
function isTenorMediaFormat(value: unknown): value is TenorMediaFormat {
if (!isJsonRecord(value) || typeof value.url !== 'string' || !Array.isArray(value.dims)) return false;
return value.dims.length === 2 && value.dims.every((dimension) => typeof dimension === 'number');
}
function isTenorGif(value: unknown): value is TenorGif {
if (!isJsonRecord(value) || typeof value.id !== 'string') return false;
const mediaFormats = value.media_formats;
return (
(value.title === undefined || typeof value.title === 'string') &&
(value.content_description === undefined || typeof value.content_description === 'string') &&
(mediaFormats === undefined ||
(isJsonRecord(mediaFormats) && Object.values(mediaFormats).every(isTenorMediaFormat))) &&
(value.itemurl === undefined || typeof value.itemurl === 'string') &&
(value.url === undefined || typeof value.url === 'string')
);
}
function isTenorCategoryTag(value: unknown): value is TenorCategoryTag {
return isJsonRecord(value) && typeof value.searchterm === 'string' && typeof value.image === 'string';
}
function readResultsArray(value: unknown): Array<unknown> {
if (!isJsonRecord(value) || !Array.isArray(value.results)) {
throw new Error('Tenor API response did not include a results array');
}
return value.results;
}
function readTagsArray(value: unknown): Array<unknown> {
if (!isJsonRecord(value) || !Array.isArray(value.tags)) {
throw new Error('Tenor API response did not include a tags array');
}
return value.tags;
}
type CacheEntry<T> = {
data: T;
timestamp: number;
};
export class TenorGifProvider implements IGifProvider {
readonly meta = TENOR_PROVIDER_META;
private refreshingKeys: Map<string, boolean> = new Map();
private featuredCacheKey(locale: string, country: string): string {
return `tenor:featured:${locale}:${country}`;
}
private trendingCacheKey(locale: string, country: string): string {
return `tenor:trending:${locale}:${country}`;
}
constructor(
private cacheService: ICacheService,
private mediaService: IMediaService,
private apiKeyResolver: GifApiKeyResolver = async () => Config.tenor.apiKey || null,
) {}
async isAvailable(): Promise<boolean> {
return Boolean(await this.apiKeyResolver());
}
private async getApiKey(): Promise<string> {
const apiKey = await this.apiKeyResolver();
if (!apiKey) {
throw new Error('Tenor API key is not configured');
}
return apiKey;
}
private createURL({endpoint, params}: {endpoint: string; params: Record<string, string | number | undefined>}): URL {
const url = new URL(`${TENOR_BASE_URL}/${endpoint}`);
for (const [key, value] of Object.entries(params)) {
if (value !== undefined && value !== '') {
url.searchParams.append(key, value.toString());
}
}
return url;
}
private async fetchTenorData(url: URL): Promise<unknown> {
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
try {
const response = await fetch(url.toString(), {
headers: {'User-Agent': FLUXER_USER_AGENT},
signal: AbortSignal.timeout(ms('30 seconds')),
});
if (!response.ok) {
throw new Error(`Failed to fetch Tenor data: ${response.statusText}`);
}
const responseText = await FetchUtils.streamToStringWithLimit(response.body, {
maxBytes: EXTERNAL_RESPONSE_LIMITS.tenorApiBytes,
headers: response.headers,
url: response.url,
description: 'Tenor API response',
});
return parseJsonUnknown(responseText);
} catch (error) {
if (attempt < MAX_RETRIES - 1) {
const delay = BACKOFF_BASE_DELAY * 2 ** attempt;
await new Promise((resolve) => setTimeout(resolve, delay));
} else {
throw error;
}
}
}
throw new Error('Exceeded maximum retries');
}
private async fetchAndTransformGifs(url: URL): Promise<Array<GifResponse>> {
const results = readResultsArray(await this.fetchTenorData(url)).filter(isTenorGif);
return results.map((gif) => this.transformTenorGif(gif)).filter((gif): gif is GifResponse => gif !== null);
}
private async getCache<T>(key: string): Promise<{
data: T;
isStale: boolean;
} | null> {
const cached = await this.cacheService.get<CacheEntry<T>>(key);
if (!cached) return null;
const age = Date.now() - cached.timestamp;
const isStale = age > CACHE_EXPIRATION_TIME;
return {data: cached.data, isStale};
}
private async setCache<T>(key: string, data: T): Promise<void> {
const cacheEntry: CacheEntry<T> = {
data,
timestamp: Date.now(),
};
await this.cacheService.set(key, cacheEntry);
}
private triggerBackgroundRefresh<T>(key: string, refreshFn: () => Promise<T>): void {
if (this.refreshingKeys.get(key)) {
return;
}
this.refreshingKeys.set(key, true);
setImmediate(async () => {
try {
const freshData = await refreshFn();
await this.setCache(key, freshData);
} catch (error) {
Logger.debug({key, error}, `Background refresh failed for key ${key}`);
} finally {
this.refreshingKeys.delete(key);
}
});
}
async search(params: {q: string; locale: string; country: string}): Promise<Array<GifResponse>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'search',
params: {
key: apiKey,
client_key: CLIENT_KEY,
q: params.q,
country: params.country,
locale: params.locale,
contentfilter: DEFAULT_CONTENT_FILTER,
media_filter: DEFAULT_MEDIA_FILTER,
limit: 50,
},
});
return this.fetchAndTransformGifs(url);
}
async registerShare(params: {id: string; q: string; locale: string; country: string}): Promise<void> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'registershare',
params: {
key: apiKey,
client_key: CLIENT_KEY,
id: params.id,
country: params.country,
locale: params.locale,
q: params.q,
},
});
await fetch(url.toString(), {
headers: {'User-Agent': FLUXER_USER_AGENT},
signal: AbortSignal.timeout(ms('30 seconds')),
});
}
async getFeatured(params: {locale: string; country: string}): Promise<{
gifs: Array<GifResponse>;
categories: Array<GifCategoryTagResponse>;
}> {
const cacheKey = this.featuredCacheKey(params.locale, params.country);
const cached = await this.getCache<{
gifs: Array<GifResponse>;
categories: Array<GifCategoryTagResponse>;
}>(cacheKey);
if (cached) {
if (cached.isStale) {
this.triggerBackgroundRefresh(cacheKey, () => this.fetchFeaturedData(params));
}
return cached.data;
}
const data = await this.fetchFeaturedData(params);
await this.setCache(cacheKey, data);
return data;
}
private async fetchFeaturedData(params: {locale: string; country: string}): Promise<{
gifs: Array<GifResponse>;
categories: Array<GifCategoryTagResponse>;
}> {
const [gifs, categories] = await Promise.all([this.getFeaturedGifs(params), this.getFeaturedCategories(params)]);
return {gifs, categories};
}
async getTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
const cacheKey = this.trendingCacheKey(params.locale, params.country);
const cached = await this.getCache<Array<GifResponse>>(cacheKey);
if (cached) {
if (cached.isStale) {
this.triggerBackgroundRefresh(cacheKey, () => this.fetchTrendingGifs(params));
}
return cached.data;
}
const gifs = await this.fetchTrendingGifs(params);
await this.setCache(cacheKey, gifs);
return gifs;
}
private async fetchTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'featured',
params: {
key: apiKey,
client_key: CLIENT_KEY,
country: params.country,
locale: params.locale,
contentfilter: DEFAULT_CONTENT_FILTER,
media_filter: DEFAULT_MEDIA_FILTER,
limit: 50,
},
});
return this.fetchAndTransformGifs(url);
}
async suggest(params: {q: string; locale: string}): Promise<Array<string>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'search_suggestions',
params: {
key: apiKey,
client_key: CLIENT_KEY,
q: params.q,
locale: params.locale,
limit: 20,
},
});
return readResultsArray(await this.fetchTenorData(url)).filter(
(result): result is string => typeof result === 'string',
);
}
async resolveByUrl(params: {url: string; locale: string; country: string}): Promise<GifResponse | null> {
const slug = this.extractSlugFromUrl(params.url);
if (!slug) return null;
const id = this.extractIdFromSlug(slug);
if (!id) return null;
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'posts',
params: {
key: apiKey,
client_key: CLIENT_KEY,
ids: id,
country: params.country,
locale: params.locale,
media_filter: DEFAULT_MEDIA_FILTER,
},
});
const [gif] = await this.fetchAndTransformGifs(url);
return gif ?? null;
}
private async getFeaturedGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'featured',
params: {
key: apiKey,
client_key: CLIENT_KEY,
country: params.country,
locale: params.locale,
contentfilter: DEFAULT_CONTENT_FILTER,
media_filter: DEFAULT_MEDIA_FILTER,
limit: 1,
},
});
return this.fetchAndTransformGifs(url);
}
private async getFeaturedCategories(params: {
locale: string;
country: string;
}): Promise<Array<GifCategoryTagResponse>> {
trackSeenLocale(this.cacheService, this.meta.name, params.locale, params.country).catch((error) => {
Logger.debug({err: error, ...params}, 'Failed to track seen GIF locale');
});
const rawTags = await this.fetchRawCategoryTags(params);
const cached = await readEnrichedCategoriesCache(this.cacheService, this.meta.name, params.locale, params.country);
if (cached) {
if (cached.isStale) {
this.scheduleEnrichmentRefresh(params);
}
const byName = new Map(cached.data.map((entry) => [entry.name, entry]));
return rawTags.map((tag) => byName.get(tag.name) ?? tag);
}
this.scheduleEnrichmentRefresh(params);
return rawTags;
}
private async fetchRawCategoryTags(params: {
locale: string;
country: string;
}): Promise<Array<GifCategoryTagResponse>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'categories',
params: {
key: apiKey,
client_key: CLIENT_KEY,
country: params.country,
locale: params.locale,
contentfilter: DEFAULT_CONTENT_FILTER,
type: 'featured',
},
});
const tags = readTagsArray(await this.fetchTenorData(url)).filter(isTenorCategoryTag);
return tags
.filter((tag) => Boolean(tag.searchterm) && Boolean(tag.image))
.map((tag) => ({
name: tag.searchterm,
src: tag.image,
proxy_src: this.mediaService.getExternalMediaProxyURL(tag.image),
gif: null,
}));
}
private scheduleEnrichmentRefresh(params: {locale: string; country: string}): void {
(async () => {
try {
const workerService = getWorkerService();
await workerService.addJob('refreshGifFeaturedCategories', {
provider: this.meta.name,
locale: params.locale,
country: params.country,
});
} catch (error) {
Logger.debug({err: error, ...params}, 'Failed to enqueue GIF featured-categories refresh');
}
})();
}
async refreshFeaturedCategories(params: {locale: string; country: string}): Promise<void> {
const lockKey = refreshLockKey(this.meta.name, params.locale, params.country);
const token = await this.cacheService.acquireLock(lockKey, REFRESH_LOCK_TTL_SECONDS);
if (!token) {
Logger.debug(params, 'Skipping enriched GIF categories refresh; another worker holds the lock');
return;
}
try {
const rawTags = await this.fetchRawCategoryTags(params);
const enriched = await Promise.all(
rawTags.map(async (tag) => {
try {
const [gif] = await this.search({q: tag.name, locale: params.locale, country: params.country});
return {...tag, gif: gif ?? null};
} catch (error) {
Logger.debug({err: error, tag: tag.name, ...params}, 'Failed to enrich GIF category');
return tag;
}
}),
);
await writeEnrichedCategoriesCache(this.cacheService, this.meta.name, params.locale, params.country, enriched);
} catch (error) {
Logger.warn({err: error, ...params}, 'Failed to refresh enriched GIF categories');
throw error;
} finally {
await this.cacheService.releaseLock(lockKey, token).catch(() => undefined);
}
}
private selectMediaFormat(mediaFormats: Record<string, TenorMediaFormat> | undefined): TenorMediaFormat | null {
if (!mediaFormats) return null;
const preferredKeys = ['webm', 'mp4', 'tinywebm', 'tinymp4', 'webp', 'gif', 'tinygif', 'nanogif'];
for (const key of preferredKeys) {
const candidate = mediaFormats[key];
if (candidate?.url && candidate.dims?.length === 2) return candidate;
}
for (const candidate of Object.values(mediaFormats)) {
if (candidate?.url && candidate.dims?.length === 2) return candidate;
}
return null;
}
private transformTenorGif(input: TenorGif): GifResponse | null {
const best = this.selectMediaFormat(input.media_formats);
if (!best) return null;
const title = input.title?.trim() || input.content_description?.trim() || '';
const fallbackUrl = `https://tenor.com/view/${encodeURIComponent(input.id)}`;
const url = input.itemurl?.trim() || input.url?.trim() || fallbackUrl;
const slug = this.extractSlugFromUrl(url) ?? `view/${input.id}`;
const media: Record<string, GifMediaFormat> = {};
for (const key of PUBLIC_MEDIA_FORMATS) {
const candidate = input.media_formats?.[key];
if (candidate?.url && candidate.dims?.length === 2) {
media[key] = {
src: candidate.url,
proxy_src: this.mediaService.getExternalMediaProxyURL(candidate.url),
width: candidate.dims[0],
height: candidate.dims[1],
};
}
}
return {
id: input.id,
slug,
provider: this.meta.name,
title,
url,
src: best.url,
proxy_src: this.mediaService.getExternalMediaProxyURL(best.url),
width: best.dims[0],
height: best.dims[1],
media,
};
}
extractSlugFromUrl(url: string): string | null {
try {
const parsed = new URL(url);
const hostname = parsed.hostname.toLowerCase();
if (hostname !== 'tenor.com' && hostname !== 'www.tenor.com') return null;
const match = parsed.pathname.match(/^\/(?:[a-z]{2}\/)?view\/([^/]+)/i);
if (!match?.[1]) return null;
const slug = decodeURIComponent(match[1]).trim();
return slug ? `view/${slug}` : null;
} catch {
return null;
}
}
private extractIdFromSlug(slug: string): string | null {
const normalized = slug.trim().replace(/^view\//i, '');
if (!normalized) return null;
const lastDashIndex = normalized.lastIndexOf('-');
const candidate = lastDashIndex === -1 ? normalized : normalized.slice(lastDashIndex + 1);
return candidate.trim() || null;
}
buildShareUrl(slug: string): string {
const trimmed = slug.trim().replace(/^\/+|\/+$/g, '');
const normalized = trimmed.toLowerCase().startsWith('view/') ? trimmed : `view/${trimmed}`;
return `https://tenor.com/${normalized}`;
}
}
@@ -70,6 +70,7 @@ describe('NatsUnfurlerService', () => {
bypass_cache: true,
cache_only: false,
youtube_api_key: null,
klipy_api_key: null,
},
timeout: 12000,
},
@@ -19,6 +19,7 @@ interface NatsUnfurlRequest {
bypass_cache: boolean;
cache_only: boolean;
youtube_api_key: string | null;
klipy_api_key: string | null;
}
interface NatsUnfurlInnerResult {
@@ -59,6 +60,7 @@ export class NatsUnfurlerService extends IUnfurlerService {
constructor(
connectionManager: INatsConnectionManager,
private readonly resolveYoutubeApiKey: (() => Promise<string | null>) | null = null,
private readonly resolveKlipyApiKey: (() => Promise<string | null>) | null = null,
) {
super();
this.connectionManager = connectionManager;
@@ -77,6 +79,7 @@ export class NatsUnfurlerService extends IUnfurlerService {
bypass_cache: options.bypassCache === true,
cache_only: options.cacheOnly === true,
youtube_api_key: this.resolveYoutubeApiKey ? await this.resolveYoutubeApiKey() : null,
klipy_api_key: this.resolveKlipyApiKey ? await this.resolveKlipyApiKey() : null,
};
if (this.connectionManager.isClosed()) {
await this.connectionManager.connect();
@@ -99,13 +99,10 @@ interface InstanceServicesPublicConfig {
bluesky_enabled: boolean;
}
export type InstanceGifProvider = 'tenor' | 'klipy';
export type InstanceCaptchaProvider = 'hcaptcha' | 'turnstile' | 'none';
type InstanceEmailProvider = 'smtp' | 'none';
interface InstanceGifIntegrationConfig {
provider: InstanceGifProvider | null;
tenor_api_key: string | null;
klipy_api_key: string | null;
}
@@ -160,9 +157,7 @@ interface InstanceIntegrationsConfig {
bluesky: InstanceBlueskyIntegrationConfig;
}
export interface InstanceGifEffectiveConfig {
provider: InstanceGifProvider;
tenor_api_key: string | null;
interface InstanceGifEffectiveConfig {
klipy_api_key: string | null;
active_api_key: string | null;
available: boolean;
@@ -179,9 +174,6 @@ export interface InstanceCaptchaEffectiveConfig {
interface InstanceIntegrationsAdminConfig {
gif: {
provider: InstanceGifProvider | null;
effective_provider: InstanceGifProvider;
tenor_api_key_set: boolean;
klipy_api_key_set: boolean;
effective_available: boolean;
};
@@ -438,8 +430,6 @@ function normalizeInstancePolicyConfig(value: unknown): InstancePolicyConfig {
const DEFAULT_INSTANCE_INTEGRATIONS_CONFIG: InstanceIntegrationsConfig = {
gif: {
provider: null,
tenor_api_key: null,
klipy_api_key: null,
},
youtube: {
@@ -492,10 +482,6 @@ const DEFAULT_INSTANCE_MEDIA_CONFIG: InstanceMediaConfig = {
attachment_decay: DEFAULT_INSTANCE_ATTACHMENT_DECAY_CONFIG,
};
function isGifProvider(value: unknown): value is InstanceGifProvider {
return value === 'tenor' || value === 'klipy';
}
function isCaptchaProvider(value: unknown): value is InstanceCaptchaProvider {
return value === 'hcaptcha' || value === 'turnstile' || value === 'none';
}
@@ -561,8 +547,6 @@ function normalizeInstanceIntegrationsConfig(value: unknown): InstanceIntegratio
: defaults.bluesky.keys;
return {
gif: {
provider: isGifProvider(gif.provider) ? gif.provider : defaults.gif.provider,
tenor_api_key: normalizeSecretString(gif.tenor_api_key),
klipy_api_key: normalizeSecretString(gif.klipy_api_key),
},
youtube: {
@@ -1214,16 +1198,11 @@ export class InstanceConfigRepository {
async getEffectiveGifConfig(): Promise<InstanceGifEffectiveConfig> {
const integrations = await this.getInstanceIntegrationsConfig();
const provider = integrations.gif.provider ?? Config.gif.provider;
const tenorApiKey = integrations.gif.tenor_api_key ?? normalizeSecretString(Config.tenor.apiKey);
const klipyApiKey = integrations.gif.klipy_api_key ?? normalizeSecretString(Config.klipy.apiKey);
const activeApiKey = provider === 'tenor' ? tenorApiKey : klipyApiKey;
return {
provider,
tenor_api_key: tenorApiKey,
klipy_api_key: klipyApiKey,
active_api_key: activeApiKey,
available: Boolean(activeApiKey),
active_api_key: klipyApiKey,
available: Boolean(klipyApiKey),
};
}
@@ -1323,9 +1302,6 @@ export class InstanceConfigRepository {
]);
return {
gif: {
provider: integrations.gif.provider,
effective_provider: gif.provider,
tenor_api_key_set: secretIsSet(integrations.gif.tenor_api_key) || secretIsSet(Config.tenor.apiKey),
klipy_api_key_set: secretIsSet(integrations.gif.klipy_api_key) || secretIsSet(Config.klipy.apiKey),
effective_available: gif.available,
},
@@ -12,7 +12,7 @@ import {RateLimitMiddleware} from '../middleware/RateLimitMiddleware';
import {OpenAPI} from '../middleware/ResponseTypeMiddleware';
import {RateLimitConfigs} from '../RateLimitConfig';
import type {HonoEnv} from '../types/HonoEnv';
import type {InstanceCaptchaEffectiveConfig, InstanceGifEffectiveConfig} from './InstanceConfigRepository';
import type {InstanceCaptchaEffectiveConfig} from './InstanceConfigRepository';
function buildDiscoveryStaticInput(
gifService: GifService | undefined,
@@ -20,12 +20,11 @@ function buildDiscoveryStaticInput(
runtime: {
captcha: InstanceCaptchaEffectiveConfig;
emailEnabled: boolean;
gif: InstanceGifEffectiveConfig;
},
): DiscoveryStaticInput {
const apiClientEndpoint = Config.endpoints.apiClient;
const apiPublicEndpoint = Config.endpoints.apiPublic;
const activeGif = gifService?.getByName(runtime.gif.provider);
const gifProvider = gifService?.getProvider();
return {
apiCodeVersion: API_CODE_VERSION,
endpoints: {
@@ -54,9 +53,9 @@ function buildDiscoveryStaticInput(
emails_enabled: runtime.emailEnabled,
},
gif: {
provider: runtime.gif.provider,
display_name: activeGif?.meta.displayName ?? runtime.gif.provider,
attribution_required: activeGif?.meta.attributionRequired ?? false,
provider: gifProvider?.meta.name ?? 'klipy',
display_name: gifProvider?.meta.displayName ?? 'KLIPY',
attribution_required: gifProvider?.meta.attributionRequired ?? true,
},
push: {
public_vapid_key: Config.push.publicVapidKey ?? null,
@@ -86,14 +85,13 @@ export function InstanceController(app: Hono<HonoEnv>) {
const limits = limitConfigService?.getConfigWireFormat();
const sso = await ctx.get('ssoService').getPublicStatus();
const instanceConfigRepository = ctx.get('instanceConfigRepository');
const [registration, community, services, appPublicConfig, captcha, email, gif] = await Promise.all([
const [registration, community, services, appPublicConfig, captcha, email] = await Promise.all([
instanceConfigRepository.getRegistrationPublicConfig(),
instanceConfigRepository.getInstanceCommunityPublicConfig(),
instanceConfigRepository.getResolvedServicesConfig(),
instanceConfigRepository.getAppPublicConfig(),
instanceConfigRepository.getEffectiveCaptchaConfig(),
instanceConfigRepository.getEffectiveEmailConfig(),
instanceConfigRepository.getEffectiveGifConfig(),
]);
if (!limits) {
throw new Error('limit_config_service is not bound');
@@ -111,7 +109,6 @@ export function InstanceController(app: Hono<HonoEnv>) {
{
captcha,
emailEnabled: email.enabled,
gif,
},
),
{
@@ -2,6 +2,7 @@
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
import {ForbiddenError} from '@fluxer/errors/src/domains/core/ForbiddenError';
import {resolveClientIpHeaderName} from '@fluxer/ip_utils/src/ClientIp';
import {createMiddleware} from 'hono/factory';
import {Config} from '../Config';
import {Logger} from '../Logger';
@@ -23,7 +24,7 @@ const defaultExemptPaths: Array<string> = [
export function RequireClientIpMiddleware({
exemptPaths = defaultExemptPaths,
requiredHeaders = ['x-forwarded-for'],
requiredHeaders = [resolveClientIpHeaderName(Config.proxy.client_ip_header)],
}: RequireClientIpOptions = {}) {
return createMiddleware<HonoEnv>(async (ctx, next) => {
if (Config.dev.testModeEnabled) {
@@ -82,6 +82,7 @@ import {UserChannelRequestService} from '../user/services/UserChannelRequestServ
import {UserContentRequestService} from '../user/services/UserContentRequestService';
import {UserRelationshipRequestService} from '../user/services/UserRelationshipRequestService';
import {UserService} from '../user/services/UserService';
import {resolveRequestClientIp} from '../utils/IpUtils';
import {VoicePresenceHeartbeatStore} from '../voice/VoicePresenceHeartbeatStore';
import {VoiceService} from '../voice/VoiceService';
import {WebhookRequestService} from '../webhook/WebhookRequestService';
@@ -364,7 +365,7 @@ function getLiveKitWebhookService(): LiveKitWebhookService | null {
export const ServiceMiddleware = createMiddleware<HonoEnv>(async (ctx, next) => {
const apiContext = createApiContext({
requestId: ctx.get('requestId') ?? crypto.randomUUID(),
clientIp: ctx.req.header('x-forwarded-for') ?? null,
clientIp: resolveRequestClientIp(ctx.req.raw),
userAgent: ctx.req.header('user-agent') ?? null,
});
ctx.set('apiContext', apiContext);
@@ -38,8 +38,7 @@ import {DownloadService} from '../download/DownloadService';
import {createEmailProvider} from '../email/EmailProviderFactory';
import {FavoriteMemeRepository} from '../favorite_meme/FavoriteMemeRepository';
import {GifService} from '../gif/GifService';
import {KlipyGifProvider} from '../gif/KlipyGifProvider';
import {TenorGifProvider} from '../gif/TenorGifProvider';
import {createNatsGifProvider} from '../gif/NatsGifProvider';
import {GuildAuditLogService} from '../guild/GuildAuditLogService';
import {GuildDiscoveryRepository} from '../guild/repositories/GuildDiscoveryRepository';
import {GuildRepository} from '../guild/repositories/GuildRepository';
@@ -349,6 +348,7 @@ export function setInjectedUnfurlerService(service: IUnfurlerService | undefined
}
const getDefaultUnfurlerService = singleton(() => {
const instanceConfigRepository = getInstanceConfigRepository();
const manager = new NatsConnectionManager({
url: Config.nats.coreUrl,
token: Config.nats.authToken || undefined,
@@ -357,7 +357,11 @@ const getDefaultUnfurlerService = singleton(() => {
void manager.connect().catch((error) => {
Logger.error({error}, '[nats-unfurl] Failed to establish NATS connection');
});
return new NatsUnfurlerService(manager, async () => getInstanceConfigRepository().getEffectiveYoutubeApiKey());
return new NatsUnfurlerService(
manager,
async () => instanceConfigRepository.getEffectiveYoutubeApiKey(),
async () => (await instanceConfigRepository.getEffectiveGifConfig()).klipy_api_key,
);
});
export function getUnfurlerService(): IUnfurlerService {
@@ -376,24 +380,10 @@ export const getBotMfaMirrorService = singleton(
() => new BotMfaMirrorService(getApplicationRepository(), getUserRepository(), getGatewayService()),
);
export const getGifService = singleton(() => {
const cache = getCacheService();
const media = getMediaService();
const instanceConfigRepository = getInstanceConfigRepository();
return new GifService({
providers: [
new TenorGifProvider(
cache,
media,
async () => (await instanceConfigRepository.getEffectiveGifConfig()).tenor_api_key,
),
new KlipyGifProvider(
cache,
media,
async () => (await instanceConfigRepository.getEffectiveGifConfig()).klipy_api_key,
),
],
activeName: async () => (await instanceConfigRepository.getEffectiveGifConfig()).provider,
});
return new GifService(
createNatsGifProvider(async () => (await instanceConfigRepository.getEffectiveGifConfig()).klipy_api_key),
);
});
export const getExpressionAssetPurger = singleton(() => new ExpressionAssetPurger(getAssetDeletionQueue()));
export const getGuildAuditLogService = singleton(
+3 -6
View File
@@ -24808,10 +24808,7 @@
"gif": {
"type": "object",
"properties": {
"provider": {
"type": "string",
"description": "Stable machine name of the active GIF provider (e.g. \"klipy\", \"tenor\")"
},
"provider": {"type": "string", "description": "Stable machine name of the active GIF provider."},
"display_name": {"type": "string", "description": "Human-readable provider name shown in the UI"},
"attribution_required": {
"type": "boolean",
@@ -29258,7 +29255,7 @@
},
"gif_provider": {
"anyOf": [{"type": "string"}, {"type": "null"}],
"description": "Stable name of the GIF provider that issued gif_slug (e.g. \"klipy\", \"tenor\"), if any"
"description": "Stable name of the GIF provider that issued gif_slug, if any. Legacy records may contain older provider names."
},
"media": {
"anyOf": [
@@ -35767,7 +35764,7 @@
},
"gif_provider": {
"anyOf": [{"type": "string"}, {"type": "null"}],
"description": "Stable name of the GIF provider that issued gif_slug (e.g. \"klipy\", \"tenor\")"
"description": "Stable name of the GIF provider that issued gif_slug. New provider GIFs are sourced from KLIPY."
},
"media": {
"anyOf": [
@@ -12,6 +12,7 @@ import {Logger} from '../Logger';
import type {HonoApp} from '../types/HonoEnv';
import type {IUserRepository} from '../user/IUserRepository';
import {mapUserToPrivateResponse} from '../user/UserMappers';
import {resolveRequestClientIp} from '../utils/IpUtils';
interface TwilioInboundSmsWebhookContext {
authToken: string;
@@ -29,7 +30,7 @@ export function installTwilioInboundSmsWebhook(app: HonoApp, ctx: TwilioInboundS
const params = parseFormUrlEncoded(rawBody);
const signature = c.req.header('x-twilio-signature') ?? '';
if (!verifyTwilioSignature(ctx.authToken, ctx.publicWebhookUrl, params, signature)) {
Logger.warn({ip: c.req.header('cf-connecting-ip') ?? '?'}, 'Twilio webhook signature failed; rejecting');
Logger.warn({ip: resolveRequestClientIp(c.req.raw) ?? '?'}, 'Twilio webhook signature failed; rejecting');
return c.text('forbidden', 403);
}
const fromPhone = params.get('From') ?? '';
@@ -18,8 +18,6 @@ export const EXTERNAL_RESPONSE_LIMITS = {
blueskyProfileBytes: 256 * 1024,
wikipediaSummaryBytes: 256 * 1024,
oEmbedBytes: 256 * 1024,
klipyApiBytes: 512 * 1024,
tenorApiBytes: 512 * 1024,
pwnedPasswordsBytes: 1024 * 1024,
rdapBytes: 512 * 1024,
externalTemplateBytes: 512 * 1024,
+8 -7
View File
@@ -16,16 +16,17 @@ interface GetIpAddressReverseOptions {
cacheTtlSeconds?: number;
}
export function resolveRequestClientIp(req: Request): string | null {
return extractClientIp(req, {
trustClientIpHeader: Config.proxy.trust_client_ip_header,
clientIpHeaderName: Config.proxy.client_ip_header,
});
}
export async function lookupGeoip(req: Request): Promise<GeoipResult>;
export async function lookupGeoip(ip: string): Promise<GeoipResult>;
export async function lookupGeoip(input: string | Request): Promise<GeoipResult> {
const ip =
typeof input === 'string'
? input
: extractClientIp(input, {
trustClientIpHeader: Config.proxy.trust_client_ip_header,
clientIpHeaderName: Config.proxy.client_ip_header,
});
const ip = typeof input === 'string' ? input : resolveRequestClientIp(input);
if (!ip) {
return {countryCode: null, normalizedIp: null, city: null, region: null, countryName: null};
}
@@ -63,7 +63,6 @@ const LANE_CONFIG = {
batch: {
consumerName: 'workers_batch',
tasks: [
'enqueueGifFeaturedCategoriesRefresh',
'expireAttachments',
'indexChannelMessages',
'indexGuildMembers',
@@ -74,7 +73,6 @@ const LANE_CONFIG = {
'processPendingBulkMessageDeletions',
'processPremiumStateReconciliationQueue',
'prunePostgresKvTtl',
'refreshGifFeaturedCategories',
'refreshSearchIndex',
'syncDiscoveryIndex',
'syncDisposableEmailDomains',
-1
View File
@@ -52,7 +52,6 @@ function registerCronJobs(cron: CronScheduler): void {
cron.upsert('prunePostgresKvTtl', 'prunePostgresKvTtl', {}, '0 */5 * * * *');
cron.upsert('syncDiscoveryIndex', 'syncDiscoveryIndex', {}, '0 */15 * * * *');
cron.upsert('syncDisposableEmailDomains', 'syncDisposableEmailDomains', {}, '0 */30 * * * *');
cron.upsert('enqueueGifFeaturedCategoriesRefresh', 'enqueueGifFeaturedCategoriesRefresh', {}, '0 */30 * * * *');
cron.upsert('syncUrlBlocklists', 'syncUrlBlocklists', {}, '0 0 */6 * * *');
cron.upsert('syncFileShaBlocklists', 'syncFileShaBlocklists', {}, '0 0 */12 * * *');
cron.upsert('flushUserActivityBuffer', 'flushUserActivityBuffer', {}, '*/10 * * * * *');
@@ -13,7 +13,6 @@ import bulkDeleteSelfMessagesImmediate from './tasks/BulkDeleteSelfMessagesImmed
import bulkDeleteUserMessages from './tasks/BulkDeleteUserMessages';
import bulkDeleteUserMessagesScoped from './tasks/BulkDeleteUserMessagesScoped';
import deleteUserMessagesInGuildByTime from './tasks/DeleteUserMessagesInGuildByTime';
import enqueueGifFeaturedCategoriesRefresh from './tasks/EnqueueGifFeaturedCategoriesRefresh';
import expireAttachments from './tasks/ExpireAttachments';
import extractEmbeds from './tasks/ExtractEmbeds';
import finalizeNcmecAttachmentReport from './tasks/FinalizeNcmecAttachmentReport';
@@ -34,7 +33,6 @@ import processPremiumStateReconciliationQueue from './tasks/ProcessPremiumStateR
import processStripeWebhook from './tasks/ProcessStripeWebhook';
import prunePostgresKvTtl from './tasks/PrunePostgresKvTtl';
import reconcileUserPayments from './tasks/ReconcileUserPayments';
import refreshGifFeaturedCategories from './tasks/RefreshGifFeaturedCategories';
import refreshSearchIndex from './tasks/RefreshSearchIndex';
import revalidateUserConnections from './tasks/RevalidateUserConnections';
import {sendScheduledMessage} from './tasks/SendScheduledMessage';
@@ -60,7 +58,6 @@ export const workerTasks: Record<WorkerTaskName, WorkerTaskHandler> = {
bulkUpdateSuspiciousActivityFlags: bulkUpdateSuspiciousActivityFlags,
bulkUpdateUserFlags: bulkUpdateUserFlags,
deleteUserMessagesInGuildByTime,
enqueueGifFeaturedCategoriesRefresh,
expireAttachments,
extractEmbeds,
finalizeNcmecAttachmentReport,
@@ -80,7 +77,6 @@ export const workerTasks: Record<WorkerTaskName, WorkerTaskHandler> = {
processPremiumStateReconciliationQueue,
reconcileUserPayments,
prunePostgresKvTtl,
refreshGifFeaturedCategories,
refreshSearchIndex,
revalidateUserConnections,
sendScheduledMessage,
@@ -1,39 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
import {listSeenLocales} from '../../gif/GifFeaturedCategoriesCache';
import {getGifService} from '../../middleware/ServiceSingletons';
import {getWorkerDependencies} from '../WorkerContext';
const enqueueGifFeaturedCategoriesRefresh: WorkerTaskHandler = async (_payload, helpers) => {
const {cacheService, workerService} = getWorkerDependencies();
const gifService = getGifService();
for (const provider of gifService.listProviders()) {
if (!(await provider.isAvailable())) continue;
const locales = await listSeenLocales(cacheService, provider.meta.name);
if (locales.length === 0) {
helpers.logger.debug({provider: provider.meta.name}, 'No seen locales for GIF provider; nothing to refresh');
continue;
}
helpers.logger.debug(
{provider: provider.meta.name, count: locales.length},
'Fanning out enriched GIF categories refresh',
);
for (const {locale, country} of locales) {
try {
await workerService.addJob('refreshGifFeaturedCategories', {
provider: provider.meta.name,
locale,
country,
});
} catch (error) {
helpers.logger.warn(
{err: error, provider: provider.meta.name, locale, country},
'Failed to enqueue per-locale GIF categories refresh',
);
}
}
}
};
export default enqueueGifFeaturedCategoriesRefresh;
@@ -1,33 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
import {getGifService} from '../../middleware/ServiceSingletons';
interface RefreshPayload {
provider?: string;
locale?: string;
country?: string;
}
const refreshGifFeaturedCategories: WorkerTaskHandler = async (rawPayload, helpers) => {
const payload = (rawPayload ?? {}) as RefreshPayload;
const {provider, locale, country} = payload;
if (!provider || !locale || !country) {
helpers.logger.warn({payload}, 'refreshGifFeaturedCategories called without required fields');
return;
}
const gifService = getGifService();
const target = gifService.getByName(provider);
if (!target) {
helpers.logger.warn({provider}, 'refreshGifFeaturedCategories: unknown provider');
return;
}
if (!(await target.isAvailable())) {
helpers.logger.debug({provider}, 'refreshGifFeaturedCategories: provider not configured, skipping');
return;
}
helpers.logger.debug({provider, locale, country}, 'Refreshing enriched GIF featured categories');
await target.refreshFeaturedCategories({locale, country});
};
export default refreshGifFeaturedCategories;