mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-17 21:15:32 +09:00
fix: migrate gifs to klipy service (#1220)
This commit is contained in:
1 parent
99eedf6d1e
commit
4a741d8396
79 files changed
+3032
-1653
No files matched your search
@@ -360,15 +360,9 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
|
||||
bluesky: master.auth.bluesky as BlueskyOAuthConfig,
|
||||
},
|
||||
cookie: master.cookie,
|
||||
gif: {
|
||||
provider: master.integrations.gif.provider,
|
||||
},
|
||||
klipy: {
|
||||
apiKey: master.integrations.klipy.api_key,
|
||||
},
|
||||
tenor: {
|
||||
apiKey: master.integrations.tenor.api_key,
|
||||
},
|
||||
youtube: {
|
||||
apiKey: master.integrations.youtube.api_key,
|
||||
},
|
||||
|
||||
@@ -290,8 +290,6 @@ export function InstanceConfigAdminController(app: HonoApp) {
|
||||
await instanceConfigRepository.setInstanceIntegrationsConfig({
|
||||
gif: data.integrations.gif
|
||||
? omitUndefinedFields({
|
||||
provider: readOptionalField(data.integrations.gif, 'provider'),
|
||||
tenor_api_key: readOptionalField(data.integrations.gif, 'tenor_api_key'),
|
||||
klipy_api_key: readOptionalField(data.integrations.gif, 'klipy_api_key'),
|
||||
})
|
||||
: undefined,
|
||||
|
||||
@@ -68,7 +68,7 @@ export function configureMiddleware(routes: HonoApp, options: MiddlewarePipeline
|
||||
enabled: true,
|
||||
logger,
|
||||
trustClientIpHeader,
|
||||
clientIpHeaderName: clientIpHeaderName ?? 'x-forwarded-for',
|
||||
clientIpHeaderName: resolvedHeader,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -271,15 +271,9 @@ export interface APIConfig {
|
||||
domain: string;
|
||||
secure: boolean;
|
||||
};
|
||||
gif: {
|
||||
provider: 'klipy' | 'tenor';
|
||||
};
|
||||
klipy: {
|
||||
apiKey?: string;
|
||||
};
|
||||
tenor: {
|
||||
apiKey?: string;
|
||||
};
|
||||
youtube: {
|
||||
apiKey?: string;
|
||||
};
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import {ResolveGifUrlsBodySchema, ResolveGifUrlsResponse} from '@fluxer/schema/src/domains/gif/FavoriteGifSchemas';
|
||||
import type {Context} from 'hono';
|
||||
import {resolveGifRequestCountry} from '../gif/GifRequestCountry';
|
||||
import {DefaultUserOnly, LoginRequired} from '../middleware/AuthMiddleware';
|
||||
import {RateLimitMiddleware} from '../middleware/RateLimitMiddleware';
|
||||
import {OpenAPI} from '../middleware/ResponseTypeMiddleware';
|
||||
@@ -11,8 +12,8 @@ import type {HonoApp, HonoEnv} from '../types/HonoEnv';
|
||||
import {Validator} from '../Validator';
|
||||
import {resolveFavoriteGifEntry} from './FavoriteGifResolver';
|
||||
|
||||
function getCountry(ctx: Context<HonoEnv>): string {
|
||||
return ctx.req.header('CF-IPCountry') || 'US';
|
||||
async function getCountry(ctx: Context<HonoEnv>): Promise<string> {
|
||||
return resolveGifRequestCountry(ctx.req.raw);
|
||||
}
|
||||
|
||||
function getLocale(ctx: Context<HonoEnv>): string {
|
||||
@@ -42,7 +43,7 @@ export function FavoriteGifController(app: HonoApp) {
|
||||
const gifService = ctx.get('gifService');
|
||||
const unfurlerService = getUnfurlerService();
|
||||
const locale = getLocale(ctx);
|
||||
const country = getCountry(ctx);
|
||||
const country = await getCountry(ctx);
|
||||
const entries = await Promise.all(
|
||||
urls.map((url) => resolveFavoriteGifEntry({url, locale, country, gifService, mediaService, unfurlerService})),
|
||||
);
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
|
||||
import {describe, expect, it, vi} from 'vitest';
|
||||
import {GifService} from '../gif/GifService';
|
||||
import type {IGifProvider} from '../gif/IGifProvider';
|
||||
import type {IMediaService} from '../infrastructure/IMediaService';
|
||||
import type {IUnfurlerService} from '../infrastructure/IUnfurlerService';
|
||||
import {resolveFavoriteGifEntry} from './FavoriteGifResolver';
|
||||
|
||||
function createProvider(gif: GifResponse): IGifProvider {
|
||||
return {
|
||||
meta: {
|
||||
name: 'klipy',
|
||||
displayName: 'KLIPY',
|
||||
attributionRequired: true,
|
||||
},
|
||||
isAvailable: async () => true,
|
||||
search: async () => [],
|
||||
registerShare: async () => undefined,
|
||||
getFeatured: async () => ({gifs: [], categories: []}),
|
||||
getTrendingGifs: async () => [],
|
||||
suggest: async () => [],
|
||||
resolveByUrl: async () => gif,
|
||||
buildShareUrl: (slug) => `https://klipy.com/gifs/${slug}`,
|
||||
extractSlugFromUrl: () => gif.slug,
|
||||
};
|
||||
}
|
||||
|
||||
describe('resolveFavoriteGifEntry', () => {
|
||||
it('returns provider GIF media without probing generic media fallbacks', async () => {
|
||||
const gif: GifResponse = {
|
||||
id: 'goatplaybanjo-chat-4',
|
||||
slug: 'goatplaybanjo-chat-4',
|
||||
provider: 'klipy',
|
||||
title: 'Goatplaybanjo Chat',
|
||||
url: 'https://klipy.com/gifs/goatplaybanjo-chat-4',
|
||||
src: 'https://static.klipy.example/fallback.gif',
|
||||
proxy_src: 'https://media.example/fallback.gif',
|
||||
width: 120,
|
||||
height: 100,
|
||||
media: {
|
||||
tinygif: {
|
||||
src: 'https://static.klipy.example/tiny.gif',
|
||||
proxy_src: 'https://media.example/tiny.gif',
|
||||
width: 80,
|
||||
height: 60,
|
||||
},
|
||||
webm: {
|
||||
src: 'https://static.klipy.example/full.webm',
|
||||
proxy_src: 'https://media.example/full.webm',
|
||||
width: 220,
|
||||
height: 229,
|
||||
},
|
||||
},
|
||||
placeholder: 'thumbhash',
|
||||
};
|
||||
const mediaService = {
|
||||
getMetadata: vi.fn(),
|
||||
getExternalMediaProxyURL: vi.fn(),
|
||||
} as unknown as IMediaService;
|
||||
const unfurlerService = {
|
||||
unfurl: vi.fn(),
|
||||
} as unknown as IUnfurlerService;
|
||||
|
||||
await expect(
|
||||
resolveFavoriteGifEntry({
|
||||
url: gif.url,
|
||||
locale: 'en-US',
|
||||
country: 'US',
|
||||
gifService: new GifService(createProvider(gif)),
|
||||
mediaService,
|
||||
unfurlerService,
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
url: gif.url,
|
||||
proxy_url: 'https://media.example/full.webm',
|
||||
width: 220,
|
||||
height: 229,
|
||||
media: gif.media,
|
||||
content_type: 'video/webm',
|
||||
placeholder: 'thumbhash',
|
||||
});
|
||||
expect(mediaService.getMetadata).not.toHaveBeenCalled();
|
||||
expect(unfurlerService.unfurl).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -4,6 +4,7 @@ import {Logger} from '@fluxer/logger/src/Logger';
|
||||
import type {ResolvedGifEntrySchema} from '@fluxer/schema/src/domains/gif/FavoriteGifSchemas';
|
||||
import type {GifMediaFormat, GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
|
||||
import type {EmbedMediaResponse} from '@fluxer/schema/src/domains/message/EmbedSchemas';
|
||||
import {tryExtractGifProviderSlug} from '../gif/GifProviderUtils';
|
||||
import type {GifService} from '../gif/GifService';
|
||||
import type {IGifProvider} from '../gif/IGifProvider';
|
||||
import type {IMediaService, MediaProxyMetadataResponse} from '../infrastructure/IMediaService';
|
||||
@@ -91,12 +92,9 @@ async function resolveProviderGifUrl({
|
||||
country: string;
|
||||
gifService: GifService;
|
||||
}): Promise<GifResponse | null> {
|
||||
for (const provider of gifService.listProviders()) {
|
||||
if (!(await provider.isAvailable()) || !provider.extractSlugFromUrl(url)) continue;
|
||||
const gif = await resolveProviderUrl(provider, {url, locale, country});
|
||||
if (gif) return gif;
|
||||
}
|
||||
return null;
|
||||
const provider = gifService.getProvider();
|
||||
if (!(await tryExtractGifProviderSlug(provider, url))) return null;
|
||||
return resolveProviderUrl(provider, {url, locale, country});
|
||||
}
|
||||
|
||||
async function resolveProviderUrl(
|
||||
|
||||
@@ -18,8 +18,12 @@ import {createAttachmentID, createMemeID, userIdToChannelId} from '../BrandedTyp
|
||||
import {Config} from '../Config';
|
||||
import type {ChannelService} from '../channel/services/ChannelService';
|
||||
import {makeAttachmentCdnKey, makeAttachmentCdnUrl} from '../channel/services/message/MessageHelpers';
|
||||
import {
|
||||
isOptionalGifProviderError,
|
||||
type ResolvedGifProviderSlug,
|
||||
tryExtractGifProviderSlug,
|
||||
} from '../gif/GifProviderUtils';
|
||||
import type {GifService} from '../gif/GifService';
|
||||
import type {IGifProvider} from '../gif/IGifProvider';
|
||||
import type {MediaProxyMetadataResponse} from '../infrastructure/IMediaService';
|
||||
import type {IStorageService} from '../infrastructure/IStorageService';
|
||||
import type {IUnfurlerService} from '../infrastructure/IUnfurlerService';
|
||||
@@ -122,7 +126,7 @@ export class FavoriteMemeService {
|
||||
if (!message) {
|
||||
throw new UnknownMessageError();
|
||||
}
|
||||
const media = this.findMediaInMessage(message, attachmentId, embedIndex);
|
||||
const media = await this.findMediaInMessage(message, attachmentId, embedIndex);
|
||||
if (!media) {
|
||||
throw InputValidationError.fromCode('media', ValidationErrorCodes.NO_VALID_MEDIA_IN_MESSAGE);
|
||||
}
|
||||
@@ -319,18 +323,9 @@ export class FavoriteMemeService {
|
||||
throw new MediaMetadataError('URL');
|
||||
}
|
||||
let contentHash = metadata.content_hash;
|
||||
const fileData = Buffer.from(metadata.base64 ?? '', 'base64');
|
||||
const resolvedGif = this.resolveGifFromInputs({slug: gifSlug, providerName: gifProvider, url});
|
||||
const resolvedGif = await this.resolveGifFromInputs({slug: gifSlug, providerName: gifProvider, url});
|
||||
if (resolvedGif) {
|
||||
const canonicalUrl = resolvedGif.provider.buildShareUrl(resolvedGif.slug);
|
||||
const unfurled = await this.unfurlerService.unfurl(canonicalUrl, 'allow');
|
||||
if (unfurled.length > 0 && unfurled[0].video?.content_hash) {
|
||||
contentHash = unfurled[0].video.content_hash;
|
||||
Logger.debug(
|
||||
{provider: resolvedGif.provider.meta.name, gifSlug: resolvedGif.slug, contentHash},
|
||||
'Using unfurled video content_hash for provider GIF',
|
||||
);
|
||||
}
|
||||
contentHash = await this.resolveProviderGifContentHash(resolvedGif, contentHash);
|
||||
}
|
||||
const existingMemes = await this.favoriteMemeRepository.findByUserId(user.id);
|
||||
const duplicate = existingMemes.find((meme) => meme.contentHash === contentHash);
|
||||
@@ -343,6 +338,7 @@ export class FavoriteMemeService {
|
||||
const userChannelId = userIdToChannelId(user.id);
|
||||
const newAttachmentId = createAttachmentID(await this.apiContext.services.snowflake.generate());
|
||||
const storageKey = makeAttachmentCdnKey(userChannelId, newAttachmentId, filename);
|
||||
const fileData = Buffer.from(metadata.base64 ?? '', 'base64');
|
||||
await this.storageService.uploadObject({
|
||||
bucket: Config.s3.buckets.cdn,
|
||||
key: storageKey,
|
||||
@@ -470,7 +466,7 @@ export class FavoriteMemeService {
|
||||
}
|
||||
}
|
||||
|
||||
private resolveGifFromInputs({
|
||||
private async resolveGifFromInputs({
|
||||
slug,
|
||||
providerName,
|
||||
url,
|
||||
@@ -478,34 +474,51 @@ export class FavoriteMemeService {
|
||||
slug?: string | null;
|
||||
providerName?: string | null;
|
||||
url: string;
|
||||
}): {
|
||||
provider: IGifProvider;
|
||||
slug: string;
|
||||
} | null {
|
||||
}): Promise<ResolvedGifProviderSlug | null> {
|
||||
if (slug && providerName) {
|
||||
const provider = this.gifService.getByName(providerName);
|
||||
const trimmed = slug.trim();
|
||||
if (provider && trimmed) {
|
||||
const normalized = provider.extractSlugFromUrl(trimmed) ?? trimmed;
|
||||
const normalized = (await tryExtractGifProviderSlug(provider, trimmed)) ?? trimmed;
|
||||
return {provider, slug: normalized};
|
||||
}
|
||||
}
|
||||
for (const provider of this.gifService.listProviders()) {
|
||||
const extracted = provider.extractSlugFromUrl(url);
|
||||
if (extracted) {
|
||||
return {provider, slug: extracted};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
const provider = this.gifService.getProvider();
|
||||
const extracted = await tryExtractGifProviderSlug(provider, url);
|
||||
return extracted ? {provider, slug: extracted} : null;
|
||||
}
|
||||
|
||||
private detectGifFromUrl(url: string): {
|
||||
provider: IGifProvider;
|
||||
slug: string;
|
||||
} | null {
|
||||
private async detectGifFromUrl(url: string): Promise<ResolvedGifProviderSlug | null> {
|
||||
return this.resolveGifFromInputs({url});
|
||||
}
|
||||
|
||||
private async resolveProviderGifContentHash(
|
||||
resolvedGif: ResolvedGifProviderSlug,
|
||||
fallbackContentHash: string,
|
||||
): Promise<string> {
|
||||
try {
|
||||
const canonicalUrl = resolvedGif.provider.buildShareUrl(resolvedGif.slug);
|
||||
const unfurled = await this.unfurlerService.unfurl(canonicalUrl, 'allow');
|
||||
if (unfurled.length > 0 && unfurled[0].video?.content_hash) {
|
||||
Logger.debug(
|
||||
{
|
||||
provider: resolvedGif.provider.meta.name,
|
||||
gifSlug: resolvedGif.slug,
|
||||
contentHash: unfurled[0].video.content_hash,
|
||||
},
|
||||
'Using unfurled video content_hash for provider GIF',
|
||||
);
|
||||
return unfurled[0].video.content_hash;
|
||||
}
|
||||
} catch (error) {
|
||||
if (!isOptionalGifProviderError(error)) {
|
||||
throw error;
|
||||
}
|
||||
Logger.debug({error, provider: resolvedGif.provider.meta.name}, 'Skipping unavailable GIF provider enrichment');
|
||||
}
|
||||
return fallbackContentHash;
|
||||
}
|
||||
|
||||
private resolveFavoriteMemeName(name: string | undefined | null, fallbackFilename: string): string {
|
||||
const normalizedInput = typeof name === 'string' ? name.trim() : '';
|
||||
const fallbackName = fallbackFilename.trim() || 'favorite meme';
|
||||
@@ -517,11 +530,11 @@ export class FavoriteMemeService {
|
||||
return finalName;
|
||||
}
|
||||
|
||||
private findMediaInMessage(
|
||||
private async findMediaInMessage(
|
||||
message: Message,
|
||||
preferredAttachmentId?: string,
|
||||
preferredEmbedIndex?: number,
|
||||
): FavoriteMemeMedia | null {
|
||||
): Promise<FavoriteMemeMedia | null> {
|
||||
const attachments = this.getMessageAttachmentCandidates(message);
|
||||
const embeds = this.getMessageEmbedCandidates(message);
|
||||
if (preferredEmbedIndex !== undefined) {
|
||||
@@ -553,7 +566,7 @@ export class FavoriteMemeService {
|
||||
}
|
||||
}
|
||||
for (const embed of embeds) {
|
||||
const media = this.mediaFromEmbed(embed, 'media');
|
||||
const media = await this.mediaFromEmbed(embed, 'media');
|
||||
if (media) return media;
|
||||
}
|
||||
return null;
|
||||
@@ -591,7 +604,10 @@ export class FavoriteMemeService {
|
||||
};
|
||||
}
|
||||
|
||||
private mediaFromEmbed(embed: MessageEmbedCandidate, fallbackFilename: string): FavoriteMemeMedia | null {
|
||||
private async mediaFromEmbed(
|
||||
embed: MessageEmbedCandidate,
|
||||
fallbackFilename: string,
|
||||
): Promise<FavoriteMemeMedia | null> {
|
||||
const media = embed.image || embed.video || embed.thumbnail;
|
||||
if (!media?.url) {
|
||||
return null;
|
||||
@@ -603,7 +619,7 @@ export class FavoriteMemeService {
|
||||
}
|
||||
const isExternal = !this.isInternalCDNUrl(media.url);
|
||||
const isGifv = embed.type === 'gifv' || isAnimatedEmbedMedia(media.contentType, media.flags);
|
||||
const detectedGif = embed.type === 'gifv' ? this.detectGifFromUrl(media.url) : null;
|
||||
const detectedGif = embed.type === 'gifv' ? await this.detectGifFromUrl(media.url) : null;
|
||||
return {
|
||||
isExternal,
|
||||
url: media.url,
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {GIF_PROVIDER_HEADER, type GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
|
||||
import {Hono} from 'hono';
|
||||
import {describe, expect, it, vi} from 'vitest';
|
||||
import type {HonoEnv} from '../types/HonoEnv';
|
||||
import {GifController} from './GifController';
|
||||
import {GifService} from './GifService';
|
||||
import type {IGifProvider} from './IGifProvider';
|
||||
|
||||
function createProvider(gifs: Array<GifResponse>): IGifProvider {
|
||||
return {
|
||||
meta: {
|
||||
name: 'klipy',
|
||||
displayName: 'KLIPY',
|
||||
attributionRequired: true,
|
||||
},
|
||||
isAvailable: async () => true,
|
||||
search: vi.fn(async () => gifs),
|
||||
registerShare: async () => undefined,
|
||||
getFeatured: async () => ({gifs: [], categories: []}),
|
||||
getTrendingGifs: async () => [],
|
||||
suggest: async () => [],
|
||||
resolveByUrl: async () => null,
|
||||
buildShareUrl: (slug) => `https://klipy.com/gifs/${slug}`,
|
||||
extractSlugFromUrl: () => null,
|
||||
};
|
||||
}
|
||||
|
||||
function createApp(gifService: GifService): Hono<HonoEnv> {
|
||||
const app = new Hono<HonoEnv>({strict: true});
|
||||
app.use('*', async (ctx, next) => {
|
||||
ctx.set('gifService', gifService);
|
||||
ctx.set('user', {
|
||||
isBot: false,
|
||||
suspiciousActivityFlags: 0,
|
||||
} as HonoEnv['Variables']['user']);
|
||||
ctx.set('authTokenType', 'session');
|
||||
await next();
|
||||
});
|
||||
GifController(app);
|
||||
return app;
|
||||
}
|
||||
|
||||
describe('GifController', () => {
|
||||
it('serves deprecated Tenor routes from the configured KLIPY provider', async () => {
|
||||
const gifs: Array<GifResponse> = [
|
||||
{
|
||||
id: 'goatplaybanjo-chat-4',
|
||||
slug: 'goatplaybanjo-chat-4',
|
||||
provider: 'klipy',
|
||||
title: 'Goatplaybanjo Chat',
|
||||
url: 'https://klipy.com/gifs/goatplaybanjo-chat-4',
|
||||
src: 'https://static.klipy.example/full.webm',
|
||||
proxy_src: 'https://media.example/full.webm',
|
||||
width: 220,
|
||||
height: 229,
|
||||
media: {},
|
||||
placeholder: null,
|
||||
},
|
||||
];
|
||||
const provider = createProvider(gifs);
|
||||
const response = await createApp(new GifService(provider)).request('/tenor/search?q=cat&locale=en-US');
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get('Deprecation')).toBe('true');
|
||||
expect(response.headers.get(GIF_PROVIDER_HEADER)).toBe('klipy');
|
||||
await expect(response.json()).resolves.toEqual(gifs);
|
||||
expect(provider.search).toHaveBeenCalledWith({q: 'cat', locale: 'en_US', country: 'US'});
|
||||
});
|
||||
});
|
||||
@@ -17,6 +17,7 @@ import {RateLimitConfigs} from '../RateLimitConfig';
|
||||
import type {HonoApp, HonoEnv} from '../types/HonoEnv';
|
||||
import {Validator} from '../Validator';
|
||||
import {GifProviderHeaderMiddleware} from './GifProviderHeaderMiddleware';
|
||||
import {resolveGifRequestCountry} from './GifRequestCountry';
|
||||
|
||||
const TAGS = ['GIFs'];
|
||||
|
||||
@@ -34,8 +35,8 @@ const PREFIXES: ReadonlyArray<PrefixConfig> = [
|
||||
];
|
||||
const DEPRECATION_NOTICE = 'Use /gifs/* instead - these vendor-specific paths are deprecated and will be removed.';
|
||||
|
||||
function getCountry(ctx: Context<HonoEnv>): string {
|
||||
return ctx.req.header('CF-IPCountry') || 'US';
|
||||
async function getCountry(ctx: Context<HonoEnv>): Promise<string> {
|
||||
return resolveGifRequestCountry(ctx.req.raw);
|
||||
}
|
||||
|
||||
function deprecationMiddleware(deprecated: boolean): MiddlewareHandler<HonoEnv> {
|
||||
@@ -72,7 +73,8 @@ function registerRoutes(app: HonoApp, cfg: PrefixConfig) {
|
||||
async (ctx) => {
|
||||
const {q, locale} = ctx.req.valid('query');
|
||||
const provider = await ctx.get('gifService').getActive();
|
||||
return ctx.json(await provider.search({q, locale, country: getCountry(ctx)}));
|
||||
const country = await getCountry(ctx);
|
||||
return ctx.json(await provider.search({q, locale, country}));
|
||||
},
|
||||
);
|
||||
app.get(
|
||||
@@ -94,7 +96,8 @@ function registerRoutes(app: HonoApp, cfg: PrefixConfig) {
|
||||
Validator('query', GifLocaleQuery),
|
||||
async (ctx) => {
|
||||
const provider = await ctx.get('gifService').getActive();
|
||||
return ctx.json(await provider.getFeatured({locale: ctx.req.valid('query').locale, country: getCountry(ctx)}));
|
||||
const country = await getCountry(ctx);
|
||||
return ctx.json(await provider.getFeatured({locale: ctx.req.valid('query').locale, country}));
|
||||
},
|
||||
);
|
||||
const trendingPath = prefix === '/gifs' ? `${prefix}/trending` : `${prefix}/trending-gifs`;
|
||||
@@ -117,10 +120,11 @@ function registerRoutes(app: HonoApp, cfg: PrefixConfig) {
|
||||
Validator('query', GifLocaleQuery),
|
||||
async (ctx) => {
|
||||
const provider = await ctx.get('gifService').getActive();
|
||||
const country = await getCountry(ctx);
|
||||
return ctx.json(
|
||||
await provider.getTrendingGifs({
|
||||
locale: ctx.req.valid('query').locale,
|
||||
country: getCountry(ctx),
|
||||
country,
|
||||
}),
|
||||
);
|
||||
},
|
||||
@@ -145,7 +149,8 @@ function registerRoutes(app: HonoApp, cfg: PrefixConfig) {
|
||||
async (ctx) => {
|
||||
const {id, q, locale} = ctx.req.valid('json');
|
||||
const provider = await ctx.get('gifService').getActive();
|
||||
await provider.registerShare({id, q: q ?? '', locale, country: getCountry(ctx)});
|
||||
const country = await getCountry(ctx);
|
||||
await provider.registerShare({id, q: q ?? '', locale, country});
|
||||
return ctx.body(null, 204);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {GifCategoryTagResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
|
||||
import type {ICacheService} from '@pkgs/cache/src/ICacheService';
|
||||
import {ms} from 'itty-time';
|
||||
|
||||
const ENRICHED_CATEGORIES_FRESH_MS = ms('30 minutes');
|
||||
const ENRICHED_CATEGORIES_TTL_SECONDS = Math.floor(ms('6 hours') / 1000);
|
||||
const SEEN_LOCALES_TTL_SECONDS = Math.floor(ms('7 days') / 1000);
|
||||
export const REFRESH_LOCK_TTL_SECONDS = Math.floor(ms('5 minutes') / 1000);
|
||||
|
||||
interface EnrichedCategoriesEntry {
|
||||
data: Array<GifCategoryTagResponse>;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
function enrichedCategoriesCacheKey(provider: string, locale: string, country: string): string {
|
||||
return `gif:featured_categories_enriched:${provider}:${locale}:${country}`;
|
||||
}
|
||||
|
||||
function seenLocalesSetKey(provider: string): string {
|
||||
return `gif:featured_categories_seen_locales:${provider}`;
|
||||
}
|
||||
|
||||
function encodeLocaleMember(locale: string, country: string): string {
|
||||
return `${locale}|${country}`;
|
||||
}
|
||||
|
||||
function decodeLocaleMember(member: string): {
|
||||
locale: string;
|
||||
country: string;
|
||||
} | null {
|
||||
const idx = member.indexOf('|');
|
||||
if (idx <= 0) return null;
|
||||
const locale = member.slice(0, idx);
|
||||
const country = member.slice(idx + 1);
|
||||
if (!locale || !country) return null;
|
||||
return {locale, country};
|
||||
}
|
||||
|
||||
export function refreshLockKey(provider: string, locale: string, country: string): string {
|
||||
return `gif:featured_categories_refresh_lock:${provider}:${locale}:${country}`;
|
||||
}
|
||||
|
||||
export async function readEnrichedCategoriesCache(
|
||||
cache: ICacheService,
|
||||
provider: string,
|
||||
locale: string,
|
||||
country: string,
|
||||
): Promise<{
|
||||
data: Array<GifCategoryTagResponse>;
|
||||
isStale: boolean;
|
||||
} | null> {
|
||||
const entry = await cache.get<EnrichedCategoriesEntry>(enrichedCategoriesCacheKey(provider, locale, country));
|
||||
if (!entry) return null;
|
||||
const isStale = Date.now() - entry.timestamp > ENRICHED_CATEGORIES_FRESH_MS;
|
||||
return {data: entry.data, isStale};
|
||||
}
|
||||
|
||||
export async function writeEnrichedCategoriesCache(
|
||||
cache: ICacheService,
|
||||
provider: string,
|
||||
locale: string,
|
||||
country: string,
|
||||
data: Array<GifCategoryTagResponse>,
|
||||
): Promise<void> {
|
||||
const entry: EnrichedCategoriesEntry = {data, timestamp: Date.now()};
|
||||
await cache.set(enrichedCategoriesCacheKey(provider, locale, country), entry, ENRICHED_CATEGORIES_TTL_SECONDS);
|
||||
}
|
||||
|
||||
export async function trackSeenLocale(
|
||||
cache: ICacheService,
|
||||
provider: string,
|
||||
locale: string,
|
||||
country: string,
|
||||
): Promise<void> {
|
||||
await cache.sadd(seenLocalesSetKey(provider), encodeLocaleMember(locale, country), SEEN_LOCALES_TTL_SECONDS);
|
||||
}
|
||||
|
||||
export async function listSeenLocales(
|
||||
cache: ICacheService,
|
||||
provider: string,
|
||||
): Promise<
|
||||
Array<{
|
||||
locale: string;
|
||||
country: string;
|
||||
}>
|
||||
> {
|
||||
const members = await cache.smembers(seenLocalesSetKey(provider));
|
||||
const out: Array<{
|
||||
locale: string;
|
||||
country: string;
|
||||
}> = [];
|
||||
for (const member of members) {
|
||||
const decoded = decodeLocaleMember(member);
|
||||
if (decoded) out.push(decoded);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -25,7 +25,6 @@ function createProvider(): IGifProvider {
|
||||
getFeatured: async () => ({gifs: [], categories: []}),
|
||||
getTrendingGifs: async () => [],
|
||||
suggest: async () => [],
|
||||
refreshFeaturedCategories: async () => undefined,
|
||||
resolveByUrl: async () => null,
|
||||
buildShareUrl: (slug) => `https://klipy.example/${slug}`,
|
||||
extractSlugFromUrl: () => null,
|
||||
@@ -47,7 +46,7 @@ function createApp(gifService?: GifService): Hono<HonoEnv> {
|
||||
|
||||
describe('GifProviderHeaderMiddleware', () => {
|
||||
it('emits active GIF provider metadata headers', async () => {
|
||||
const gifService = new GifService({providers: [createProvider()], activeName: 'klipy'});
|
||||
const gifService = new GifService(createProvider());
|
||||
const response = await createApp(gifService).request('/probe');
|
||||
|
||||
expect(response.headers.get(GIF_PROVIDER_HEADER)).toBe('klipy');
|
||||
|
||||
@@ -13,8 +13,8 @@ export const GifProviderHeaderMiddleware = createMiddleware<HonoEnv>(async (ctx,
|
||||
await next();
|
||||
const gifService = ctx.get('gifService') as GifService | undefined;
|
||||
if (!gifService) return;
|
||||
const provider = gifService.getByName(await gifService.getActiveName());
|
||||
if (!provider || !(await provider.isAvailable())) return;
|
||||
const provider = await gifService.getActive().catch(() => null);
|
||||
if (!provider) return;
|
||||
ctx.header(GIF_PROVIDER_HEADER, provider.meta.name);
|
||||
ctx.header(GIF_PROVIDER_DISPLAY_NAME_HEADER, provider.meta.displayName);
|
||||
ctx.header(GIF_PROVIDER_ATTRIBUTION_HEADER, provider.meta.attributionRequired ? 'true' : 'false');
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {ServiceUnavailableError} from '@fluxer/errors/src/domains/core/ServiceUnavailableError';
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import {tryExtractGifProviderSlug} from './GifProviderUtils';
|
||||
import type {IGifProvider} from './IGifProvider';
|
||||
|
||||
function createProvider(overrides: Partial<IGifProvider> = {}): IGifProvider {
|
||||
return {
|
||||
meta: {
|
||||
name: 'klipy',
|
||||
displayName: 'KLIPY',
|
||||
attributionRequired: true,
|
||||
},
|
||||
isAvailable: async () => true,
|
||||
search: async () => [],
|
||||
registerShare: async () => undefined,
|
||||
getFeatured: async () => ({gifs: [], categories: []}),
|
||||
getTrendingGifs: async () => [],
|
||||
suggest: async () => [],
|
||||
resolveByUrl: async () => null,
|
||||
buildShareUrl: (slug) => `https://klipy.example/${slug}`,
|
||||
extractSlugFromUrl: () => 'slug',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('GifProviderUtils', () => {
|
||||
it('returns a trimmed slug from an available provider', async () => {
|
||||
await expect(
|
||||
tryExtractGifProviderSlug(
|
||||
createProvider({
|
||||
extractSlugFromUrl: () => ' goatplaybanjo-chat-4 ',
|
||||
}),
|
||||
'https://klipy.com/gifs/goatplaybanjo-chat-4',
|
||||
),
|
||||
).resolves.toBe('goatplaybanjo-chat-4');
|
||||
});
|
||||
|
||||
it('treats unavailable optional provider failures as no match', async () => {
|
||||
await expect(
|
||||
tryExtractGifProviderSlug(
|
||||
createProvider({
|
||||
extractSlugFromUrl: () => {
|
||||
throw new ServiceUnavailableError();
|
||||
},
|
||||
}),
|
||||
'https://klipy.com/gifs/goatplaybanjo-chat-4',
|
||||
),
|
||||
).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it('does not check availability when a provider cannot extract a slug', async () => {
|
||||
let availabilityChecks = 0;
|
||||
await expect(
|
||||
tryExtractGifProviderSlug(
|
||||
createProvider({
|
||||
isAvailable: async () => {
|
||||
availabilityChecks += 1;
|
||||
return true;
|
||||
},
|
||||
extractSlugFromUrl: () => null,
|
||||
}),
|
||||
'https://example.com/media.gif',
|
||||
),
|
||||
).resolves.toBeNull();
|
||||
|
||||
expect(availabilityChecks).toBe(0);
|
||||
});
|
||||
|
||||
it('returns null when the configured provider is unavailable', async () => {
|
||||
await expect(
|
||||
tryExtractGifProviderSlug(
|
||||
createProvider({
|
||||
isAvailable: async () => false,
|
||||
extractSlugFromUrl: () => 'matched',
|
||||
}),
|
||||
'https://klipy.com/gifs/matched',
|
||||
),
|
||||
).resolves.toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,30 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';
|
||||
import {ServiceUnavailableError} from '@fluxer/errors/src/domains/core/ServiceUnavailableError';
|
||||
import type {IGifProvider} from './IGifProvider';
|
||||
|
||||
export type ResolvedGifProviderSlug = {
|
||||
provider: IGifProvider;
|
||||
slug: string;
|
||||
};
|
||||
|
||||
export function isOptionalGifProviderError(error: unknown): boolean {
|
||||
return error instanceof FeatureTemporarilyDisabledError || error instanceof ServiceUnavailableError;
|
||||
}
|
||||
|
||||
export async function tryExtractGifProviderSlug(provider: IGifProvider, value: string): Promise<string | null> {
|
||||
try {
|
||||
const slug = provider.extractSlugFromUrl(value);
|
||||
const trimmed = slug?.trim() ?? '';
|
||||
if (trimmed.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return (await provider.isAvailable()) ? trimmed : null;
|
||||
} catch (error) {
|
||||
if (isOptionalGifProviderError(error)) {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {GeoipResult} from '@pkgs/geoip/src/GeoipLookup';
|
||||
import {beforeEach, describe, expect, it, vi} from 'vitest';
|
||||
import {resolveGifRequestCountry} from './GifRequestCountry';
|
||||
|
||||
const {lookupGeoipMock} = vi.hoisted(() => ({
|
||||
lookupGeoipMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../utils/IpUtils', () => ({
|
||||
lookupGeoip: lookupGeoipMock,
|
||||
}));
|
||||
|
||||
function geoip(countryCode: string | null): GeoipResult {
|
||||
return {
|
||||
countryCode,
|
||||
normalizedIp: '203.0.113.10',
|
||||
city: null,
|
||||
region: null,
|
||||
countryName: null,
|
||||
};
|
||||
}
|
||||
|
||||
describe('resolveGifRequestCountry', () => {
|
||||
const req = new Request('https://fluxer.test/gifs/search');
|
||||
|
||||
beforeEach(() => {
|
||||
lookupGeoipMock.mockReset();
|
||||
});
|
||||
|
||||
it('uses the GeoIP country code for GIF provider requests', async () => {
|
||||
lookupGeoipMock.mockResolvedValue(geoip('se'));
|
||||
|
||||
await expect(resolveGifRequestCountry(req)).resolves.toBe('SE');
|
||||
});
|
||||
|
||||
it('falls back to US when GeoIP has no country', async () => {
|
||||
lookupGeoipMock.mockResolvedValue(geoip(null));
|
||||
|
||||
await expect(resolveGifRequestCountry(req)).resolves.toBe('US');
|
||||
});
|
||||
|
||||
it('falls back to US when GeoIP lookup fails', async () => {
|
||||
lookupGeoipMock.mockRejectedValue(new Error('geoip unavailable'));
|
||||
|
||||
await expect(resolveGifRequestCountry(req)).resolves.toBe('US');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,14 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {lookupGeoip} from '../utils/IpUtils';
|
||||
|
||||
const KLIPY_DEFAULT_COUNTRY = 'US';
|
||||
|
||||
export async function resolveGifRequestCountry(req: Request): Promise<string> {
|
||||
try {
|
||||
const geoip = await lookupGeoip(req);
|
||||
return geoip.countryCode?.trim().toUpperCase() || KLIPY_DEFAULT_COUNTRY;
|
||||
} catch {
|
||||
return KLIPY_DEFAULT_COUNTRY;
|
||||
}
|
||||
}
|
||||
@@ -1,46 +1,27 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';
|
||||
import {Logger} from '../Logger';
|
||||
import type {IGifProvider} from './IGifProvider';
|
||||
|
||||
export class GifService {
|
||||
private readonly providersByName: Map<string, IGifProvider>;
|
||||
private readonly activeName: string | (() => Promise<string>);
|
||||
private readonly provider: IGifProvider;
|
||||
|
||||
constructor(params: {
|
||||
providers: ReadonlyArray<IGifProvider>;
|
||||
activeName: string | (() => Promise<string>);
|
||||
}) {
|
||||
this.providersByName = new Map(params.providers.map((p) => [p.meta.name, p]));
|
||||
this.activeName = params.activeName;
|
||||
if (typeof params.activeName === 'string' && !this.providersByName.has(params.activeName)) {
|
||||
Logger.warn(
|
||||
{activeName: params.activeName, registered: Array.from(this.providersByName.keys())},
|
||||
'Active GIF provider is not registered; /gifs requests will fail until configuration is fixed',
|
||||
);
|
||||
}
|
||||
constructor(provider: IGifProvider) {
|
||||
this.provider = provider;
|
||||
}
|
||||
|
||||
getProvider(): IGifProvider {
|
||||
return this.provider;
|
||||
}
|
||||
|
||||
async getActive(): Promise<IGifProvider> {
|
||||
const activeName = await this.getActiveName();
|
||||
const provider = this.providersByName.get(activeName);
|
||||
if (!provider || !(await provider.isAvailable())) {
|
||||
Logger.debug({activeName}, 'Active GIF provider unavailable');
|
||||
if (!(await this.provider.isAvailable())) {
|
||||
throw new FeatureTemporarilyDisabledError();
|
||||
}
|
||||
return provider;
|
||||
}
|
||||
|
||||
async getActiveName(): Promise<string> {
|
||||
return typeof this.activeName === 'string' ? this.activeName : this.activeName();
|
||||
return this.provider;
|
||||
}
|
||||
|
||||
getByName(name: string): IGifProvider | null {
|
||||
return this.providersByName.get(name) ?? null;
|
||||
}
|
||||
|
||||
listProviders(): ReadonlyArray<IGifProvider> {
|
||||
return Array.from(this.providersByName.values());
|
||||
return name === this.provider.meta.name ? this.provider : null;
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,6 @@ export interface IGifProvider {
|
||||
}>;
|
||||
getTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>>;
|
||||
suggest(params: {q: string; locale: string}): Promise<Array<string>>;
|
||||
refreshFeaturedCategories(params: {locale: string; country: string}): Promise<void>;
|
||||
resolveByUrl(params: {url: string; locale: string; country: string}): Promise<GifResponse | null>;
|
||||
buildShareUrl(slug: string): string;
|
||||
extractSlugFromUrl(url: string): string | null;
|
||||
|
||||
@@ -1,563 +0,0 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {FLUXER_USER_AGENT} from '@fluxer/constants/src/Core';
|
||||
import type {GifCategoryTagResponse, GifMediaFormat, GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
|
||||
import type {ICacheService} from '@pkgs/cache/src/ICacheService';
|
||||
import {ms} from 'itty-time';
|
||||
import {Config} from '../Config';
|
||||
import type {IMediaService} from '../infrastructure/IMediaService';
|
||||
import {Logger} from '../Logger';
|
||||
import {getWorkerService} from '../middleware/ServiceRegistry';
|
||||
import {EXTERNAL_RESPONSE_LIMITS} from '../utils/ExternalResponseLimits';
|
||||
import * as FetchUtils from '../utils/FetchUtils';
|
||||
import {isJsonRecord, parseJsonUnknown} from '../utils/JsonBoundaryUtils';
|
||||
import {
|
||||
REFRESH_LOCK_TTL_SECONDS,
|
||||
readEnrichedCategoriesCache,
|
||||
refreshLockKey,
|
||||
trackSeenLocale,
|
||||
writeEnrichedCategoriesCache,
|
||||
} from './GifFeaturedCategoriesCache';
|
||||
import type {GifProviderMeta, IGifProvider} from './IGifProvider';
|
||||
|
||||
const KLIPY_BASE_URL = 'https://api.klipy.com/v2';
|
||||
const DEFAULT_CONTENT_FILTER = 'low';
|
||||
const CLIENT_KEY = 'fluxer';
|
||||
const MAX_RETRIES = 3;
|
||||
const BACKOFF_BASE_DELAY = ms('1 second');
|
||||
const CACHE_EXPIRATION_TIME = ms('5 minutes');
|
||||
const KLIPY_PROVIDER_META: GifProviderMeta = {
|
||||
name: 'klipy',
|
||||
displayName: 'KLIPY',
|
||||
attributionRequired: true,
|
||||
};
|
||||
|
||||
interface KlipyFileEntry {
|
||||
url?: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
size?: number;
|
||||
}
|
||||
|
||||
type KlipyFormatKey = 'gif' | 'webp' | 'mp4' | 'webm';
|
||||
type KlipySizeKey = 'hd' | 'md' | 'sm' | 'xs';
|
||||
type KlipyFileBucket = Partial<Record<KlipyFormatKey, KlipyFileEntry>>;
|
||||
|
||||
interface KlipyGif {
|
||||
id: string;
|
||||
slug?: string;
|
||||
title: string;
|
||||
itemurl: string;
|
||||
file?: Partial<Record<KlipySizeKey, KlipyFileBucket>>;
|
||||
media_formats?: {
|
||||
webm?: {
|
||||
url: string;
|
||||
dims: [number, number];
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
const KLIPY_SIZE_PREFERENCE: ReadonlyArray<KlipySizeKey> = ['hd', 'md', 'sm', 'xs'];
|
||||
const KLIPY_FORMAT_KEYS: ReadonlyArray<KlipyFormatKey> = ['webm', 'mp4', 'webp', 'gif'];
|
||||
const KLIPY_PUBLIC_FORMAT_KEYS: Record<KlipySizeKey, Record<KlipyFormatKey, string>> = {
|
||||
hd: {webm: 'webm', mp4: 'mp4', webp: 'webp', gif: 'gif'},
|
||||
md: {webm: 'mediumwebm', mp4: 'mediummp4', webp: 'mediumwebp', gif: 'mediumgif'},
|
||||
sm: {webm: 'tinywebm', mp4: 'tinymp4', webp: 'tinywebp', gif: 'tinygif'},
|
||||
xs: {webm: 'nanowebm', mp4: 'nanomp4', webp: 'nanowebp', gif: 'nanogif'},
|
||||
};
|
||||
|
||||
interface KlipyCategoryTag {
|
||||
searchterm: string;
|
||||
}
|
||||
|
||||
function isKlipyFileEntry(value: unknown): value is KlipyFileEntry {
|
||||
return (
|
||||
isJsonRecord(value) &&
|
||||
(value.url === undefined || typeof value.url === 'string') &&
|
||||
(value.width === undefined || typeof value.width === 'number') &&
|
||||
(value.height === undefined || typeof value.height === 'number') &&
|
||||
(value.size === undefined || typeof value.size === 'number')
|
||||
);
|
||||
}
|
||||
|
||||
function isKlipyFileBucket(value: unknown): value is KlipyFileBucket {
|
||||
return isJsonRecord(value) && Object.values(value).every(isKlipyFileEntry);
|
||||
}
|
||||
|
||||
function isKlipyFallbackMediaFormat(value: unknown): value is NonNullable<KlipyGif['media_formats']>['webm'] {
|
||||
if (!isJsonRecord(value) || typeof value.url !== 'string' || !Array.isArray(value.dims)) return false;
|
||||
return value.dims.length === 2 && value.dims.every((dimension) => typeof dimension === 'number');
|
||||
}
|
||||
|
||||
function isKlipyGif(value: unknown): value is KlipyGif {
|
||||
if (
|
||||
!isJsonRecord(value) ||
|
||||
typeof value.id !== 'string' ||
|
||||
typeof value.title !== 'string' ||
|
||||
typeof value.itemurl !== 'string'
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
(value.slug === undefined || typeof value.slug === 'string') &&
|
||||
(value.file === undefined || (isJsonRecord(value.file) && Object.values(value.file).every(isKlipyFileBucket))) &&
|
||||
(value.media_formats === undefined ||
|
||||
(isJsonRecord(value.media_formats) &&
|
||||
(value.media_formats.webm === undefined || isKlipyFallbackMediaFormat(value.media_formats.webm))))
|
||||
);
|
||||
}
|
||||
|
||||
function isKlipyCategoryTag(value: unknown): value is KlipyCategoryTag {
|
||||
return isJsonRecord(value) && typeof value.searchterm === 'string';
|
||||
}
|
||||
|
||||
function readResultsArray(value: unknown): Array<unknown> {
|
||||
if (!isJsonRecord(value) || !Array.isArray(value.results)) {
|
||||
throw new Error('KLIPY API response did not include a results array');
|
||||
}
|
||||
return value.results;
|
||||
}
|
||||
|
||||
function readTagsArray(value: unknown): Array<unknown> {
|
||||
if (!isJsonRecord(value) || !Array.isArray(value.tags)) {
|
||||
throw new Error('KLIPY API response did not include a tags array');
|
||||
}
|
||||
return value.tags;
|
||||
}
|
||||
|
||||
type CacheEntry<T> = {
|
||||
data: T;
|
||||
timestamp: number;
|
||||
};
|
||||
|
||||
interface KlipyPath {
|
||||
type: 'gif' | 'clip';
|
||||
slug: string;
|
||||
}
|
||||
type GifApiKeyResolver = () => Promise<string | null>;
|
||||
|
||||
export class KlipyGifProvider implements IGifProvider {
|
||||
readonly meta = KLIPY_PROVIDER_META;
|
||||
private readonly FEATURED_CACHE_KEY = 'klipy:featured';
|
||||
private readonly TRENDING_CACHE_KEY = 'klipy:trending';
|
||||
private refreshingKeys: Map<string, boolean> = new Map();
|
||||
|
||||
constructor(
|
||||
private cacheService: ICacheService,
|
||||
private mediaService: IMediaService,
|
||||
private apiKeyResolver: GifApiKeyResolver = async () => Config.klipy.apiKey || null,
|
||||
) {}
|
||||
|
||||
async isAvailable(): Promise<boolean> {
|
||||
return Boolean(await this.apiKeyResolver());
|
||||
}
|
||||
|
||||
private async getApiKey(): Promise<string> {
|
||||
const apiKey = await this.apiKeyResolver();
|
||||
if (!apiKey) {
|
||||
throw new Error('KLIPY API key is not configured');
|
||||
}
|
||||
return apiKey;
|
||||
}
|
||||
|
||||
private createURL({endpoint, params}: {endpoint: string; params: Record<string, string | number | undefined>}): URL {
|
||||
const url = new URL(`${KLIPY_BASE_URL}/${endpoint}`);
|
||||
const defaultParams = {
|
||||
client_key: CLIENT_KEY,
|
||||
contentfilter: DEFAULT_CONTENT_FILTER,
|
||||
...params,
|
||||
};
|
||||
for (const [key, value] of Object.entries(defaultParams)) {
|
||||
if (value !== undefined) {
|
||||
url.searchParams.append(key, value.toString());
|
||||
}
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
private async fetchKlipyData(url: URL): Promise<unknown> {
|
||||
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
const response = await fetch(url.toString(), {
|
||||
headers: {'User-Agent': FLUXER_USER_AGENT},
|
||||
signal: AbortSignal.timeout(ms('30 seconds')),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch KLIPY data: ${response.statusText}`);
|
||||
}
|
||||
const responseText = await FetchUtils.streamToStringWithLimit(response.body, {
|
||||
maxBytes: EXTERNAL_RESPONSE_LIMITS.klipyApiBytes,
|
||||
headers: response.headers,
|
||||
url: response.url,
|
||||
description: 'KLIPY API response',
|
||||
});
|
||||
return parseJsonUnknown(responseText);
|
||||
} catch (error) {
|
||||
if (attempt < MAX_RETRIES - 1) {
|
||||
const delay = BACKOFF_BASE_DELAY * 2 ** attempt;
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new Error('Exceeded maximum retries');
|
||||
}
|
||||
|
||||
private async fetchAndTransformGifs(url: URL): Promise<Array<GifResponse>> {
|
||||
const results = readResultsArray(await this.fetchKlipyData(url)).filter(isKlipyGif);
|
||||
return results.map((gif) => this.transformKlipyGif(gif)).filter((gif): gif is GifResponse => gif !== null);
|
||||
}
|
||||
|
||||
private async getCache<T>(key: string): Promise<{
|
||||
data: T;
|
||||
isStale: boolean;
|
||||
} | null> {
|
||||
const cached = await this.cacheService.get<CacheEntry<T>>(key);
|
||||
if (!cached) return null;
|
||||
const age = Date.now() - cached.timestamp;
|
||||
const isStale = age > CACHE_EXPIRATION_TIME;
|
||||
return {data: cached.data, isStale};
|
||||
}
|
||||
|
||||
private async setCache<T>(key: string, data: T): Promise<void> {
|
||||
const cacheEntry: CacheEntry<T> = {
|
||||
data,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
await this.cacheService.set(key, cacheEntry);
|
||||
}
|
||||
|
||||
private triggerBackgroundRefresh<T>(key: string, refreshFn: () => Promise<T>): void {
|
||||
if (this.refreshingKeys.get(key)) {
|
||||
return;
|
||||
}
|
||||
this.refreshingKeys.set(key, true);
|
||||
setImmediate(async () => {
|
||||
try {
|
||||
const freshData = await refreshFn();
|
||||
await this.setCache(key, freshData);
|
||||
} catch (error) {
|
||||
Logger.debug({key, error}, `Background refresh failed for key ${key}`);
|
||||
} finally {
|
||||
this.refreshingKeys.delete(key);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async search(params: {q: string; locale: string; country: string}): Promise<Array<GifResponse>> {
|
||||
const apiKey = await this.getApiKey();
|
||||
const url = this.createURL({
|
||||
endpoint: 'search',
|
||||
params: {
|
||||
key: apiKey,
|
||||
q: params.q,
|
||||
country: params.country,
|
||||
locale: params.locale,
|
||||
limit: 50,
|
||||
},
|
||||
});
|
||||
return this.fetchAndTransformGifs(url);
|
||||
}
|
||||
|
||||
async registerShare(params: {id: string; q: string; locale: string; country: string}): Promise<void> {
|
||||
const apiKey = await this.getApiKey();
|
||||
const url = this.createURL({
|
||||
endpoint: 'registershare',
|
||||
params: {
|
||||
key: apiKey,
|
||||
id: params.id,
|
||||
country: params.country,
|
||||
locale: params.locale,
|
||||
q: params.q,
|
||||
},
|
||||
});
|
||||
await fetch(url.toString(), {
|
||||
headers: {'User-Agent': FLUXER_USER_AGENT},
|
||||
signal: AbortSignal.timeout(ms('30 seconds')),
|
||||
});
|
||||
}
|
||||
|
||||
async getFeatured(params: {locale: string; country: string}): Promise<{
|
||||
gifs: Array<GifResponse>;
|
||||
categories: Array<GifCategoryTagResponse>;
|
||||
}> {
|
||||
const cached = await this.getCache<{
|
||||
gifs: Array<GifResponse>;
|
||||
categories: Array<GifCategoryTagResponse>;
|
||||
}>(this.FEATURED_CACHE_KEY);
|
||||
if (cached) {
|
||||
if (cached.isStale) {
|
||||
this.triggerBackgroundRefresh(this.FEATURED_CACHE_KEY, () => this.fetchFeaturedData(params));
|
||||
}
|
||||
return cached.data;
|
||||
}
|
||||
const data = await this.fetchFeaturedData(params);
|
||||
await this.setCache(this.FEATURED_CACHE_KEY, data);
|
||||
return data;
|
||||
}
|
||||
|
||||
private async fetchFeaturedData(params: {locale: string; country: string}): Promise<{
|
||||
gifs: Array<GifResponse>;
|
||||
categories: Array<GifCategoryTagResponse>;
|
||||
}> {
|
||||
const [gifs, categories] = await Promise.all([this.getFeaturedGifs(params), this.getFeaturedCategories(params)]);
|
||||
return {gifs, categories};
|
||||
}
|
||||
|
||||
async getTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
|
||||
const cached = await this.getCache<Array<GifResponse>>(this.TRENDING_CACHE_KEY);
|
||||
if (cached) {
|
||||
if (cached.isStale) {
|
||||
this.triggerBackgroundRefresh(this.TRENDING_CACHE_KEY, () => this.fetchTrendingGifs(params));
|
||||
}
|
||||
return cached.data;
|
||||
}
|
||||
const gifs = await this.fetchTrendingGifs(params);
|
||||
await this.setCache(this.TRENDING_CACHE_KEY, gifs);
|
||||
return gifs;
|
||||
}
|
||||
|
||||
private async fetchTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
|
||||
const apiKey = await this.getApiKey();
|
||||
const url = this.createURL({
|
||||
endpoint: 'featured',
|
||||
params: {
|
||||
key: apiKey,
|
||||
country: params.country,
|
||||
locale: params.locale,
|
||||
limit: 50,
|
||||
},
|
||||
});
|
||||
return this.fetchAndTransformGifs(url);
|
||||
}
|
||||
|
||||
async suggest(params: {q: string; locale: string}): Promise<Array<string>> {
|
||||
const apiKey = await this.getApiKey();
|
||||
const url = this.createURL({
|
||||
endpoint: 'autocomplete',
|
||||
params: {
|
||||
key: apiKey,
|
||||
q: params.q,
|
||||
locale: params.locale,
|
||||
},
|
||||
});
|
||||
return readResultsArray(await this.fetchKlipyData(url)).filter(
|
||||
(result): result is string => typeof result === 'string',
|
||||
);
|
||||
}
|
||||
|
||||
async resolveByUrl(params: {url: string; locale: string; country: string}): Promise<GifResponse | null> {
|
||||
const slug = this.extractSlugFromUrl(params.url);
|
||||
if (!slug) return null;
|
||||
const results = await this.search({q: slug, locale: params.locale, country: params.country});
|
||||
return results.find((gif) => gif.slug === slug || this.extractSlugFromUrl(gif.url) === slug) ?? results[0] ?? null;
|
||||
}
|
||||
|
||||
private async getFeaturedGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
|
||||
const apiKey = await this.getApiKey();
|
||||
const url = this.createURL({
|
||||
endpoint: 'featured',
|
||||
params: {
|
||||
key: apiKey,
|
||||
country: params.country,
|
||||
locale: params.locale,
|
||||
limit: 1,
|
||||
},
|
||||
});
|
||||
return this.fetchAndTransformGifs(url);
|
||||
}
|
||||
|
||||
private async getFeaturedCategories(params: {
|
||||
locale: string;
|
||||
country: string;
|
||||
}): Promise<Array<GifCategoryTagResponse>> {
|
||||
trackSeenLocale(this.cacheService, this.meta.name, params.locale, params.country).catch((error) => {
|
||||
Logger.debug({err: error, ...params}, 'Failed to track seen GIF locale');
|
||||
});
|
||||
const rawTags = await this.fetchRawCategoryTags(params);
|
||||
const cached = await readEnrichedCategoriesCache(this.cacheService, this.meta.name, params.locale, params.country);
|
||||
if (cached) {
|
||||
if (cached.isStale) {
|
||||
this.scheduleEnrichmentRefresh(params);
|
||||
}
|
||||
const byName = new Map(cached.data.map((entry) => [entry.name, entry]));
|
||||
return rawTags.map((tag) => byName.get(tag.name) ?? tag);
|
||||
}
|
||||
this.scheduleEnrichmentRefresh(params);
|
||||
return rawTags;
|
||||
}
|
||||
|
||||
private async fetchRawCategoryTags(params: {
|
||||
locale: string;
|
||||
country: string;
|
||||
}): Promise<Array<GifCategoryTagResponse>> {
|
||||
const apiKey = await this.getApiKey();
|
||||
const url = this.createURL({
|
||||
endpoint: 'categories',
|
||||
params: {
|
||||
key: apiKey,
|
||||
country: params.country,
|
||||
locale: params.locale,
|
||||
type: 'featured',
|
||||
},
|
||||
});
|
||||
const tags = readTagsArray(await this.fetchKlipyData(url)).filter(isKlipyCategoryTag);
|
||||
return tags
|
||||
.filter((tag) => Boolean(tag.searchterm))
|
||||
.map((tag) => ({
|
||||
name: tag.searchterm,
|
||||
src: '',
|
||||
proxy_src: '',
|
||||
gif: null,
|
||||
}));
|
||||
}
|
||||
|
||||
private scheduleEnrichmentRefresh(params: {locale: string; country: string}): void {
|
||||
(async () => {
|
||||
try {
|
||||
const workerService = getWorkerService();
|
||||
await workerService.addJob('refreshGifFeaturedCategories', {
|
||||
provider: this.meta.name,
|
||||
locale: params.locale,
|
||||
country: params.country,
|
||||
});
|
||||
} catch (error) {
|
||||
Logger.debug({err: error, ...params}, 'Failed to enqueue GIF featured-categories refresh');
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
async refreshFeaturedCategories(params: {locale: string; country: string}): Promise<void> {
|
||||
const lockKey = refreshLockKey(this.meta.name, params.locale, params.country);
|
||||
const token = await this.cacheService.acquireLock(lockKey, REFRESH_LOCK_TTL_SECONDS);
|
||||
if (!token) {
|
||||
Logger.debug(params, 'Skipping enriched GIF categories refresh; another worker holds the lock');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const rawTags = await this.fetchRawCategoryTags(params);
|
||||
const enriched = await Promise.all(
|
||||
rawTags.map(async (tag) => {
|
||||
try {
|
||||
const [gif] = await this.search({q: tag.name, locale: params.locale, country: params.country});
|
||||
if (!gif) return tag;
|
||||
return {
|
||||
...tag,
|
||||
src: gif.src,
|
||||
proxy_src: gif.proxy_src,
|
||||
gif,
|
||||
};
|
||||
} catch (error) {
|
||||
Logger.debug({err: error, tag: tag.name, ...params}, 'Failed to enrich GIF category');
|
||||
return tag;
|
||||
}
|
||||
}),
|
||||
);
|
||||
await writeEnrichedCategoriesCache(this.cacheService, this.meta.name, params.locale, params.country, enriched);
|
||||
} catch (error) {
|
||||
Logger.warn({err: error, ...params}, 'Failed to refresh enriched GIF categories');
|
||||
throw error;
|
||||
} finally {
|
||||
await this.cacheService.releaseLock(lockKey, token).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
private parseKlipyPath(url: string): KlipyPath | null {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
const hostname = parsed.hostname.toLowerCase();
|
||||
if (hostname !== 'klipy.com' && hostname !== 'www.klipy.com') return null;
|
||||
const match = parsed.pathname.match(/^\/(gif|gifs|clip|clips)\/([^/]+)/i);
|
||||
if (!match?.[1] || !match[2]) return null;
|
||||
const type = match[1].toLowerCase().startsWith('clip') ? 'clip' : 'gif';
|
||||
const slug = decodeURIComponent(match[2]).trim();
|
||||
return slug ? {type, slug} : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private buildKlipyPageUrl(path: KlipyPath): string {
|
||||
const basePath = path.type === 'clip' ? 'clips' : 'gifs';
|
||||
return `https://klipy.com/${basePath}/${encodeURIComponent(path.slug)}`;
|
||||
}
|
||||
|
||||
private toMediaFormat(entry: KlipyFileEntry | undefined): GifMediaFormat | null {
|
||||
if (!entry?.url) return null;
|
||||
const width = typeof entry.width === 'number' && entry.width > 0 ? entry.width : 0;
|
||||
const height = typeof entry.height === 'number' && entry.height > 0 ? entry.height : 0;
|
||||
if (!width || !height) return null;
|
||||
return {
|
||||
src: entry.url,
|
||||
proxy_src: this.mediaService.getExternalMediaProxyURL(entry.url),
|
||||
width,
|
||||
height,
|
||||
};
|
||||
}
|
||||
|
||||
private collectKlipyMedia(input: KlipyGif): {
|
||||
media: Record<string, GifMediaFormat>;
|
||||
preferred: GifMediaFormat | null;
|
||||
} {
|
||||
const media: Record<string, GifMediaFormat> = {};
|
||||
let preferred: GifMediaFormat | null = null;
|
||||
for (const size of KLIPY_SIZE_PREFERENCE) {
|
||||
const bucket = input.file?.[size];
|
||||
if (!bucket) continue;
|
||||
for (const format of KLIPY_FORMAT_KEYS) {
|
||||
const entry = this.toMediaFormat(bucket[format]);
|
||||
if (!entry) continue;
|
||||
const publicKey = KLIPY_PUBLIC_FORMAT_KEYS[size][format];
|
||||
media[publicKey] = entry;
|
||||
if (!preferred) preferred = entry;
|
||||
}
|
||||
}
|
||||
if (Object.keys(media).length === 0 && input.media_formats?.webm) {
|
||||
const webm = input.media_formats.webm;
|
||||
const fallback: GifMediaFormat = {
|
||||
src: webm.url,
|
||||
proxy_src: this.mediaService.getExternalMediaProxyURL(webm.url),
|
||||
width: webm.dims[0],
|
||||
height: webm.dims[1],
|
||||
};
|
||||
media.webm = fallback;
|
||||
preferred = fallback;
|
||||
}
|
||||
return {media, preferred};
|
||||
}
|
||||
|
||||
private transformKlipyGif(input: KlipyGif): GifResponse | null {
|
||||
const parsedPath = this.parseKlipyPath(input.itemurl);
|
||||
const explicitSlug = input.slug?.trim();
|
||||
const normalizedSlug = explicitSlug || parsedPath?.slug || input.id;
|
||||
const normalizedType = parsedPath?.type ?? 'gif';
|
||||
const normalizedUrl =
|
||||
parsedPath || explicitSlug ? this.buildKlipyPageUrl({type: normalizedType, slug: normalizedSlug}) : input.itemurl;
|
||||
const {media, preferred} = this.collectKlipyMedia(input);
|
||||
const top = media.webm ?? preferred;
|
||||
if (!top) return null;
|
||||
return {
|
||||
id: normalizedSlug,
|
||||
slug: normalizedSlug,
|
||||
provider: this.meta.name,
|
||||
title: input.title,
|
||||
url: normalizedUrl,
|
||||
src: top.src,
|
||||
proxy_src: top.proxy_src,
|
||||
width: top.width,
|
||||
height: top.height,
|
||||
media,
|
||||
};
|
||||
}
|
||||
|
||||
extractSlugFromUrl(url: string): string | null {
|
||||
return this.parseKlipyPath(url)?.slug ?? null;
|
||||
}
|
||||
|
||||
buildShareUrl(slug: string): string {
|
||||
const trimmed = slug.trim();
|
||||
if (!trimmed) return 'https://klipy.com/gifs';
|
||||
return this.buildKlipyPageUrl({type: 'gif', slug: trimmed});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import {buildKlipyShareUrl, extractKlipySlugFromUrl} from './NatsGifProvider';
|
||||
|
||||
describe('NatsGifProvider KLIPY URL helpers', () => {
|
||||
it('extracts GIF and clip slugs from KLIPY share URLs', () => {
|
||||
expect(extractKlipySlugFromUrl('https://klipy.com/gifs/goatplaybanjo-chat-4')).toBe('goatplaybanjo-chat-4');
|
||||
expect(extractKlipySlugFromUrl('https://www.klipy.com/gif/funny-123')).toBe('funny-123');
|
||||
expect(extractKlipySlugFromUrl('https://klipy.com/clips/clip-123')).toBe('clip-123');
|
||||
});
|
||||
|
||||
it('rejects non-KLIPY and unsupported URLs', () => {
|
||||
expect(extractKlipySlugFromUrl('https://notklipy.com/gifs/funny-123')).toBeNull();
|
||||
expect(extractKlipySlugFromUrl('https://klipy.com/search/funny-123')).toBeNull();
|
||||
expect(extractKlipySlugFromUrl('not a url')).toBeNull();
|
||||
});
|
||||
|
||||
it('builds canonical GIF share URLs locally', () => {
|
||||
expect(buildKlipyShareUrl('goatplaybanjo-chat-4')).toBe('https://klipy.com/gifs/goatplaybanjo-chat-4');
|
||||
expect(buildKlipyShareUrl(' ')).toBe('https://klipy.com/gifs');
|
||||
expect(buildKlipyShareUrl('a slug')).toBe('https://klipy.com/gifs/a%20slug');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,290 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';
|
||||
import {ServiceUnavailableError} from '@fluxer/errors/src/domains/core/ServiceUnavailableError';
|
||||
import {
|
||||
type GifCategoryTagResponse,
|
||||
GifCategoryTagResponse as GifCategoryTagResponseSchema,
|
||||
type GifResponse,
|
||||
GifResponse as GifResponseSchema,
|
||||
} from '@fluxer/schema/src/domains/gif/GifSchemas';
|
||||
import type {INatsConnectionManager} from '@pkgs/nats/src/INatsConnectionManager';
|
||||
import {NatsConnectionManager} from '@pkgs/nats/src/NatsConnectionManager';
|
||||
import {StringCodec} from 'nats';
|
||||
import {Config} from '../Config';
|
||||
import {Logger} from '../Logger';
|
||||
import {isJsonRecord, parseJsonUnknown} from '../utils/JsonBoundaryUtils';
|
||||
import type {GifProviderMeta, IGifProvider} from './IGifProvider';
|
||||
|
||||
const GIF_SERVICE_SUBJECT = process.env.FLUXER_GIF_SERVICE_SUBJECT || 'svc.gifs';
|
||||
const DEFAULT_GIF_SERVICE_TIMEOUT_MS = 12_000;
|
||||
const DEFAULT_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS = 3_000;
|
||||
const GIF_PROVIDER_META: GifProviderMeta = {
|
||||
name: 'klipy',
|
||||
displayName: 'KLIPY',
|
||||
attributionRequired: true,
|
||||
};
|
||||
const KLIPY_SHARE_ORIGIN = 'https://klipy.com';
|
||||
const KLIPY_SHARE_HOSTS = new Set(['klipy.com', 'www.klipy.com']);
|
||||
|
||||
type GifApiKeyResolver = () => Promise<string | null>;
|
||||
|
||||
type NatsGifRequest =
|
||||
| {op: 'IsAvailable'; api_key: string | null}
|
||||
| {op: 'Search'; api_key: string; q: string; locale: string; country: string}
|
||||
| {op: 'GetFeatured'; api_key: string; locale: string; country: string}
|
||||
| {op: 'GetTrendingGifs'; api_key: string; locale: string; country: string}
|
||||
| {op: 'Suggest'; api_key: string; q: string; locale: string}
|
||||
| {op: 'RegisterShare'; api_key: string; id: string; q: string; locale: string; country: string}
|
||||
| {op: 'ResolveByUrl'; api_key: string; url: string; locale: string; country: string};
|
||||
|
||||
export function extractKlipySlugFromUrl(rawUrl: string): string | null {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(rawUrl);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (!KLIPY_SHARE_HOSTS.has(parsed.hostname.toLowerCase())) {
|
||||
return null;
|
||||
}
|
||||
const segments = parsed.pathname.split('/').filter(Boolean);
|
||||
const kind = segments[0]?.toLowerCase();
|
||||
const slug = segments[1]?.trim();
|
||||
if (!slug) {
|
||||
return null;
|
||||
}
|
||||
switch (kind) {
|
||||
case 'gif':
|
||||
case 'gifs':
|
||||
case 'clip':
|
||||
case 'clips':
|
||||
return slug;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function buildKlipyShareUrl(slug: string): string {
|
||||
const trimmed = slug.trim();
|
||||
if (!trimmed) {
|
||||
return `${KLIPY_SHARE_ORIGIN}/gifs`;
|
||||
}
|
||||
return `${KLIPY_SHARE_ORIGIN}/gifs/${encodeURIComponent(trimmed)}`;
|
||||
}
|
||||
|
||||
function readPositiveIntegerEnv(name: string, fallback: number): number {
|
||||
const value = process.env[name];
|
||||
if (!value) return fallback;
|
||||
const parsed = Number(value);
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function readFailedMessage(value: unknown): string | null {
|
||||
if (!isJsonRecord(value)) return null;
|
||||
if (typeof value.error === 'string') return value.error;
|
||||
if (!('Failed' in value)) return null;
|
||||
const failed = value.Failed;
|
||||
return isJsonRecord(failed) && typeof failed.message === 'string' ? failed.message : 'GIF service failed';
|
||||
}
|
||||
|
||||
function readVariant(value: unknown, variant: string): unknown {
|
||||
if (!isJsonRecord(value) || !(variant in value)) {
|
||||
throw new ServiceUnavailableError({message: `GIF service returned an unexpected ${variant} response`});
|
||||
}
|
||||
return value[variant];
|
||||
}
|
||||
|
||||
function readGifList(value: unknown, variant: string): Array<GifResponse> {
|
||||
const parsed = GifResponseSchema.array().safeParse(readVariant(value, variant));
|
||||
if (!parsed.success) {
|
||||
throw new ServiceUnavailableError({message: `GIF service returned invalid ${variant} data`});
|
||||
}
|
||||
return parsed.data;
|
||||
}
|
||||
|
||||
function readFeatured(value: unknown): {
|
||||
gifs: Array<GifResponse>;
|
||||
categories: Array<GifCategoryTagResponse>;
|
||||
} {
|
||||
const featured = readVariant(value, 'Featured');
|
||||
if (!isJsonRecord(featured)) {
|
||||
throw new ServiceUnavailableError({message: 'GIF service returned invalid featured data'});
|
||||
}
|
||||
const gifs = GifResponseSchema.array().safeParse(featured.gifs);
|
||||
const categories = GifCategoryTagResponseSchema.array().safeParse(featured.categories);
|
||||
if (!gifs.success || !categories.success) {
|
||||
throw new ServiceUnavailableError({message: 'GIF service returned invalid featured data'});
|
||||
}
|
||||
return {gifs: gifs.data, categories: categories.data};
|
||||
}
|
||||
|
||||
function readSuggestions(value: unknown): Array<string> {
|
||||
const suggestions = readVariant(value, 'Suggestions');
|
||||
if (!Array.isArray(suggestions) || !suggestions.every((suggestion) => typeof suggestion === 'string')) {
|
||||
throw new ServiceUnavailableError({message: 'GIF service returned invalid suggestions'});
|
||||
}
|
||||
return suggestions;
|
||||
}
|
||||
|
||||
function readResolved(value: unknown): GifResponse | null {
|
||||
const resolved = readVariant(value, 'Resolved');
|
||||
if (!isJsonRecord(resolved)) {
|
||||
throw new ServiceUnavailableError({message: 'GIF service returned invalid resolved data'});
|
||||
}
|
||||
if (resolved.gif === null || resolved.gif === undefined) return null;
|
||||
const parsed = GifResponseSchema.safeParse(resolved.gif);
|
||||
if (!parsed.success) {
|
||||
throw new ServiceUnavailableError({message: 'GIF service returned invalid resolved GIF'});
|
||||
}
|
||||
return parsed.data;
|
||||
}
|
||||
|
||||
class NatsGifProvider implements IGifProvider {
|
||||
readonly meta = GIF_PROVIDER_META;
|
||||
private readonly codec = StringCodec();
|
||||
|
||||
constructor(
|
||||
private readonly connectionManager: INatsConnectionManager,
|
||||
private readonly apiKeyResolver: GifApiKeyResolver,
|
||||
private readonly requestTimeoutMs = readPositiveIntegerEnv(
|
||||
'FLUXER_GIF_SERVICE_TIMEOUT_MS',
|
||||
DEFAULT_GIF_SERVICE_TIMEOUT_MS,
|
||||
),
|
||||
private readonly registerShareTimeoutMs = readPositiveIntegerEnv(
|
||||
'FLUXER_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS',
|
||||
DEFAULT_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS,
|
||||
),
|
||||
private readonly subject = GIF_SERVICE_SUBJECT,
|
||||
) {}
|
||||
|
||||
async isAvailable(): Promise<boolean> {
|
||||
return Boolean((await this.apiKeyResolver())?.trim());
|
||||
}
|
||||
|
||||
async search(params: {q: string; locale: string; country: string}): Promise<Array<GifResponse>> {
|
||||
const response = await this.request({
|
||||
op: 'Search',
|
||||
api_key: await this.getApiKey(),
|
||||
q: params.q,
|
||||
locale: params.locale,
|
||||
country: params.country,
|
||||
});
|
||||
return readGifList(response, 'SearchResults');
|
||||
}
|
||||
|
||||
async registerShare(params: {id: string; q: string; locale: string; country: string}): Promise<void> {
|
||||
const response = await this.request(
|
||||
{
|
||||
op: 'RegisterShare',
|
||||
api_key: await this.getApiKey(),
|
||||
id: params.id,
|
||||
q: params.q,
|
||||
locale: params.locale,
|
||||
country: params.country,
|
||||
},
|
||||
this.registerShareTimeoutMs,
|
||||
);
|
||||
if (response !== 'Registered') {
|
||||
throw new ServiceUnavailableError({message: 'GIF service returned an unexpected register-share response'});
|
||||
}
|
||||
}
|
||||
|
||||
async getFeatured(params: {locale: string; country: string}): Promise<{
|
||||
gifs: Array<GifResponse>;
|
||||
categories: Array<GifCategoryTagResponse>;
|
||||
}> {
|
||||
return readFeatured(
|
||||
await this.request({
|
||||
op: 'GetFeatured',
|
||||
api_key: await this.getApiKey(),
|
||||
locale: params.locale,
|
||||
country: params.country,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async getTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
|
||||
const response = await this.request({
|
||||
op: 'GetTrendingGifs',
|
||||
api_key: await this.getApiKey(),
|
||||
locale: params.locale,
|
||||
country: params.country,
|
||||
});
|
||||
return readGifList(response, 'TrendingResults');
|
||||
}
|
||||
|
||||
async suggest(params: {q: string; locale: string}): Promise<Array<string>> {
|
||||
return readSuggestions(
|
||||
await this.request({
|
||||
op: 'Suggest',
|
||||
api_key: await this.getApiKey(),
|
||||
q: params.q,
|
||||
locale: params.locale,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async resolveByUrl(params: {url: string; locale: string; country: string}): Promise<GifResponse | null> {
|
||||
return readResolved(
|
||||
await this.request({
|
||||
op: 'ResolveByUrl',
|
||||
api_key: await this.getApiKey(),
|
||||
url: params.url,
|
||||
locale: params.locale,
|
||||
country: params.country,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
buildShareUrl(slug: string): string {
|
||||
return buildKlipyShareUrl(slug);
|
||||
}
|
||||
|
||||
extractSlugFromUrl(url: string): string | null {
|
||||
return extractKlipySlugFromUrl(url);
|
||||
}
|
||||
|
||||
private async getApiKey(): Promise<string> {
|
||||
const apiKey = (await this.apiKeyResolver())?.trim();
|
||||
if (!apiKey) {
|
||||
throw new FeatureTemporarilyDisabledError();
|
||||
}
|
||||
return apiKey;
|
||||
}
|
||||
|
||||
private async request(payload: NatsGifRequest, timeout = this.requestTimeoutMs): Promise<unknown> {
|
||||
try {
|
||||
if (this.connectionManager.isClosed()) {
|
||||
await this.connectionManager.connect();
|
||||
}
|
||||
const connection = this.connectionManager.getConnection();
|
||||
const response = await connection.request(this.subject, this.codec.encode(JSON.stringify(payload)), {timeout});
|
||||
const decoded = this.codec.decode(response.data);
|
||||
const parsed = parseJsonUnknown(decoded);
|
||||
const failedMessage = readFailedMessage(parsed);
|
||||
if (failedMessage) {
|
||||
throw new ServiceUnavailableError({message: failedMessage});
|
||||
}
|
||||
return parsed;
|
||||
} catch (error) {
|
||||
Logger.warn({error, op: payload.op}, '[gif-service] request failed');
|
||||
if (error instanceof FeatureTemporarilyDisabledError || error instanceof ServiceUnavailableError) {
|
||||
throw error;
|
||||
}
|
||||
throw new ServiceUnavailableError({message: 'GIF service is temporarily unavailable'});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function createNatsGifProvider(apiKeyResolver: GifApiKeyResolver): NatsGifProvider {
|
||||
const manager = new NatsConnectionManager({
|
||||
url: Config.nats.coreUrl,
|
||||
token: Config.nats.authToken || undefined,
|
||||
name: process.env.FLUXER_GIF_SERVICE_NATS_CLIENT_NAME || 'fluxer-api-gifs',
|
||||
});
|
||||
void manager.connect().catch((error) => {
|
||||
Logger.warn({error}, '[gif-service] Failed to establish NATS connection');
|
||||
});
|
||||
return new NatsGifProvider(manager, apiKeyResolver);
|
||||
}
|
||||
@@ -1,523 +0,0 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {FLUXER_USER_AGENT} from '@fluxer/constants/src/Core';
|
||||
import type {GifCategoryTagResponse, GifMediaFormat, GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
|
||||
import type {ICacheService} from '@pkgs/cache/src/ICacheService';
|
||||
import {ms} from 'itty-time';
|
||||
import {Config} from '../Config';
|
||||
import type {IMediaService} from '../infrastructure/IMediaService';
|
||||
import {Logger} from '../Logger';
|
||||
import {getWorkerService} from '../middleware/ServiceRegistry';
|
||||
import {EXTERNAL_RESPONSE_LIMITS} from '../utils/ExternalResponseLimits';
|
||||
import * as FetchUtils from '../utils/FetchUtils';
|
||||
import {isJsonRecord, parseJsonUnknown} from '../utils/JsonBoundaryUtils';
|
||||
import {
|
||||
REFRESH_LOCK_TTL_SECONDS,
|
||||
readEnrichedCategoriesCache,
|
||||
refreshLockKey,
|
||||
trackSeenLocale,
|
||||
writeEnrichedCategoriesCache,
|
||||
} from './GifFeaturedCategoriesCache';
|
||||
import type {GifProviderMeta, IGifProvider} from './IGifProvider';
|
||||
|
||||
const TENOR_BASE_URL = 'https://tenor.googleapis.com/v2';
|
||||
const DEFAULT_MEDIA_FILTER = 'webm,mp4,webp,gif,tinywebm,tinymp4,tinygif,nanogif';
|
||||
const PUBLIC_MEDIA_FORMATS = ['webm', 'mp4', 'webp', 'gif', 'tinywebm', 'tinymp4', 'tinygif', 'nanogif'] as const;
|
||||
const DEFAULT_CONTENT_FILTER = 'low';
|
||||
const CLIENT_KEY = 'fluxer';
|
||||
const MAX_RETRIES = 3;
|
||||
const BACKOFF_BASE_DELAY = ms('1 second');
|
||||
const CACHE_EXPIRATION_TIME = ms('5 minutes');
|
||||
const TENOR_PROVIDER_META: GifProviderMeta = {
|
||||
name: 'tenor',
|
||||
displayName: 'Tenor',
|
||||
attributionRequired: false,
|
||||
};
|
||||
type GifApiKeyResolver = () => Promise<string | null>;
|
||||
|
||||
interface TenorMediaFormat {
|
||||
url: string;
|
||||
dims: [number, number];
|
||||
}
|
||||
|
||||
interface TenorGif {
|
||||
id: string;
|
||||
title?: string;
|
||||
content_description?: string;
|
||||
media_formats?: Record<string, TenorMediaFormat>;
|
||||
itemurl?: string;
|
||||
url?: string;
|
||||
}
|
||||
|
||||
interface TenorCategoryTag {
|
||||
searchterm: string;
|
||||
image: string;
|
||||
}
|
||||
|
||||
function isTenorMediaFormat(value: unknown): value is TenorMediaFormat {
|
||||
if (!isJsonRecord(value) || typeof value.url !== 'string' || !Array.isArray(value.dims)) return false;
|
||||
return value.dims.length === 2 && value.dims.every((dimension) => typeof dimension === 'number');
|
||||
}
|
||||
|
||||
function isTenorGif(value: unknown): value is TenorGif {
|
||||
if (!isJsonRecord(value) || typeof value.id !== 'string') return false;
|
||||
const mediaFormats = value.media_formats;
|
||||
return (
|
||||
(value.title === undefined || typeof value.title === 'string') &&
|
||||
(value.content_description === undefined || typeof value.content_description === 'string') &&
|
||||
(mediaFormats === undefined ||
|
||||
(isJsonRecord(mediaFormats) && Object.values(mediaFormats).every(isTenorMediaFormat))) &&
|
||||
(value.itemurl === undefined || typeof value.itemurl === 'string') &&
|
||||
(value.url === undefined || typeof value.url === 'string')
|
||||
);
|
||||
}
|
||||
|
||||
function isTenorCategoryTag(value: unknown): value is TenorCategoryTag {
|
||||
return isJsonRecord(value) && typeof value.searchterm === 'string' && typeof value.image === 'string';
|
||||
}
|
||||
|
||||
function readResultsArray(value: unknown): Array<unknown> {
|
||||
if (!isJsonRecord(value) || !Array.isArray(value.results)) {
|
||||
throw new Error('Tenor API response did not include a results array');
|
||||
}
|
||||
return value.results;
|
||||
}
|
||||
|
||||
function readTagsArray(value: unknown): Array<unknown> {
|
||||
if (!isJsonRecord(value) || !Array.isArray(value.tags)) {
|
||||
throw new Error('Tenor API response did not include a tags array');
|
||||
}
|
||||
return value.tags;
|
||||
}
|
||||
|
||||
type CacheEntry<T> = {
|
||||
data: T;
|
||||
timestamp: number;
|
||||
};
|
||||
|
||||
export class TenorGifProvider implements IGifProvider {
|
||||
readonly meta = TENOR_PROVIDER_META;
|
||||
private refreshingKeys: Map<string, boolean> = new Map();
|
||||
|
||||
private featuredCacheKey(locale: string, country: string): string {
|
||||
return `tenor:featured:${locale}:${country}`;
|
||||
}
|
||||
|
||||
private trendingCacheKey(locale: string, country: string): string {
|
||||
return `tenor:trending:${locale}:${country}`;
|
||||
}
|
||||
|
||||
constructor(
|
||||
private cacheService: ICacheService,
|
||||
private mediaService: IMediaService,
|
||||
private apiKeyResolver: GifApiKeyResolver = async () => Config.tenor.apiKey || null,
|
||||
) {}
|
||||
|
||||
async isAvailable(): Promise<boolean> {
|
||||
return Boolean(await this.apiKeyResolver());
|
||||
}
|
||||
|
||||
private async getApiKey(): Promise<string> {
|
||||
const apiKey = await this.apiKeyResolver();
|
||||
if (!apiKey) {
|
||||
throw new Error('Tenor API key is not configured');
|
||||
}
|
||||
return apiKey;
|
||||
}
|
||||
|
||||
private createURL({endpoint, params}: {endpoint: string; params: Record<string, string | number | undefined>}): URL {
|
||||
const url = new URL(`${TENOR_BASE_URL}/${endpoint}`);
|
||||
for (const [key, value] of Object.entries(params)) {
|
||||
if (value !== undefined && value !== '') {
|
||||
url.searchParams.append(key, value.toString());
|
||||
}
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
private async fetchTenorData(url: URL): Promise<unknown> {
|
||||
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
const response = await fetch(url.toString(), {
|
||||
headers: {'User-Agent': FLUXER_USER_AGENT},
|
||||
signal: AbortSignal.timeout(ms('30 seconds')),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch Tenor data: ${response.statusText}`);
|
||||
}
|
||||
const responseText = await FetchUtils.streamToStringWithLimit(response.body, {
|
||||
maxBytes: EXTERNAL_RESPONSE_LIMITS.tenorApiBytes,
|
||||
headers: response.headers,
|
||||
url: response.url,
|
||||
description: 'Tenor API response',
|
||||
});
|
||||
return parseJsonUnknown(responseText);
|
||||
} catch (error) {
|
||||
if (attempt < MAX_RETRIES - 1) {
|
||||
const delay = BACKOFF_BASE_DELAY * 2 ** attempt;
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new Error('Exceeded maximum retries');
|
||||
}
|
||||
|
||||
private async fetchAndTransformGifs(url: URL): Promise<Array<GifResponse>> {
|
||||
const results = readResultsArray(await this.fetchTenorData(url)).filter(isTenorGif);
|
||||
return results.map((gif) => this.transformTenorGif(gif)).filter((gif): gif is GifResponse => gif !== null);
|
||||
}
|
||||
|
||||
private async getCache<T>(key: string): Promise<{
|
||||
data: T;
|
||||
isStale: boolean;
|
||||
} | null> {
|
||||
const cached = await this.cacheService.get<CacheEntry<T>>(key);
|
||||
if (!cached) return null;
|
||||
const age = Date.now() - cached.timestamp;
|
||||
const isStale = age > CACHE_EXPIRATION_TIME;
|
||||
return {data: cached.data, isStale};
|
||||
}
|
||||
|
||||
private async setCache<T>(key: string, data: T): Promise<void> {
|
||||
const cacheEntry: CacheEntry<T> = {
|
||||
data,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
await this.cacheService.set(key, cacheEntry);
|
||||
}
|
||||
|
||||
private triggerBackgroundRefresh<T>(key: string, refreshFn: () => Promise<T>): void {
|
||||
if (this.refreshingKeys.get(key)) {
|
||||
return;
|
||||
}
|
||||
this.refreshingKeys.set(key, true);
|
||||
setImmediate(async () => {
|
||||
try {
|
||||
const freshData = await refreshFn();
|
||||
await this.setCache(key, freshData);
|
||||
} catch (error) {
|
||||
Logger.debug({key, error}, `Background refresh failed for key ${key}`);
|
||||
} finally {
|
||||
this.refreshingKeys.delete(key);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async search(params: {q: string; locale: string; country: string}): Promise<Array<GifResponse>> {
|
||||
const apiKey = await this.getApiKey();
|
||||
const url = this.createURL({
|
||||
endpoint: 'search',
|
||||
params: {
|
||||
key: apiKey,
|
||||
client_key: CLIENT_KEY,
|
||||
q: params.q,
|
||||
country: params.country,
|
||||
locale: params.locale,
|
||||
contentfilter: DEFAULT_CONTENT_FILTER,
|
||||
media_filter: DEFAULT_MEDIA_FILTER,
|
||||
limit: 50,
|
||||
},
|
||||
});
|
||||
return this.fetchAndTransformGifs(url);
|
||||
}
|
||||
|
||||
async registerShare(params: {id: string; q: string; locale: string; country: string}): Promise<void> {
|
||||
const apiKey = await this.getApiKey();
|
||||
const url = this.createURL({
|
||||
endpoint: 'registershare',
|
||||
params: {
|
||||
key: apiKey,
|
||||
client_key: CLIENT_KEY,
|
||||
id: params.id,
|
||||
country: params.country,
|
||||
locale: params.locale,
|
||||
q: params.q,
|
||||
},
|
||||
});
|
||||
await fetch(url.toString(), {
|
||||
headers: {'User-Agent': FLUXER_USER_AGENT},
|
||||
signal: AbortSignal.timeout(ms('30 seconds')),
|
||||
});
|
||||
}
|
||||
|
||||
async getFeatured(params: {locale: string; country: string}): Promise<{
|
||||
gifs: Array<GifResponse>;
|
||||
categories: Array<GifCategoryTagResponse>;
|
||||
}> {
|
||||
const cacheKey = this.featuredCacheKey(params.locale, params.country);
|
||||
const cached = await this.getCache<{
|
||||
gifs: Array<GifResponse>;
|
||||
categories: Array<GifCategoryTagResponse>;
|
||||
}>(cacheKey);
|
||||
if (cached) {
|
||||
if (cached.isStale) {
|
||||
this.triggerBackgroundRefresh(cacheKey, () => this.fetchFeaturedData(params));
|
||||
}
|
||||
return cached.data;
|
||||
}
|
||||
const data = await this.fetchFeaturedData(params);
|
||||
await this.setCache(cacheKey, data);
|
||||
return data;
|
||||
}
|
||||
|
||||
private async fetchFeaturedData(params: {locale: string; country: string}): Promise<{
|
||||
gifs: Array<GifResponse>;
|
||||
categories: Array<GifCategoryTagResponse>;
|
||||
}> {
|
||||
const [gifs, categories] = await Promise.all([this.getFeaturedGifs(params), this.getFeaturedCategories(params)]);
|
||||
return {gifs, categories};
|
||||
}
|
||||
|
||||
async getTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
|
||||
const cacheKey = this.trendingCacheKey(params.locale, params.country);
|
||||
const cached = await this.getCache<Array<GifResponse>>(cacheKey);
|
||||
if (cached) {
|
||||
if (cached.isStale) {
|
||||
this.triggerBackgroundRefresh(cacheKey, () => this.fetchTrendingGifs(params));
|
||||
}
|
||||
return cached.data;
|
||||
}
|
||||
const gifs = await this.fetchTrendingGifs(params);
|
||||
await this.setCache(cacheKey, gifs);
|
||||
return gifs;
|
||||
}
|
||||
|
||||
private async fetchTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
|
||||
const apiKey = await this.getApiKey();
|
||||
const url = this.createURL({
|
||||
endpoint: 'featured',
|
||||
params: {
|
||||
key: apiKey,
|
||||
client_key: CLIENT_KEY,
|
||||
country: params.country,
|
||||
locale: params.locale,
|
||||
contentfilter: DEFAULT_CONTENT_FILTER,
|
||||
media_filter: DEFAULT_MEDIA_FILTER,
|
||||
limit: 50,
|
||||
},
|
||||
});
|
||||
return this.fetchAndTransformGifs(url);
|
||||
}
|
||||
|
||||
async suggest(params: {q: string; locale: string}): Promise<Array<string>> {
|
||||
const apiKey = await this.getApiKey();
|
||||
const url = this.createURL({
|
||||
endpoint: 'search_suggestions',
|
||||
params: {
|
||||
key: apiKey,
|
||||
client_key: CLIENT_KEY,
|
||||
q: params.q,
|
||||
locale: params.locale,
|
||||
limit: 20,
|
||||
},
|
||||
});
|
||||
return readResultsArray(await this.fetchTenorData(url)).filter(
|
||||
(result): result is string => typeof result === 'string',
|
||||
);
|
||||
}
|
||||
|
||||
async resolveByUrl(params: {url: string; locale: string; country: string}): Promise<GifResponse | null> {
|
||||
const slug = this.extractSlugFromUrl(params.url);
|
||||
if (!slug) return null;
|
||||
const id = this.extractIdFromSlug(slug);
|
||||
if (!id) return null;
|
||||
const apiKey = await this.getApiKey();
|
||||
const url = this.createURL({
|
||||
endpoint: 'posts',
|
||||
params: {
|
||||
key: apiKey,
|
||||
client_key: CLIENT_KEY,
|
||||
ids: id,
|
||||
country: params.country,
|
||||
locale: params.locale,
|
||||
media_filter: DEFAULT_MEDIA_FILTER,
|
||||
},
|
||||
});
|
||||
const [gif] = await this.fetchAndTransformGifs(url);
|
||||
return gif ?? null;
|
||||
}
|
||||
|
||||
private async getFeaturedGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
|
||||
const apiKey = await this.getApiKey();
|
||||
const url = this.createURL({
|
||||
endpoint: 'featured',
|
||||
params: {
|
||||
key: apiKey,
|
||||
client_key: CLIENT_KEY,
|
||||
country: params.country,
|
||||
locale: params.locale,
|
||||
contentfilter: DEFAULT_CONTENT_FILTER,
|
||||
media_filter: DEFAULT_MEDIA_FILTER,
|
||||
limit: 1,
|
||||
},
|
||||
});
|
||||
return this.fetchAndTransformGifs(url);
|
||||
}
|
||||
|
||||
private async getFeaturedCategories(params: {
|
||||
locale: string;
|
||||
country: string;
|
||||
}): Promise<Array<GifCategoryTagResponse>> {
|
||||
trackSeenLocale(this.cacheService, this.meta.name, params.locale, params.country).catch((error) => {
|
||||
Logger.debug({err: error, ...params}, 'Failed to track seen GIF locale');
|
||||
});
|
||||
const rawTags = await this.fetchRawCategoryTags(params);
|
||||
const cached = await readEnrichedCategoriesCache(this.cacheService, this.meta.name, params.locale, params.country);
|
||||
if (cached) {
|
||||
if (cached.isStale) {
|
||||
this.scheduleEnrichmentRefresh(params);
|
||||
}
|
||||
const byName = new Map(cached.data.map((entry) => [entry.name, entry]));
|
||||
return rawTags.map((tag) => byName.get(tag.name) ?? tag);
|
||||
}
|
||||
this.scheduleEnrichmentRefresh(params);
|
||||
return rawTags;
|
||||
}
|
||||
|
||||
private async fetchRawCategoryTags(params: {
|
||||
locale: string;
|
||||
country: string;
|
||||
}): Promise<Array<GifCategoryTagResponse>> {
|
||||
const apiKey = await this.getApiKey();
|
||||
const url = this.createURL({
|
||||
endpoint: 'categories',
|
||||
params: {
|
||||
key: apiKey,
|
||||
client_key: CLIENT_KEY,
|
||||
country: params.country,
|
||||
locale: params.locale,
|
||||
contentfilter: DEFAULT_CONTENT_FILTER,
|
||||
type: 'featured',
|
||||
},
|
||||
});
|
||||
const tags = readTagsArray(await this.fetchTenorData(url)).filter(isTenorCategoryTag);
|
||||
return tags
|
||||
.filter((tag) => Boolean(tag.searchterm) && Boolean(tag.image))
|
||||
.map((tag) => ({
|
||||
name: tag.searchterm,
|
||||
src: tag.image,
|
||||
proxy_src: this.mediaService.getExternalMediaProxyURL(tag.image),
|
||||
gif: null,
|
||||
}));
|
||||
}
|
||||
|
||||
private scheduleEnrichmentRefresh(params: {locale: string; country: string}): void {
|
||||
(async () => {
|
||||
try {
|
||||
const workerService = getWorkerService();
|
||||
await workerService.addJob('refreshGifFeaturedCategories', {
|
||||
provider: this.meta.name,
|
||||
locale: params.locale,
|
||||
country: params.country,
|
||||
});
|
||||
} catch (error) {
|
||||
Logger.debug({err: error, ...params}, 'Failed to enqueue GIF featured-categories refresh');
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
async refreshFeaturedCategories(params: {locale: string; country: string}): Promise<void> {
|
||||
const lockKey = refreshLockKey(this.meta.name, params.locale, params.country);
|
||||
const token = await this.cacheService.acquireLock(lockKey, REFRESH_LOCK_TTL_SECONDS);
|
||||
if (!token) {
|
||||
Logger.debug(params, 'Skipping enriched GIF categories refresh; another worker holds the lock');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const rawTags = await this.fetchRawCategoryTags(params);
|
||||
const enriched = await Promise.all(
|
||||
rawTags.map(async (tag) => {
|
||||
try {
|
||||
const [gif] = await this.search({q: tag.name, locale: params.locale, country: params.country});
|
||||
return {...tag, gif: gif ?? null};
|
||||
} catch (error) {
|
||||
Logger.debug({err: error, tag: tag.name, ...params}, 'Failed to enrich GIF category');
|
||||
return tag;
|
||||
}
|
||||
}),
|
||||
);
|
||||
await writeEnrichedCategoriesCache(this.cacheService, this.meta.name, params.locale, params.country, enriched);
|
||||
} catch (error) {
|
||||
Logger.warn({err: error, ...params}, 'Failed to refresh enriched GIF categories');
|
||||
throw error;
|
||||
} finally {
|
||||
await this.cacheService.releaseLock(lockKey, token).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
private selectMediaFormat(mediaFormats: Record<string, TenorMediaFormat> | undefined): TenorMediaFormat | null {
|
||||
if (!mediaFormats) return null;
|
||||
const preferredKeys = ['webm', 'mp4', 'tinywebm', 'tinymp4', 'webp', 'gif', 'tinygif', 'nanogif'];
|
||||
for (const key of preferredKeys) {
|
||||
const candidate = mediaFormats[key];
|
||||
if (candidate?.url && candidate.dims?.length === 2) return candidate;
|
||||
}
|
||||
for (const candidate of Object.values(mediaFormats)) {
|
||||
if (candidate?.url && candidate.dims?.length === 2) return candidate;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private transformTenorGif(input: TenorGif): GifResponse | null {
|
||||
const best = this.selectMediaFormat(input.media_formats);
|
||||
if (!best) return null;
|
||||
const title = input.title?.trim() || input.content_description?.trim() || '';
|
||||
const fallbackUrl = `https://tenor.com/view/${encodeURIComponent(input.id)}`;
|
||||
const url = input.itemurl?.trim() || input.url?.trim() || fallbackUrl;
|
||||
const slug = this.extractSlugFromUrl(url) ?? `view/${input.id}`;
|
||||
const media: Record<string, GifMediaFormat> = {};
|
||||
for (const key of PUBLIC_MEDIA_FORMATS) {
|
||||
const candidate = input.media_formats?.[key];
|
||||
if (candidate?.url && candidate.dims?.length === 2) {
|
||||
media[key] = {
|
||||
src: candidate.url,
|
||||
proxy_src: this.mediaService.getExternalMediaProxyURL(candidate.url),
|
||||
width: candidate.dims[0],
|
||||
height: candidate.dims[1],
|
||||
};
|
||||
}
|
||||
}
|
||||
return {
|
||||
id: input.id,
|
||||
slug,
|
||||
provider: this.meta.name,
|
||||
title,
|
||||
url,
|
||||
src: best.url,
|
||||
proxy_src: this.mediaService.getExternalMediaProxyURL(best.url),
|
||||
width: best.dims[0],
|
||||
height: best.dims[1],
|
||||
media,
|
||||
};
|
||||
}
|
||||
|
||||
extractSlugFromUrl(url: string): string | null {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
const hostname = parsed.hostname.toLowerCase();
|
||||
if (hostname !== 'tenor.com' && hostname !== 'www.tenor.com') return null;
|
||||
const match = parsed.pathname.match(/^\/(?:[a-z]{2}\/)?view\/([^/]+)/i);
|
||||
if (!match?.[1]) return null;
|
||||
const slug = decodeURIComponent(match[1]).trim();
|
||||
return slug ? `view/${slug}` : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private extractIdFromSlug(slug: string): string | null {
|
||||
const normalized = slug.trim().replace(/^view\//i, '');
|
||||
if (!normalized) return null;
|
||||
const lastDashIndex = normalized.lastIndexOf('-');
|
||||
const candidate = lastDashIndex === -1 ? normalized : normalized.slice(lastDashIndex + 1);
|
||||
return candidate.trim() || null;
|
||||
}
|
||||
|
||||
buildShareUrl(slug: string): string {
|
||||
const trimmed = slug.trim().replace(/^\/+|\/+$/g, '');
|
||||
const normalized = trimmed.toLowerCase().startsWith('view/') ? trimmed : `view/${trimmed}`;
|
||||
return `https://tenor.com/${normalized}`;
|
||||
}
|
||||
}
|
||||
@@ -70,6 +70,7 @@ describe('NatsUnfurlerService', () => {
|
||||
bypass_cache: true,
|
||||
cache_only: false,
|
||||
youtube_api_key: null,
|
||||
klipy_api_key: null,
|
||||
},
|
||||
timeout: 12000,
|
||||
},
|
||||
|
||||
@@ -19,6 +19,7 @@ interface NatsUnfurlRequest {
|
||||
bypass_cache: boolean;
|
||||
cache_only: boolean;
|
||||
youtube_api_key: string | null;
|
||||
klipy_api_key: string | null;
|
||||
}
|
||||
|
||||
interface NatsUnfurlInnerResult {
|
||||
@@ -59,6 +60,7 @@ export class NatsUnfurlerService extends IUnfurlerService {
|
||||
constructor(
|
||||
connectionManager: INatsConnectionManager,
|
||||
private readonly resolveYoutubeApiKey: (() => Promise<string | null>) | null = null,
|
||||
private readonly resolveKlipyApiKey: (() => Promise<string | null>) | null = null,
|
||||
) {
|
||||
super();
|
||||
this.connectionManager = connectionManager;
|
||||
@@ -77,6 +79,7 @@ export class NatsUnfurlerService extends IUnfurlerService {
|
||||
bypass_cache: options.bypassCache === true,
|
||||
cache_only: options.cacheOnly === true,
|
||||
youtube_api_key: this.resolveYoutubeApiKey ? await this.resolveYoutubeApiKey() : null,
|
||||
klipy_api_key: this.resolveKlipyApiKey ? await this.resolveKlipyApiKey() : null,
|
||||
};
|
||||
if (this.connectionManager.isClosed()) {
|
||||
await this.connectionManager.connect();
|
||||
|
||||
@@ -99,13 +99,10 @@ interface InstanceServicesPublicConfig {
|
||||
bluesky_enabled: boolean;
|
||||
}
|
||||
|
||||
export type InstanceGifProvider = 'tenor' | 'klipy';
|
||||
export type InstanceCaptchaProvider = 'hcaptcha' | 'turnstile' | 'none';
|
||||
type InstanceEmailProvider = 'smtp' | 'none';
|
||||
|
||||
interface InstanceGifIntegrationConfig {
|
||||
provider: InstanceGifProvider | null;
|
||||
tenor_api_key: string | null;
|
||||
klipy_api_key: string | null;
|
||||
}
|
||||
|
||||
@@ -160,9 +157,7 @@ interface InstanceIntegrationsConfig {
|
||||
bluesky: InstanceBlueskyIntegrationConfig;
|
||||
}
|
||||
|
||||
export interface InstanceGifEffectiveConfig {
|
||||
provider: InstanceGifProvider;
|
||||
tenor_api_key: string | null;
|
||||
interface InstanceGifEffectiveConfig {
|
||||
klipy_api_key: string | null;
|
||||
active_api_key: string | null;
|
||||
available: boolean;
|
||||
@@ -179,9 +174,6 @@ export interface InstanceCaptchaEffectiveConfig {
|
||||
|
||||
interface InstanceIntegrationsAdminConfig {
|
||||
gif: {
|
||||
provider: InstanceGifProvider | null;
|
||||
effective_provider: InstanceGifProvider;
|
||||
tenor_api_key_set: boolean;
|
||||
klipy_api_key_set: boolean;
|
||||
effective_available: boolean;
|
||||
};
|
||||
@@ -438,8 +430,6 @@ function normalizeInstancePolicyConfig(value: unknown): InstancePolicyConfig {
|
||||
|
||||
const DEFAULT_INSTANCE_INTEGRATIONS_CONFIG: InstanceIntegrationsConfig = {
|
||||
gif: {
|
||||
provider: null,
|
||||
tenor_api_key: null,
|
||||
klipy_api_key: null,
|
||||
},
|
||||
youtube: {
|
||||
@@ -492,10 +482,6 @@ const DEFAULT_INSTANCE_MEDIA_CONFIG: InstanceMediaConfig = {
|
||||
attachment_decay: DEFAULT_INSTANCE_ATTACHMENT_DECAY_CONFIG,
|
||||
};
|
||||
|
||||
function isGifProvider(value: unknown): value is InstanceGifProvider {
|
||||
return value === 'tenor' || value === 'klipy';
|
||||
}
|
||||
|
||||
function isCaptchaProvider(value: unknown): value is InstanceCaptchaProvider {
|
||||
return value === 'hcaptcha' || value === 'turnstile' || value === 'none';
|
||||
}
|
||||
@@ -561,8 +547,6 @@ function normalizeInstanceIntegrationsConfig(value: unknown): InstanceIntegratio
|
||||
: defaults.bluesky.keys;
|
||||
return {
|
||||
gif: {
|
||||
provider: isGifProvider(gif.provider) ? gif.provider : defaults.gif.provider,
|
||||
tenor_api_key: normalizeSecretString(gif.tenor_api_key),
|
||||
klipy_api_key: normalizeSecretString(gif.klipy_api_key),
|
||||
},
|
||||
youtube: {
|
||||
@@ -1214,16 +1198,11 @@ export class InstanceConfigRepository {
|
||||
|
||||
async getEffectiveGifConfig(): Promise<InstanceGifEffectiveConfig> {
|
||||
const integrations = await this.getInstanceIntegrationsConfig();
|
||||
const provider = integrations.gif.provider ?? Config.gif.provider;
|
||||
const tenorApiKey = integrations.gif.tenor_api_key ?? normalizeSecretString(Config.tenor.apiKey);
|
||||
const klipyApiKey = integrations.gif.klipy_api_key ?? normalizeSecretString(Config.klipy.apiKey);
|
||||
const activeApiKey = provider === 'tenor' ? tenorApiKey : klipyApiKey;
|
||||
return {
|
||||
provider,
|
||||
tenor_api_key: tenorApiKey,
|
||||
klipy_api_key: klipyApiKey,
|
||||
active_api_key: activeApiKey,
|
||||
available: Boolean(activeApiKey),
|
||||
active_api_key: klipyApiKey,
|
||||
available: Boolean(klipyApiKey),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1323,9 +1302,6 @@ export class InstanceConfigRepository {
|
||||
]);
|
||||
return {
|
||||
gif: {
|
||||
provider: integrations.gif.provider,
|
||||
effective_provider: gif.provider,
|
||||
tenor_api_key_set: secretIsSet(integrations.gif.tenor_api_key) || secretIsSet(Config.tenor.apiKey),
|
||||
klipy_api_key_set: secretIsSet(integrations.gif.klipy_api_key) || secretIsSet(Config.klipy.apiKey),
|
||||
effective_available: gif.available,
|
||||
},
|
||||
|
||||
@@ -12,7 +12,7 @@ import {RateLimitMiddleware} from '../middleware/RateLimitMiddleware';
|
||||
import {OpenAPI} from '../middleware/ResponseTypeMiddleware';
|
||||
import {RateLimitConfigs} from '../RateLimitConfig';
|
||||
import type {HonoEnv} from '../types/HonoEnv';
|
||||
import type {InstanceCaptchaEffectiveConfig, InstanceGifEffectiveConfig} from './InstanceConfigRepository';
|
||||
import type {InstanceCaptchaEffectiveConfig} from './InstanceConfigRepository';
|
||||
|
||||
function buildDiscoveryStaticInput(
|
||||
gifService: GifService | undefined,
|
||||
@@ -20,12 +20,11 @@ function buildDiscoveryStaticInput(
|
||||
runtime: {
|
||||
captcha: InstanceCaptchaEffectiveConfig;
|
||||
emailEnabled: boolean;
|
||||
gif: InstanceGifEffectiveConfig;
|
||||
},
|
||||
): DiscoveryStaticInput {
|
||||
const apiClientEndpoint = Config.endpoints.apiClient;
|
||||
const apiPublicEndpoint = Config.endpoints.apiPublic;
|
||||
const activeGif = gifService?.getByName(runtime.gif.provider);
|
||||
const gifProvider = gifService?.getProvider();
|
||||
return {
|
||||
apiCodeVersion: API_CODE_VERSION,
|
||||
endpoints: {
|
||||
@@ -54,9 +53,9 @@ function buildDiscoveryStaticInput(
|
||||
emails_enabled: runtime.emailEnabled,
|
||||
},
|
||||
gif: {
|
||||
provider: runtime.gif.provider,
|
||||
display_name: activeGif?.meta.displayName ?? runtime.gif.provider,
|
||||
attribution_required: activeGif?.meta.attributionRequired ?? false,
|
||||
provider: gifProvider?.meta.name ?? 'klipy',
|
||||
display_name: gifProvider?.meta.displayName ?? 'KLIPY',
|
||||
attribution_required: gifProvider?.meta.attributionRequired ?? true,
|
||||
},
|
||||
push: {
|
||||
public_vapid_key: Config.push.publicVapidKey ?? null,
|
||||
@@ -86,14 +85,13 @@ export function InstanceController(app: Hono<HonoEnv>) {
|
||||
const limits = limitConfigService?.getConfigWireFormat();
|
||||
const sso = await ctx.get('ssoService').getPublicStatus();
|
||||
const instanceConfigRepository = ctx.get('instanceConfigRepository');
|
||||
const [registration, community, services, appPublicConfig, captcha, email, gif] = await Promise.all([
|
||||
const [registration, community, services, appPublicConfig, captcha, email] = await Promise.all([
|
||||
instanceConfigRepository.getRegistrationPublicConfig(),
|
||||
instanceConfigRepository.getInstanceCommunityPublicConfig(),
|
||||
instanceConfigRepository.getResolvedServicesConfig(),
|
||||
instanceConfigRepository.getAppPublicConfig(),
|
||||
instanceConfigRepository.getEffectiveCaptchaConfig(),
|
||||
instanceConfigRepository.getEffectiveEmailConfig(),
|
||||
instanceConfigRepository.getEffectiveGifConfig(),
|
||||
]);
|
||||
if (!limits) {
|
||||
throw new Error('limit_config_service is not bound');
|
||||
@@ -111,7 +109,6 @@ export function InstanceController(app: Hono<HonoEnv>) {
|
||||
{
|
||||
captcha,
|
||||
emailEnabled: email.enabled,
|
||||
gif,
|
||||
},
|
||||
),
|
||||
{
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
|
||||
import {ForbiddenError} from '@fluxer/errors/src/domains/core/ForbiddenError';
|
||||
import {resolveClientIpHeaderName} from '@fluxer/ip_utils/src/ClientIp';
|
||||
import {createMiddleware} from 'hono/factory';
|
||||
import {Config} from '../Config';
|
||||
import {Logger} from '../Logger';
|
||||
@@ -23,7 +24,7 @@ const defaultExemptPaths: Array<string> = [
|
||||
|
||||
export function RequireClientIpMiddleware({
|
||||
exemptPaths = defaultExemptPaths,
|
||||
requiredHeaders = ['x-forwarded-for'],
|
||||
requiredHeaders = [resolveClientIpHeaderName(Config.proxy.client_ip_header)],
|
||||
}: RequireClientIpOptions = {}) {
|
||||
return createMiddleware<HonoEnv>(async (ctx, next) => {
|
||||
if (Config.dev.testModeEnabled) {
|
||||
|
||||
@@ -82,6 +82,7 @@ import {UserChannelRequestService} from '../user/services/UserChannelRequestServ
|
||||
import {UserContentRequestService} from '../user/services/UserContentRequestService';
|
||||
import {UserRelationshipRequestService} from '../user/services/UserRelationshipRequestService';
|
||||
import {UserService} from '../user/services/UserService';
|
||||
import {resolveRequestClientIp} from '../utils/IpUtils';
|
||||
import {VoicePresenceHeartbeatStore} from '../voice/VoicePresenceHeartbeatStore';
|
||||
import {VoiceService} from '../voice/VoiceService';
|
||||
import {WebhookRequestService} from '../webhook/WebhookRequestService';
|
||||
@@ -364,7 +365,7 @@ function getLiveKitWebhookService(): LiveKitWebhookService | null {
|
||||
export const ServiceMiddleware = createMiddleware<HonoEnv>(async (ctx, next) => {
|
||||
const apiContext = createApiContext({
|
||||
requestId: ctx.get('requestId') ?? crypto.randomUUID(),
|
||||
clientIp: ctx.req.header('x-forwarded-for') ?? null,
|
||||
clientIp: resolveRequestClientIp(ctx.req.raw),
|
||||
userAgent: ctx.req.header('user-agent') ?? null,
|
||||
});
|
||||
ctx.set('apiContext', apiContext);
|
||||
|
||||
@@ -38,8 +38,7 @@ import {DownloadService} from '../download/DownloadService';
|
||||
import {createEmailProvider} from '../email/EmailProviderFactory';
|
||||
import {FavoriteMemeRepository} from '../favorite_meme/FavoriteMemeRepository';
|
||||
import {GifService} from '../gif/GifService';
|
||||
import {KlipyGifProvider} from '../gif/KlipyGifProvider';
|
||||
import {TenorGifProvider} from '../gif/TenorGifProvider';
|
||||
import {createNatsGifProvider} from '../gif/NatsGifProvider';
|
||||
import {GuildAuditLogService} from '../guild/GuildAuditLogService';
|
||||
import {GuildDiscoveryRepository} from '../guild/repositories/GuildDiscoveryRepository';
|
||||
import {GuildRepository} from '../guild/repositories/GuildRepository';
|
||||
@@ -349,6 +348,7 @@ export function setInjectedUnfurlerService(service: IUnfurlerService | undefined
|
||||
}
|
||||
|
||||
const getDefaultUnfurlerService = singleton(() => {
|
||||
const instanceConfigRepository = getInstanceConfigRepository();
|
||||
const manager = new NatsConnectionManager({
|
||||
url: Config.nats.coreUrl,
|
||||
token: Config.nats.authToken || undefined,
|
||||
@@ -357,7 +357,11 @@ const getDefaultUnfurlerService = singleton(() => {
|
||||
void manager.connect().catch((error) => {
|
||||
Logger.error({error}, '[nats-unfurl] Failed to establish NATS connection');
|
||||
});
|
||||
return new NatsUnfurlerService(manager, async () => getInstanceConfigRepository().getEffectiveYoutubeApiKey());
|
||||
return new NatsUnfurlerService(
|
||||
manager,
|
||||
async () => instanceConfigRepository.getEffectiveYoutubeApiKey(),
|
||||
async () => (await instanceConfigRepository.getEffectiveGifConfig()).klipy_api_key,
|
||||
);
|
||||
});
|
||||
|
||||
export function getUnfurlerService(): IUnfurlerService {
|
||||
@@ -376,24 +380,10 @@ export const getBotMfaMirrorService = singleton(
|
||||
() => new BotMfaMirrorService(getApplicationRepository(), getUserRepository(), getGatewayService()),
|
||||
);
|
||||
export const getGifService = singleton(() => {
|
||||
const cache = getCacheService();
|
||||
const media = getMediaService();
|
||||
const instanceConfigRepository = getInstanceConfigRepository();
|
||||
return new GifService({
|
||||
providers: [
|
||||
new TenorGifProvider(
|
||||
cache,
|
||||
media,
|
||||
async () => (await instanceConfigRepository.getEffectiveGifConfig()).tenor_api_key,
|
||||
),
|
||||
new KlipyGifProvider(
|
||||
cache,
|
||||
media,
|
||||
async () => (await instanceConfigRepository.getEffectiveGifConfig()).klipy_api_key,
|
||||
),
|
||||
],
|
||||
activeName: async () => (await instanceConfigRepository.getEffectiveGifConfig()).provider,
|
||||
});
|
||||
return new GifService(
|
||||
createNatsGifProvider(async () => (await instanceConfigRepository.getEffectiveGifConfig()).klipy_api_key),
|
||||
);
|
||||
});
|
||||
export const getExpressionAssetPurger = singleton(() => new ExpressionAssetPurger(getAssetDeletionQueue()));
|
||||
export const getGuildAuditLogService = singleton(
|
||||
|
||||
@@ -24808,10 +24808,7 @@
|
||||
"gif": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"provider": {
|
||||
"type": "string",
|
||||
"description": "Stable machine name of the active GIF provider (e.g. \"klipy\", \"tenor\")"
|
||||
},
|
||||
"provider": {"type": "string", "description": "Stable machine name of the active GIF provider."},
|
||||
"display_name": {"type": "string", "description": "Human-readable provider name shown in the UI"},
|
||||
"attribution_required": {
|
||||
"type": "boolean",
|
||||
@@ -29258,7 +29255,7 @@
|
||||
},
|
||||
"gif_provider": {
|
||||
"anyOf": [{"type": "string"}, {"type": "null"}],
|
||||
"description": "Stable name of the GIF provider that issued gif_slug (e.g. \"klipy\", \"tenor\"), if any"
|
||||
"description": "Stable name of the GIF provider that issued gif_slug, if any. Legacy records may contain older provider names."
|
||||
},
|
||||
"media": {
|
||||
"anyOf": [
|
||||
@@ -35767,7 +35764,7 @@
|
||||
},
|
||||
"gif_provider": {
|
||||
"anyOf": [{"type": "string"}, {"type": "null"}],
|
||||
"description": "Stable name of the GIF provider that issued gif_slug (e.g. \"klipy\", \"tenor\")"
|
||||
"description": "Stable name of the GIF provider that issued gif_slug. New provider GIFs are sourced from KLIPY."
|
||||
},
|
||||
"media": {
|
||||
"anyOf": [
|
||||
|
||||
@@ -12,6 +12,7 @@ import {Logger} from '../Logger';
|
||||
import type {HonoApp} from '../types/HonoEnv';
|
||||
import type {IUserRepository} from '../user/IUserRepository';
|
||||
import {mapUserToPrivateResponse} from '../user/UserMappers';
|
||||
import {resolveRequestClientIp} from '../utils/IpUtils';
|
||||
|
||||
interface TwilioInboundSmsWebhookContext {
|
||||
authToken: string;
|
||||
@@ -29,7 +30,7 @@ export function installTwilioInboundSmsWebhook(app: HonoApp, ctx: TwilioInboundS
|
||||
const params = parseFormUrlEncoded(rawBody);
|
||||
const signature = c.req.header('x-twilio-signature') ?? '';
|
||||
if (!verifyTwilioSignature(ctx.authToken, ctx.publicWebhookUrl, params, signature)) {
|
||||
Logger.warn({ip: c.req.header('cf-connecting-ip') ?? '?'}, 'Twilio webhook signature failed; rejecting');
|
||||
Logger.warn({ip: resolveRequestClientIp(c.req.raw) ?? '?'}, 'Twilio webhook signature failed; rejecting');
|
||||
return c.text('forbidden', 403);
|
||||
}
|
||||
const fromPhone = params.get('From') ?? '';
|
||||
|
||||
@@ -18,8 +18,6 @@ export const EXTERNAL_RESPONSE_LIMITS = {
|
||||
blueskyProfileBytes: 256 * 1024,
|
||||
wikipediaSummaryBytes: 256 * 1024,
|
||||
oEmbedBytes: 256 * 1024,
|
||||
klipyApiBytes: 512 * 1024,
|
||||
tenorApiBytes: 512 * 1024,
|
||||
pwnedPasswordsBytes: 1024 * 1024,
|
||||
rdapBytes: 512 * 1024,
|
||||
externalTemplateBytes: 512 * 1024,
|
||||
|
||||
@@ -16,16 +16,17 @@ interface GetIpAddressReverseOptions {
|
||||
cacheTtlSeconds?: number;
|
||||
}
|
||||
|
||||
export function resolveRequestClientIp(req: Request): string | null {
|
||||
return extractClientIp(req, {
|
||||
trustClientIpHeader: Config.proxy.trust_client_ip_header,
|
||||
clientIpHeaderName: Config.proxy.client_ip_header,
|
||||
});
|
||||
}
|
||||
|
||||
export async function lookupGeoip(req: Request): Promise<GeoipResult>;
|
||||
export async function lookupGeoip(ip: string): Promise<GeoipResult>;
|
||||
export async function lookupGeoip(input: string | Request): Promise<GeoipResult> {
|
||||
const ip =
|
||||
typeof input === 'string'
|
||||
? input
|
||||
: extractClientIp(input, {
|
||||
trustClientIpHeader: Config.proxy.trust_client_ip_header,
|
||||
clientIpHeaderName: Config.proxy.client_ip_header,
|
||||
});
|
||||
const ip = typeof input === 'string' ? input : resolveRequestClientIp(input);
|
||||
if (!ip) {
|
||||
return {countryCode: null, normalizedIp: null, city: null, region: null, countryName: null};
|
||||
}
|
||||
|
||||
@@ -63,7 +63,6 @@ const LANE_CONFIG = {
|
||||
batch: {
|
||||
consumerName: 'workers_batch',
|
||||
tasks: [
|
||||
'enqueueGifFeaturedCategoriesRefresh',
|
||||
'expireAttachments',
|
||||
'indexChannelMessages',
|
||||
'indexGuildMembers',
|
||||
@@ -74,7 +73,6 @@ const LANE_CONFIG = {
|
||||
'processPendingBulkMessageDeletions',
|
||||
'processPremiumStateReconciliationQueue',
|
||||
'prunePostgresKvTtl',
|
||||
'refreshGifFeaturedCategories',
|
||||
'refreshSearchIndex',
|
||||
'syncDiscoveryIndex',
|
||||
'syncDisposableEmailDomains',
|
||||
|
||||
@@ -52,7 +52,6 @@ function registerCronJobs(cron: CronScheduler): void {
|
||||
cron.upsert('prunePostgresKvTtl', 'prunePostgresKvTtl', {}, '0 */5 * * * *');
|
||||
cron.upsert('syncDiscoveryIndex', 'syncDiscoveryIndex', {}, '0 */15 * * * *');
|
||||
cron.upsert('syncDisposableEmailDomains', 'syncDisposableEmailDomains', {}, '0 */30 * * * *');
|
||||
cron.upsert('enqueueGifFeaturedCategoriesRefresh', 'enqueueGifFeaturedCategoriesRefresh', {}, '0 */30 * * * *');
|
||||
cron.upsert('syncUrlBlocklists', 'syncUrlBlocklists', {}, '0 0 */6 * * *');
|
||||
cron.upsert('syncFileShaBlocklists', 'syncFileShaBlocklists', {}, '0 0 */12 * * *');
|
||||
cron.upsert('flushUserActivityBuffer', 'flushUserActivityBuffer', {}, '*/10 * * * * *');
|
||||
|
||||
@@ -13,7 +13,6 @@ import bulkDeleteSelfMessagesImmediate from './tasks/BulkDeleteSelfMessagesImmed
|
||||
import bulkDeleteUserMessages from './tasks/BulkDeleteUserMessages';
|
||||
import bulkDeleteUserMessagesScoped from './tasks/BulkDeleteUserMessagesScoped';
|
||||
import deleteUserMessagesInGuildByTime from './tasks/DeleteUserMessagesInGuildByTime';
|
||||
import enqueueGifFeaturedCategoriesRefresh from './tasks/EnqueueGifFeaturedCategoriesRefresh';
|
||||
import expireAttachments from './tasks/ExpireAttachments';
|
||||
import extractEmbeds from './tasks/ExtractEmbeds';
|
||||
import finalizeNcmecAttachmentReport from './tasks/FinalizeNcmecAttachmentReport';
|
||||
@@ -34,7 +33,6 @@ import processPremiumStateReconciliationQueue from './tasks/ProcessPremiumStateR
|
||||
import processStripeWebhook from './tasks/ProcessStripeWebhook';
|
||||
import prunePostgresKvTtl from './tasks/PrunePostgresKvTtl';
|
||||
import reconcileUserPayments from './tasks/ReconcileUserPayments';
|
||||
import refreshGifFeaturedCategories from './tasks/RefreshGifFeaturedCategories';
|
||||
import refreshSearchIndex from './tasks/RefreshSearchIndex';
|
||||
import revalidateUserConnections from './tasks/RevalidateUserConnections';
|
||||
import {sendScheduledMessage} from './tasks/SendScheduledMessage';
|
||||
@@ -60,7 +58,6 @@ export const workerTasks: Record<WorkerTaskName, WorkerTaskHandler> = {
|
||||
bulkUpdateSuspiciousActivityFlags: bulkUpdateSuspiciousActivityFlags,
|
||||
bulkUpdateUserFlags: bulkUpdateUserFlags,
|
||||
deleteUserMessagesInGuildByTime,
|
||||
enqueueGifFeaturedCategoriesRefresh,
|
||||
expireAttachments,
|
||||
extractEmbeds,
|
||||
finalizeNcmecAttachmentReport,
|
||||
@@ -80,7 +77,6 @@ export const workerTasks: Record<WorkerTaskName, WorkerTaskHandler> = {
|
||||
processPremiumStateReconciliationQueue,
|
||||
reconcileUserPayments,
|
||||
prunePostgresKvTtl,
|
||||
refreshGifFeaturedCategories,
|
||||
refreshSearchIndex,
|
||||
revalidateUserConnections,
|
||||
sendScheduledMessage,
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
|
||||
import {listSeenLocales} from '../../gif/GifFeaturedCategoriesCache';
|
||||
import {getGifService} from '../../middleware/ServiceSingletons';
|
||||
import {getWorkerDependencies} from '../WorkerContext';
|
||||
|
||||
const enqueueGifFeaturedCategoriesRefresh: WorkerTaskHandler = async (_payload, helpers) => {
|
||||
const {cacheService, workerService} = getWorkerDependencies();
|
||||
const gifService = getGifService();
|
||||
for (const provider of gifService.listProviders()) {
|
||||
if (!(await provider.isAvailable())) continue;
|
||||
const locales = await listSeenLocales(cacheService, provider.meta.name);
|
||||
if (locales.length === 0) {
|
||||
helpers.logger.debug({provider: provider.meta.name}, 'No seen locales for GIF provider; nothing to refresh');
|
||||
continue;
|
||||
}
|
||||
helpers.logger.debug(
|
||||
{provider: provider.meta.name, count: locales.length},
|
||||
'Fanning out enriched GIF categories refresh',
|
||||
);
|
||||
for (const {locale, country} of locales) {
|
||||
try {
|
||||
await workerService.addJob('refreshGifFeaturedCategories', {
|
||||
provider: provider.meta.name,
|
||||
locale,
|
||||
country,
|
||||
});
|
||||
} catch (error) {
|
||||
helpers.logger.warn(
|
||||
{err: error, provider: provider.meta.name, locale, country},
|
||||
'Failed to enqueue per-locale GIF categories refresh',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export default enqueueGifFeaturedCategoriesRefresh;
|
||||
@@ -1,33 +0,0 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
|
||||
import {getGifService} from '../../middleware/ServiceSingletons';
|
||||
|
||||
interface RefreshPayload {
|
||||
provider?: string;
|
||||
locale?: string;
|
||||
country?: string;
|
||||
}
|
||||
|
||||
const refreshGifFeaturedCategories: WorkerTaskHandler = async (rawPayload, helpers) => {
|
||||
const payload = (rawPayload ?? {}) as RefreshPayload;
|
||||
const {provider, locale, country} = payload;
|
||||
if (!provider || !locale || !country) {
|
||||
helpers.logger.warn({payload}, 'refreshGifFeaturedCategories called without required fields');
|
||||
return;
|
||||
}
|
||||
const gifService = getGifService();
|
||||
const target = gifService.getByName(provider);
|
||||
if (!target) {
|
||||
helpers.logger.warn({provider}, 'refreshGifFeaturedCategories: unknown provider');
|
||||
return;
|
||||
}
|
||||
if (!(await target.isAvailable())) {
|
||||
helpers.logger.debug({provider}, 'refreshGifFeaturedCategories: provider not configured, skipping');
|
||||
return;
|
||||
}
|
||||
helpers.logger.debug({provider, locale, country}, 'Refreshing enriched GIF featured categories');
|
||||
await target.refreshFeaturedCategories({locale, country});
|
||||
};
|
||||
|
||||
export default refreshGifFeaturedCategories;
|
||||
Reference in new issue
Block a user