fix: migrate gifs to klipy service (#1220)

This commit is contained in:
Hampus
2026-06-30 01:19:57 +02:00
committed by GitHub
parent 99eedf6d1e
commit 4a741d8396
79 changed files with 3032 additions and 1653 deletions
+32
View File
@@ -0,0 +1,32 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
name: build gifs
on:
workflow_dispatch:
inputs:
build-version:
description: "Explicit Fluxer CalVer build version (YYYY.MDD.MICRO, UTC HHMMSS without leading zeroes) to use instead of automatic UTC clock allocation"
type: string
required: false
default: ""
workflow_call:
inputs:
build-version:
description: "Explicit Fluxer CalVer build version (YYYY.MDD.MICRO, UTC HHMMSS without leading zeroes) to use instead of automatic UTC clock allocation"
type: string
required: false
default: ""
permissions:
actions: read
contents: read
packages: write
jobs:
image:
uses: ./.github/workflows/_build-image.yaml
with:
image: fluxer-gifs
dockerfile: fluxer_gifs/Dockerfile
build-version: ${{ inputs.build-version }}
secrets: inherit
+1 -1
View File
@@ -309,7 +309,7 @@ jobs:
*worker*) *worker*)
EXTRA_SETS+=(--set-string "workerRealtime.tag=${HELM_TEST_BUILD_VERSION}" --set-string "workerUnfurl.tag=${HELM_TEST_BUILD_VERSION}" --set-string "workerLifecycle.tag=${HELM_TEST_BUILD_VERSION}" --set-string "workerBatch.tag=${HELM_TEST_BUILD_VERSION}" --set-string "workerRealtime.build.version=${HELM_TEST_BUILD_VERSION}" --set-string "workerUnfurl.build.version=${HELM_TEST_BUILD_VERSION}" --set-string "workerLifecycle.build.version=${HELM_TEST_BUILD_VERSION}" --set-string "workerBatch.build.version=${HELM_TEST_BUILD_VERSION}") EXTRA_SETS+=(--set-string "workerRealtime.tag=${HELM_TEST_BUILD_VERSION}" --set-string "workerUnfurl.tag=${HELM_TEST_BUILD_VERSION}" --set-string "workerLifecycle.tag=${HELM_TEST_BUILD_VERSION}" --set-string "workerBatch.tag=${HELM_TEST_BUILD_VERSION}" --set-string "workerRealtime.build.version=${HELM_TEST_BUILD_VERSION}" --set-string "workerUnfurl.build.version=${HELM_TEST_BUILD_VERSION}" --set-string "workerLifecycle.build.version=${HELM_TEST_BUILD_VERSION}" --set-string "workerBatch.build.version=${HELM_TEST_BUILD_VERSION}")
;; ;;
*member-lists*|*messages*|*presence*|*snowflakes*|*unfurl*|*users*|*voice-states*) *gifs*|*member-lists*|*messages*|*presence*|*snowflakes*|*unfurl*|*users*|*voice-states*)
EXTRA_SETS+=(--set-string "svc.tag=${HELM_TEST_BUILD_VERSION}" --set-string "svc.build.version=${HELM_TEST_BUILD_VERSION}" --set-string svc.build.channel=stable) EXTRA_SETS+=(--set-string "svc.tag=${HELM_TEST_BUILD_VERSION}" --set-string "svc.build.version=${HELM_TEST_BUILD_VERSION}" --set-string svc.build.channel=stable)
;; ;;
esac esac
Generated
+19 -2
View File
@@ -1953,6 +1953,25 @@ dependencies = [
"urlencoding", "urlencoding",
] ]
[[package]]
name = "fluxer-gifs"
version = "0.1.0"
dependencies = [
"anyhow",
"base64",
"fluxer-svc",
"hmac 0.13.0",
"moka",
"reqwest",
"serde",
"serde_json",
"sha2 0.11.0",
"tokio",
"tracing",
"url",
"urlencoding",
]
[[package]] [[package]]
name = "fluxer-i18n-auto" name = "fluxer-i18n-auto"
version = "0.1.0" version = "0.1.0"
@@ -2097,11 +2116,9 @@ dependencies = [
"fluxer-svc", "fluxer-svc",
"hmac 0.13.0", "hmac 0.13.0",
"infer", "infer",
"mime_guess",
"moka", "moka",
"regex", "regex",
"reqwest", "reqwest",
"rmp-serde",
"scraper", "scraper",
"serde", "serde",
"serde_json", "serde_json",
+1
View File
@@ -5,6 +5,7 @@ members = [
"fluxer_common", "fluxer_common",
"fluxer_marketing", "fluxer_marketing",
"fluxer_media_proxy", "fluxer_media_proxy",
"fluxer_gifs",
"fluxer_svc", "fluxer_svc",
"fluxer_messages", "fluxer_messages",
"fluxer_snowflakes", "fluxer_snowflakes",
-1
View File
@@ -132,7 +132,6 @@ FLUXER_SEARCH_TLS_REJECT_UNAUTHORIZED=false
FLUXER_STRIPE_ENABLED=false FLUXER_STRIPE_ENABLED=false
FLUXER_NCMEC_ENABLED=false FLUXER_NCMEC_ENABLED=false
FLUXER_CLAMAV_ENABLED=false FLUXER_CLAMAV_ENABLED=false
FLUXER_GIF_PROVIDER=tenor
FLUXER_DISCOVERY_ENABLED=true FLUXER_DISCOVERY_ENABLED=true
FLUXER_SELF_HOSTED=true FLUXER_SELF_HOSTED=true
FLUXER_DISABLE_RATE_LIMITS=true FLUXER_DISABLE_RATE_LIMITS=true
+9
View File
@@ -0,0 +1,9 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
apiVersion: v2
name: gifs
version: 0.1.0
dependencies:
- name: svc-common
version: 0.1.0
repository: file://../svc-common
+13
View File
@@ -0,0 +1,13 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
{{ include "svc-common.statefulset" . }}
---
{{ include "svc-common.deployment" . }}
---
{{ include "svc-common.headless-service" . }}
---
{{ include "svc-common.service" . }}
---
{{ include "svc-common.pdb" . }}
---
{{ include "svc-common.router-pdb" . }}
+32
View File
@@ -0,0 +1,32 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
svc:
shard:
replicas: 4
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
memory: 1Gi
router:
replicas: 3
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
memory: 512Mi
cache:
maxEntries: 500000
ttlMs: '30000'
extraEnv:
- name: FLUXER_MEDIA_PROXY_ENDPOINT
value: http://media-proxy:8080
- name: FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT
value: https://fluxerusercontent.com
- name: FLUXER_MEDIA_PROXY_SECRET_KEY
valueFrom:
secretKeyRef:
name: fluxer-media-proxy-v2-env
key: FLUXER_MEDIA_PROXY_SECRET_KEY
+55
View File
@@ -0,0 +1,55 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
global:
namespace: fluxer
imagePullSecret: ghcr-pull-secret
registry: ""
svc:
name: gifs
image: fluxer-gifs
tag: ''
shard:
replicas: 2
port: 8090
minReadySeconds: 10
terminationGracePeriodSeconds: 60
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
memory: 512Mi
router:
replicas: 2
port: 8090
minReadySeconds: 10
maxSurge: 1
maxUnavailable: 0
terminationGracePeriodSeconds: 60
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
memory: 256Mi
nats:
url: nats://nats-core:4222
cache:
maxEntries: 250000
ttlMs: '30000'
extraEnv:
- name: FLUXER_MEDIA_PROXY_ENDPOINT
value: http://media-proxy:8080
- name: FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT
value: https://fluxerusercontent.com
- name: FLUXER_MEDIA_PROXY_SECRET_KEY
valueFrom:
secretKeyRef:
name: fluxer-media-proxy-v2-env
key: FLUXER_MEDIA_PROXY_SECRET_KEY
nodeSelector: {}
tolerations: []
pdb:
minAvailable: '50%'
+2
View File
@@ -27,6 +27,8 @@ [email protected]
LIVEKIT_API_KEY=fluxer LIVEKIT_API_KEY=fluxer
LIVEKIT_API_SECRET=CHANGE_ME LIVEKIT_API_SECRET=CHANGE_ME
FLUXER_KLIPY_API_KEY=
FLUXER_EMAIL_ENABLED=false FLUXER_EMAIL_ENABLED=false
FLUXER_EMAIL_PROVIDER=none FLUXER_EMAIL_PROVIDER=none
FLUXER_EMAIL_FROM_EMAIL=[email protected] FLUXER_EMAIL_FROM_EMAIL=[email protected]
+27
View File
@@ -49,6 +49,8 @@ x-fluxer-env: &fluxer-env
FLUXER_LIVEKIT_API_SECRET: ${LIVEKIT_API_SECRET:?set LIVEKIT_API_SECRET in .env} FLUXER_LIVEKIT_API_SECRET: ${LIVEKIT_API_SECRET:?set LIVEKIT_API_SECRET in .env}
FLUXER_LIVEKIT_WEBHOOK_URL: http://api:8080/webhooks/livekit FLUXER_LIVEKIT_WEBHOOK_URL: http://api:8080/webhooks/livekit
FLUXER_KLIPY_API_KEY: ${FLUXER_KLIPY_API_KEY:-}
FLUXER_EMAIL_ENABLED: ${FLUXER_EMAIL_ENABLED:-false} FLUXER_EMAIL_ENABLED: ${FLUXER_EMAIL_ENABLED:-false}
FLUXER_EMAIL_PROVIDER: ${FLUXER_EMAIL_PROVIDER:-none} FLUXER_EMAIL_PROVIDER: ${FLUXER_EMAIL_PROVIDER:-none}
FLUXER_EMAIL_FROM_EMAIL: ${FLUXER_EMAIL_FROM_EMAIL:-noreply@localhost} FLUXER_EMAIL_FROM_EMAIL: ${FLUXER_EMAIL_FROM_EMAIL:-noreply@localhost}
@@ -210,6 +212,8 @@ services:
nats: {condition: service_started} nats: {condition: service_started}
meilisearch: {condition: service_started} meilisearch: {condition: service_started}
seaweedfs-init: {condition: service_completed_successfully} seaweedfs-init: {condition: service_completed_successfully}
gifs: {condition: service_started}
gifs-shard: {condition: service_started}
snowflakes: {condition: service_started} snowflakes: {condition: service_started}
snowflakes-shard: {condition: service_started} snowflakes-shard: {condition: service_started}
messages: {condition: service_started} messages: {condition: service_started}
@@ -319,6 +323,29 @@ services:
depends_on: depends_on:
nats: {condition: service_started} nats: {condition: service_started}
gifs:
<<: *fluxer-service
image: ${FLUXER_REGISTRY:-ghcr.io/${FLUXER_REGISTRY_OWNER:-fluxerapp}}/fluxer-gifs:${FLUXER_IMAGE_TAG:-v1}
environment:
<<: *fluxer-env
FLUXER_SVC_NAME: gifs
FLUXER_SVC_MODE: router
FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}/media
depends_on:
nats: {condition: service_started}
gifs-shard:
<<: *fluxer-service
image: ${FLUXER_REGISTRY:-ghcr.io/${FLUXER_REGISTRY_OWNER:-fluxerapp}}/fluxer-gifs:${FLUXER_IMAGE_TAG:-v1}
environment:
<<: *fluxer-env
FLUXER_SVC_NAME: gifs
FLUXER_SVC_MODE: shard
FLUXER_SVC_SHARD_ID: "0"
FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}/media
depends_on:
nats: {condition: service_started}
messages: messages:
<<: *fluxer-service <<: *fluxer-service
image: ${FLUXER_REGISTRY:-ghcr.io/${FLUXER_REGISTRY_OWNER:-fluxerapp}}/fluxer-messages:${FLUXER_IMAGE_TAG:-v1} image: ${FLUXER_REGISTRY:-ghcr.io/${FLUXER_REGISTRY_OWNER:-fluxerapp}}/fluxer-messages:${FLUXER_IMAGE_TAG:-v1}
+3 -19
View File
@@ -12594,20 +12594,8 @@
"properties": { "properties": {
"gif": { "gif": {
"type": "object", "type": "object",
"properties": { "properties": {"klipy_api_key_set": {"type": "boolean"}, "effective_available": {"type": "boolean"}},
"provider": {"nullable": true, "enum": ["tenor", "klipy"], "type": "string"}, "required": ["klipy_api_key_set", "effective_available"]
"effective_provider": {"enum": ["tenor", "klipy"], "type": "string"},
"tenor_api_key_set": {"type": "boolean"},
"klipy_api_key_set": {"type": "boolean"},
"effective_available": {"type": "boolean"}
},
"required": [
"provider",
"effective_provider",
"tenor_api_key_set",
"klipy_api_key_set",
"effective_available"
]
}, },
"youtube": { "youtube": {
"type": "object", "type": "object",
@@ -12990,11 +12978,7 @@
"gif": { "gif": {
"nullable": true, "nullable": true,
"type": "object", "type": "object",
"properties": { "properties": {"klipy_api_key": {"nullable": true, "type": "string", "maxLength": 4096}}
"provider": {"nullable": true, "enum": ["tenor", "klipy"], "type": "string"},
"tenor_api_key": {"nullable": true, "type": "string", "maxLength": 4096},
"klipy_api_key": {"nullable": true, "type": "string", "maxLength": 4096}
}
}, },
"youtube": { "youtube": {
"nullable": true, "nullable": true,
@@ -100,12 +100,6 @@ pub struct InstanceIntegrationsResponse {
#[derive(Clone, Debug, Default, Deserialize, Serialize)] #[derive(Clone, Debug, Default, Deserialize, Serialize)]
pub struct InstanceGifIntegrationResponse { pub struct InstanceGifIntegrationResponse {
pub provider: Option<String>,
#[serde(default)]
pub effective_provider: String,
#[serde(default)]
pub tenor_api_key_set: bool,
#[serde(default)]
pub klipy_api_key_set: bool, pub klipy_api_key_set: bool,
#[serde(default)] #[serde(default)]
pub effective_available: bool, pub effective_available: bool,
@@ -549,10 +543,6 @@ pub struct InstanceIntegrationsUpdateRequest {
#[derive(Clone, Debug, Default, Serialize)] #[derive(Clone, Debug, Default, Serialize)]
pub struct InstanceGifIntegrationUpdateRequest { pub struct InstanceGifIntegrationUpdateRequest {
#[serde(skip_serializing_if = "Option::is_none")]
pub provider: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tenor_api_key: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub klipy_api_key: Option<String>, pub klipy_api_key: Option<String>,
} }
@@ -603,8 +603,6 @@ fn build_integrations_update(form: &MultiValueForm) -> InstanceConfigUpdateReque
policy: None, policy: None,
integrations: Some(InstanceIntegrationsUpdateRequest { integrations: Some(InstanceIntegrationsUpdateRequest {
gif: Some(InstanceGifIntegrationUpdateRequest { gif: Some(InstanceGifIntegrationUpdateRequest {
provider: clean("integration_gif_provider"),
tenor_api_key: clean("integration_tenor_api_key"),
klipy_api_key: clean("integration_klipy_api_key"), klipy_api_key: clean("integration_klipy_api_key"),
}), }),
youtube: Some(InstanceYoutubeIntegrationUpdateRequest { youtube: Some(InstanceYoutubeIntegrationUpdateRequest {
@@ -408,11 +408,6 @@ fn integrations_config_section(
csrf_token: &str, csrf_token: &str,
integrations: &InstanceIntegrationsResponse, integrations: &InstanceIntegrationsResponse,
) -> Markup { ) -> Markup {
let gif_provider = integrations
.gif
.provider
.as_deref()
.unwrap_or(integrations.gif.effective_provider.as_str());
let captcha_provider = integrations let captcha_provider = integrations
.captcha .captcha
.provider .provider
@@ -435,15 +430,9 @@ fn integrations_config_section(
div class="flex flex-wrap items-center gap-2" { div class="flex flex-wrap items-center gap-2" {
h3 class="text-sm font-semibold text-neutral-900" { "GIF provider" } h3 class="text-sm font-semibold text-neutral-900" { "GIF provider" }
(secret_badge("KLIPY key", integrations.gif.klipy_api_key_set)) (secret_badge("KLIPY key", integrations.gif.klipy_api_key_set))
(secret_badge("Tenor key", integrations.gif.tenor_api_key_set))
} }
div class="grid grid-cols-1 gap-4 sm:grid-cols-3" { div class="grid grid-cols-1 gap-4" {
(select_input("integration_gif_provider", "Provider", &[
("klipy", "KLIPY"),
("tenor", "Tenor"),
], gif_provider))
(password_input("integration_klipy_api_key", "KLIPY API key", Some("Leave blank to keep the current key."))) (password_input("integration_klipy_api_key", "KLIPY API key", Some("Leave blank to keep the current key.")))
(password_input("integration_tenor_api_key", "Tenor API key", Some("Leave blank to keep the current key.")))
} }
} }
-6
View File
@@ -360,15 +360,9 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
bluesky: master.auth.bluesky as BlueskyOAuthConfig, bluesky: master.auth.bluesky as BlueskyOAuthConfig,
}, },
cookie: master.cookie, cookie: master.cookie,
gif: {
provider: master.integrations.gif.provider,
},
klipy: { klipy: {
apiKey: master.integrations.klipy.api_key, apiKey: master.integrations.klipy.api_key,
}, },
tenor: {
apiKey: master.integrations.tenor.api_key,
},
youtube: { youtube: {
apiKey: master.integrations.youtube.api_key, apiKey: master.integrations.youtube.api_key,
}, },
@@ -290,8 +290,6 @@ export function InstanceConfigAdminController(app: HonoApp) {
await instanceConfigRepository.setInstanceIntegrationsConfig({ await instanceConfigRepository.setInstanceIntegrationsConfig({
gif: data.integrations.gif gif: data.integrations.gif
? omitUndefinedFields({ ? omitUndefinedFields({
provider: readOptionalField(data.integrations.gif, 'provider'),
tenor_api_key: readOptionalField(data.integrations.gif, 'tenor_api_key'),
klipy_api_key: readOptionalField(data.integrations.gif, 'klipy_api_key'), klipy_api_key: readOptionalField(data.integrations.gif, 'klipy_api_key'),
}) })
: undefined, : undefined,
+1 -1
View File
@@ -68,7 +68,7 @@ export function configureMiddleware(routes: HonoApp, options: MiddlewarePipeline
enabled: true, enabled: true,
logger, logger,
trustClientIpHeader, trustClientIpHeader,
clientIpHeaderName: clientIpHeaderName ?? 'x-forwarded-for', clientIpHeaderName: resolvedHeader,
}), }),
); );
} }
-6
View File
@@ -271,15 +271,9 @@ export interface APIConfig {
domain: string; domain: string;
secure: boolean; secure: boolean;
}; };
gif: {
provider: 'klipy' | 'tenor';
};
klipy: { klipy: {
apiKey?: string; apiKey?: string;
}; };
tenor: {
apiKey?: string;
};
youtube: { youtube: {
apiKey?: string; apiKey?: string;
}; };
@@ -2,6 +2,7 @@
import {ResolveGifUrlsBodySchema, ResolveGifUrlsResponse} from '@fluxer/schema/src/domains/gif/FavoriteGifSchemas'; import {ResolveGifUrlsBodySchema, ResolveGifUrlsResponse} from '@fluxer/schema/src/domains/gif/FavoriteGifSchemas';
import type {Context} from 'hono'; import type {Context} from 'hono';
import {resolveGifRequestCountry} from '../gif/GifRequestCountry';
import {DefaultUserOnly, LoginRequired} from '../middleware/AuthMiddleware'; import {DefaultUserOnly, LoginRequired} from '../middleware/AuthMiddleware';
import {RateLimitMiddleware} from '../middleware/RateLimitMiddleware'; import {RateLimitMiddleware} from '../middleware/RateLimitMiddleware';
import {OpenAPI} from '../middleware/ResponseTypeMiddleware'; import {OpenAPI} from '../middleware/ResponseTypeMiddleware';
@@ -11,8 +12,8 @@ import type {HonoApp, HonoEnv} from '../types/HonoEnv';
import {Validator} from '../Validator'; import {Validator} from '../Validator';
import {resolveFavoriteGifEntry} from './FavoriteGifResolver'; import {resolveFavoriteGifEntry} from './FavoriteGifResolver';
function getCountry(ctx: Context<HonoEnv>): string { async function getCountry(ctx: Context<HonoEnv>): Promise<string> {
return ctx.req.header('CF-IPCountry') || 'US'; return resolveGifRequestCountry(ctx.req.raw);
} }
function getLocale(ctx: Context<HonoEnv>): string { function getLocale(ctx: Context<HonoEnv>): string {
@@ -42,7 +43,7 @@ export function FavoriteGifController(app: HonoApp) {
const gifService = ctx.get('gifService'); const gifService = ctx.get('gifService');
const unfurlerService = getUnfurlerService(); const unfurlerService = getUnfurlerService();
const locale = getLocale(ctx); const locale = getLocale(ctx);
const country = getCountry(ctx); const country = await getCountry(ctx);
const entries = await Promise.all( const entries = await Promise.all(
urls.map((url) => resolveFavoriteGifEntry({url, locale, country, gifService, mediaService, unfurlerService})), urls.map((url) => resolveFavoriteGifEntry({url, locale, country, gifService, mediaService, unfurlerService})),
); );
@@ -0,0 +1,87 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
import {describe, expect, it, vi} from 'vitest';
import {GifService} from '../gif/GifService';
import type {IGifProvider} from '../gif/IGifProvider';
import type {IMediaService} from '../infrastructure/IMediaService';
import type {IUnfurlerService} from '../infrastructure/IUnfurlerService';
import {resolveFavoriteGifEntry} from './FavoriteGifResolver';
function createProvider(gif: GifResponse): IGifProvider {
return {
meta: {
name: 'klipy',
displayName: 'KLIPY',
attributionRequired: true,
},
isAvailable: async () => true,
search: async () => [],
registerShare: async () => undefined,
getFeatured: async () => ({gifs: [], categories: []}),
getTrendingGifs: async () => [],
suggest: async () => [],
resolveByUrl: async () => gif,
buildShareUrl: (slug) => `https://klipy.com/gifs/${slug}`,
extractSlugFromUrl: () => gif.slug,
};
}
describe('resolveFavoriteGifEntry', () => {
it('returns provider GIF media without probing generic media fallbacks', async () => {
const gif: GifResponse = {
id: 'goatplaybanjo-chat-4',
slug: 'goatplaybanjo-chat-4',
provider: 'klipy',
title: 'Goatplaybanjo Chat',
url: 'https://klipy.com/gifs/goatplaybanjo-chat-4',
src: 'https://static.klipy.example/fallback.gif',
proxy_src: 'https://media.example/fallback.gif',
width: 120,
height: 100,
media: {
tinygif: {
src: 'https://static.klipy.example/tiny.gif',
proxy_src: 'https://media.example/tiny.gif',
width: 80,
height: 60,
},
webm: {
src: 'https://static.klipy.example/full.webm',
proxy_src: 'https://media.example/full.webm',
width: 220,
height: 229,
},
},
placeholder: 'thumbhash',
};
const mediaService = {
getMetadata: vi.fn(),
getExternalMediaProxyURL: vi.fn(),
} as unknown as IMediaService;
const unfurlerService = {
unfurl: vi.fn(),
} as unknown as IUnfurlerService;
await expect(
resolveFavoriteGifEntry({
url: gif.url,
locale: 'en-US',
country: 'US',
gifService: new GifService(createProvider(gif)),
mediaService,
unfurlerService,
}),
).resolves.toEqual({
url: gif.url,
proxy_url: 'https://media.example/full.webm',
width: 220,
height: 229,
media: gif.media,
content_type: 'video/webm',
placeholder: 'thumbhash',
});
expect(mediaService.getMetadata).not.toHaveBeenCalled();
expect(unfurlerService.unfurl).not.toHaveBeenCalled();
});
});
@@ -4,6 +4,7 @@ import {Logger} from '@fluxer/logger/src/Logger';
import type {ResolvedGifEntrySchema} from '@fluxer/schema/src/domains/gif/FavoriteGifSchemas'; import type {ResolvedGifEntrySchema} from '@fluxer/schema/src/domains/gif/FavoriteGifSchemas';
import type {GifMediaFormat, GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas'; import type {GifMediaFormat, GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
import type {EmbedMediaResponse} from '@fluxer/schema/src/domains/message/EmbedSchemas'; import type {EmbedMediaResponse} from '@fluxer/schema/src/domains/message/EmbedSchemas';
import {tryExtractGifProviderSlug} from '../gif/GifProviderUtils';
import type {GifService} from '../gif/GifService'; import type {GifService} from '../gif/GifService';
import type {IGifProvider} from '../gif/IGifProvider'; import type {IGifProvider} from '../gif/IGifProvider';
import type {IMediaService, MediaProxyMetadataResponse} from '../infrastructure/IMediaService'; import type {IMediaService, MediaProxyMetadataResponse} from '../infrastructure/IMediaService';
@@ -91,12 +92,9 @@ async function resolveProviderGifUrl({
country: string; country: string;
gifService: GifService; gifService: GifService;
}): Promise<GifResponse | null> { }): Promise<GifResponse | null> {
for (const provider of gifService.listProviders()) { const provider = gifService.getProvider();
if (!(await provider.isAvailable()) || !provider.extractSlugFromUrl(url)) continue; if (!(await tryExtractGifProviderSlug(provider, url))) return null;
const gif = await resolveProviderUrl(provider, {url, locale, country}); return resolveProviderUrl(provider, {url, locale, country});
if (gif) return gif;
}
return null;
} }
async function resolveProviderUrl( async function resolveProviderUrl(
@@ -18,8 +18,12 @@ import {createAttachmentID, createMemeID, userIdToChannelId} from '../BrandedTyp
import {Config} from '../Config'; import {Config} from '../Config';
import type {ChannelService} from '../channel/services/ChannelService'; import type {ChannelService} from '../channel/services/ChannelService';
import {makeAttachmentCdnKey, makeAttachmentCdnUrl} from '../channel/services/message/MessageHelpers'; import {makeAttachmentCdnKey, makeAttachmentCdnUrl} from '../channel/services/message/MessageHelpers';
import {
isOptionalGifProviderError,
type ResolvedGifProviderSlug,
tryExtractGifProviderSlug,
} from '../gif/GifProviderUtils';
import type {GifService} from '../gif/GifService'; import type {GifService} from '../gif/GifService';
import type {IGifProvider} from '../gif/IGifProvider';
import type {MediaProxyMetadataResponse} from '../infrastructure/IMediaService'; import type {MediaProxyMetadataResponse} from '../infrastructure/IMediaService';
import type {IStorageService} from '../infrastructure/IStorageService'; import type {IStorageService} from '../infrastructure/IStorageService';
import type {IUnfurlerService} from '../infrastructure/IUnfurlerService'; import type {IUnfurlerService} from '../infrastructure/IUnfurlerService';
@@ -122,7 +126,7 @@ export class FavoriteMemeService {
if (!message) { if (!message) {
throw new UnknownMessageError(); throw new UnknownMessageError();
} }
const media = this.findMediaInMessage(message, attachmentId, embedIndex); const media = await this.findMediaInMessage(message, attachmentId, embedIndex);
if (!media) { if (!media) {
throw InputValidationError.fromCode('media', ValidationErrorCodes.NO_VALID_MEDIA_IN_MESSAGE); throw InputValidationError.fromCode('media', ValidationErrorCodes.NO_VALID_MEDIA_IN_MESSAGE);
} }
@@ -319,18 +323,9 @@ export class FavoriteMemeService {
throw new MediaMetadataError('URL'); throw new MediaMetadataError('URL');
} }
let contentHash = metadata.content_hash; let contentHash = metadata.content_hash;
const fileData = Buffer.from(metadata.base64 ?? '', 'base64'); const resolvedGif = await this.resolveGifFromInputs({slug: gifSlug, providerName: gifProvider, url});
const resolvedGif = this.resolveGifFromInputs({slug: gifSlug, providerName: gifProvider, url});
if (resolvedGif) { if (resolvedGif) {
const canonicalUrl = resolvedGif.provider.buildShareUrl(resolvedGif.slug); contentHash = await this.resolveProviderGifContentHash(resolvedGif, contentHash);
const unfurled = await this.unfurlerService.unfurl(canonicalUrl, 'allow');
if (unfurled.length > 0 && unfurled[0].video?.content_hash) {
contentHash = unfurled[0].video.content_hash;
Logger.debug(
{provider: resolvedGif.provider.meta.name, gifSlug: resolvedGif.slug, contentHash},
'Using unfurled video content_hash for provider GIF',
);
}
} }
const existingMemes = await this.favoriteMemeRepository.findByUserId(user.id); const existingMemes = await this.favoriteMemeRepository.findByUserId(user.id);
const duplicate = existingMemes.find((meme) => meme.contentHash === contentHash); const duplicate = existingMemes.find((meme) => meme.contentHash === contentHash);
@@ -343,6 +338,7 @@ export class FavoriteMemeService {
const userChannelId = userIdToChannelId(user.id); const userChannelId = userIdToChannelId(user.id);
const newAttachmentId = createAttachmentID(await this.apiContext.services.snowflake.generate()); const newAttachmentId = createAttachmentID(await this.apiContext.services.snowflake.generate());
const storageKey = makeAttachmentCdnKey(userChannelId, newAttachmentId, filename); const storageKey = makeAttachmentCdnKey(userChannelId, newAttachmentId, filename);
const fileData = Buffer.from(metadata.base64 ?? '', 'base64');
await this.storageService.uploadObject({ await this.storageService.uploadObject({
bucket: Config.s3.buckets.cdn, bucket: Config.s3.buckets.cdn,
key: storageKey, key: storageKey,
@@ -470,7 +466,7 @@ export class FavoriteMemeService {
} }
} }
private resolveGifFromInputs({ private async resolveGifFromInputs({
slug, slug,
providerName, providerName,
url, url,
@@ -478,34 +474,51 @@ export class FavoriteMemeService {
slug?: string | null; slug?: string | null;
providerName?: string | null; providerName?: string | null;
url: string; url: string;
}): { }): Promise<ResolvedGifProviderSlug | null> {
provider: IGifProvider;
slug: string;
} | null {
if (slug && providerName) { if (slug && providerName) {
const provider = this.gifService.getByName(providerName); const provider = this.gifService.getByName(providerName);
const trimmed = slug.trim(); const trimmed = slug.trim();
if (provider && trimmed) { if (provider && trimmed) {
const normalized = provider.extractSlugFromUrl(trimmed) ?? trimmed; const normalized = (await tryExtractGifProviderSlug(provider, trimmed)) ?? trimmed;
return {provider, slug: normalized}; return {provider, slug: normalized};
} }
} }
for (const provider of this.gifService.listProviders()) { const provider = this.gifService.getProvider();
const extracted = provider.extractSlugFromUrl(url); const extracted = await tryExtractGifProviderSlug(provider, url);
if (extracted) { return extracted ? {provider, slug: extracted} : null;
return {provider, slug: extracted};
}
}
return null;
} }
private detectGifFromUrl(url: string): { private async detectGifFromUrl(url: string): Promise<ResolvedGifProviderSlug | null> {
provider: IGifProvider;
slug: string;
} | null {
return this.resolveGifFromInputs({url}); return this.resolveGifFromInputs({url});
} }
private async resolveProviderGifContentHash(
resolvedGif: ResolvedGifProviderSlug,
fallbackContentHash: string,
): Promise<string> {
try {
const canonicalUrl = resolvedGif.provider.buildShareUrl(resolvedGif.slug);
const unfurled = await this.unfurlerService.unfurl(canonicalUrl, 'allow');
if (unfurled.length > 0 && unfurled[0].video?.content_hash) {
Logger.debug(
{
provider: resolvedGif.provider.meta.name,
gifSlug: resolvedGif.slug,
contentHash: unfurled[0].video.content_hash,
},
'Using unfurled video content_hash for provider GIF',
);
return unfurled[0].video.content_hash;
}
} catch (error) {
if (!isOptionalGifProviderError(error)) {
throw error;
}
Logger.debug({error, provider: resolvedGif.provider.meta.name}, 'Skipping unavailable GIF provider enrichment');
}
return fallbackContentHash;
}
private resolveFavoriteMemeName(name: string | undefined | null, fallbackFilename: string): string { private resolveFavoriteMemeName(name: string | undefined | null, fallbackFilename: string): string {
const normalizedInput = typeof name === 'string' ? name.trim() : ''; const normalizedInput = typeof name === 'string' ? name.trim() : '';
const fallbackName = fallbackFilename.trim() || 'favorite meme'; const fallbackName = fallbackFilename.trim() || 'favorite meme';
@@ -517,11 +530,11 @@ export class FavoriteMemeService {
return finalName; return finalName;
} }
private findMediaInMessage( private async findMediaInMessage(
message: Message, message: Message,
preferredAttachmentId?: string, preferredAttachmentId?: string,
preferredEmbedIndex?: number, preferredEmbedIndex?: number,
): FavoriteMemeMedia | null { ): Promise<FavoriteMemeMedia | null> {
const attachments = this.getMessageAttachmentCandidates(message); const attachments = this.getMessageAttachmentCandidates(message);
const embeds = this.getMessageEmbedCandidates(message); const embeds = this.getMessageEmbedCandidates(message);
if (preferredEmbedIndex !== undefined) { if (preferredEmbedIndex !== undefined) {
@@ -553,7 +566,7 @@ export class FavoriteMemeService {
} }
} }
for (const embed of embeds) { for (const embed of embeds) {
const media = this.mediaFromEmbed(embed, 'media'); const media = await this.mediaFromEmbed(embed, 'media');
if (media) return media; if (media) return media;
} }
return null; return null;
@@ -591,7 +604,10 @@ export class FavoriteMemeService {
}; };
} }
private mediaFromEmbed(embed: MessageEmbedCandidate, fallbackFilename: string): FavoriteMemeMedia | null { private async mediaFromEmbed(
embed: MessageEmbedCandidate,
fallbackFilename: string,
): Promise<FavoriteMemeMedia | null> {
const media = embed.image || embed.video || embed.thumbnail; const media = embed.image || embed.video || embed.thumbnail;
if (!media?.url) { if (!media?.url) {
return null; return null;
@@ -603,7 +619,7 @@ export class FavoriteMemeService {
} }
const isExternal = !this.isInternalCDNUrl(media.url); const isExternal = !this.isInternalCDNUrl(media.url);
const isGifv = embed.type === 'gifv' || isAnimatedEmbedMedia(media.contentType, media.flags); const isGifv = embed.type === 'gifv' || isAnimatedEmbedMedia(media.contentType, media.flags);
const detectedGif = embed.type === 'gifv' ? this.detectGifFromUrl(media.url) : null; const detectedGif = embed.type === 'gifv' ? await this.detectGifFromUrl(media.url) : null;
return { return {
isExternal, isExternal,
url: media.url, url: media.url,
@@ -0,0 +1,71 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {GIF_PROVIDER_HEADER, type GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
import {Hono} from 'hono';
import {describe, expect, it, vi} from 'vitest';
import type {HonoEnv} from '../types/HonoEnv';
import {GifController} from './GifController';
import {GifService} from './GifService';
import type {IGifProvider} from './IGifProvider';
function createProvider(gifs: Array<GifResponse>): IGifProvider {
return {
meta: {
name: 'klipy',
displayName: 'KLIPY',
attributionRequired: true,
},
isAvailable: async () => true,
search: vi.fn(async () => gifs),
registerShare: async () => undefined,
getFeatured: async () => ({gifs: [], categories: []}),
getTrendingGifs: async () => [],
suggest: async () => [],
resolveByUrl: async () => null,
buildShareUrl: (slug) => `https://klipy.com/gifs/${slug}`,
extractSlugFromUrl: () => null,
};
}
function createApp(gifService: GifService): Hono<HonoEnv> {
const app = new Hono<HonoEnv>({strict: true});
app.use('*', async (ctx, next) => {
ctx.set('gifService', gifService);
ctx.set('user', {
isBot: false,
suspiciousActivityFlags: 0,
} as HonoEnv['Variables']['user']);
ctx.set('authTokenType', 'session');
await next();
});
GifController(app);
return app;
}
describe('GifController', () => {
it('serves deprecated Tenor routes from the configured KLIPY provider', async () => {
const gifs: Array<GifResponse> = [
{
id: 'goatplaybanjo-chat-4',
slug: 'goatplaybanjo-chat-4',
provider: 'klipy',
title: 'Goatplaybanjo Chat',
url: 'https://klipy.com/gifs/goatplaybanjo-chat-4',
src: 'https://static.klipy.example/full.webm',
proxy_src: 'https://media.example/full.webm',
width: 220,
height: 229,
media: {},
placeholder: null,
},
];
const provider = createProvider(gifs);
const response = await createApp(new GifService(provider)).request('/tenor/search?q=cat&locale=en-US');
expect(response.status).toBe(200);
expect(response.headers.get('Deprecation')).toBe('true');
expect(response.headers.get(GIF_PROVIDER_HEADER)).toBe('klipy');
await expect(response.json()).resolves.toEqual(gifs);
expect(provider.search).toHaveBeenCalledWith({q: 'cat', locale: 'en_US', country: 'US'});
});
});
+11 -6
View File
@@ -17,6 +17,7 @@ import {RateLimitConfigs} from '../RateLimitConfig';
import type {HonoApp, HonoEnv} from '../types/HonoEnv'; import type {HonoApp, HonoEnv} from '../types/HonoEnv';
import {Validator} from '../Validator'; import {Validator} from '../Validator';
import {GifProviderHeaderMiddleware} from './GifProviderHeaderMiddleware'; import {GifProviderHeaderMiddleware} from './GifProviderHeaderMiddleware';
import {resolveGifRequestCountry} from './GifRequestCountry';
const TAGS = ['GIFs']; const TAGS = ['GIFs'];
@@ -34,8 +35,8 @@ const PREFIXES: ReadonlyArray<PrefixConfig> = [
]; ];
const DEPRECATION_NOTICE = 'Use /gifs/* instead - these vendor-specific paths are deprecated and will be removed.'; const DEPRECATION_NOTICE = 'Use /gifs/* instead - these vendor-specific paths are deprecated and will be removed.';
function getCountry(ctx: Context<HonoEnv>): string { async function getCountry(ctx: Context<HonoEnv>): Promise<string> {
return ctx.req.header('CF-IPCountry') || 'US'; return resolveGifRequestCountry(ctx.req.raw);
} }
function deprecationMiddleware(deprecated: boolean): MiddlewareHandler<HonoEnv> { function deprecationMiddleware(deprecated: boolean): MiddlewareHandler<HonoEnv> {
@@ -72,7 +73,8 @@ function registerRoutes(app: HonoApp, cfg: PrefixConfig) {
async (ctx) => { async (ctx) => {
const {q, locale} = ctx.req.valid('query'); const {q, locale} = ctx.req.valid('query');
const provider = await ctx.get('gifService').getActive(); const provider = await ctx.get('gifService').getActive();
return ctx.json(await provider.search({q, locale, country: getCountry(ctx)})); const country = await getCountry(ctx);
return ctx.json(await provider.search({q, locale, country}));
}, },
); );
app.get( app.get(
@@ -94,7 +96,8 @@ function registerRoutes(app: HonoApp, cfg: PrefixConfig) {
Validator('query', GifLocaleQuery), Validator('query', GifLocaleQuery),
async (ctx) => { async (ctx) => {
const provider = await ctx.get('gifService').getActive(); const provider = await ctx.get('gifService').getActive();
return ctx.json(await provider.getFeatured({locale: ctx.req.valid('query').locale, country: getCountry(ctx)})); const country = await getCountry(ctx);
return ctx.json(await provider.getFeatured({locale: ctx.req.valid('query').locale, country}));
}, },
); );
const trendingPath = prefix === '/gifs' ? `${prefix}/trending` : `${prefix}/trending-gifs`; const trendingPath = prefix === '/gifs' ? `${prefix}/trending` : `${prefix}/trending-gifs`;
@@ -117,10 +120,11 @@ function registerRoutes(app: HonoApp, cfg: PrefixConfig) {
Validator('query', GifLocaleQuery), Validator('query', GifLocaleQuery),
async (ctx) => { async (ctx) => {
const provider = await ctx.get('gifService').getActive(); const provider = await ctx.get('gifService').getActive();
const country = await getCountry(ctx);
return ctx.json( return ctx.json(
await provider.getTrendingGifs({ await provider.getTrendingGifs({
locale: ctx.req.valid('query').locale, locale: ctx.req.valid('query').locale,
country: getCountry(ctx), country,
}), }),
); );
}, },
@@ -145,7 +149,8 @@ function registerRoutes(app: HonoApp, cfg: PrefixConfig) {
async (ctx) => { async (ctx) => {
const {id, q, locale} = ctx.req.valid('json'); const {id, q, locale} = ctx.req.valid('json');
const provider = await ctx.get('gifService').getActive(); const provider = await ctx.get('gifService').getActive();
await provider.registerShare({id, q: q ?? '', locale, country: getCountry(ctx)}); const country = await getCountry(ctx);
await provider.registerShare({id, q: q ?? '', locale, country});
return ctx.body(null, 204); return ctx.body(null, 204);
}, },
); );
@@ -1,99 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {GifCategoryTagResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
import type {ICacheService} from '@pkgs/cache/src/ICacheService';
import {ms} from 'itty-time';
const ENRICHED_CATEGORIES_FRESH_MS = ms('30 minutes');
const ENRICHED_CATEGORIES_TTL_SECONDS = Math.floor(ms('6 hours') / 1000);
const SEEN_LOCALES_TTL_SECONDS = Math.floor(ms('7 days') / 1000);
export const REFRESH_LOCK_TTL_SECONDS = Math.floor(ms('5 minutes') / 1000);
interface EnrichedCategoriesEntry {
data: Array<GifCategoryTagResponse>;
timestamp: number;
}
function enrichedCategoriesCacheKey(provider: string, locale: string, country: string): string {
return `gif:featured_categories_enriched:${provider}:${locale}:${country}`;
}
function seenLocalesSetKey(provider: string): string {
return `gif:featured_categories_seen_locales:${provider}`;
}
function encodeLocaleMember(locale: string, country: string): string {
return `${locale}|${country}`;
}
function decodeLocaleMember(member: string): {
locale: string;
country: string;
} | null {
const idx = member.indexOf('|');
if (idx <= 0) return null;
const locale = member.slice(0, idx);
const country = member.slice(idx + 1);
if (!locale || !country) return null;
return {locale, country};
}
export function refreshLockKey(provider: string, locale: string, country: string): string {
return `gif:featured_categories_refresh_lock:${provider}:${locale}:${country}`;
}
export async function readEnrichedCategoriesCache(
cache: ICacheService,
provider: string,
locale: string,
country: string,
): Promise<{
data: Array<GifCategoryTagResponse>;
isStale: boolean;
} | null> {
const entry = await cache.get<EnrichedCategoriesEntry>(enrichedCategoriesCacheKey(provider, locale, country));
if (!entry) return null;
const isStale = Date.now() - entry.timestamp > ENRICHED_CATEGORIES_FRESH_MS;
return {data: entry.data, isStale};
}
export async function writeEnrichedCategoriesCache(
cache: ICacheService,
provider: string,
locale: string,
country: string,
data: Array<GifCategoryTagResponse>,
): Promise<void> {
const entry: EnrichedCategoriesEntry = {data, timestamp: Date.now()};
await cache.set(enrichedCategoriesCacheKey(provider, locale, country), entry, ENRICHED_CATEGORIES_TTL_SECONDS);
}
export async function trackSeenLocale(
cache: ICacheService,
provider: string,
locale: string,
country: string,
): Promise<void> {
await cache.sadd(seenLocalesSetKey(provider), encodeLocaleMember(locale, country), SEEN_LOCALES_TTL_SECONDS);
}
export async function listSeenLocales(
cache: ICacheService,
provider: string,
): Promise<
Array<{
locale: string;
country: string;
}>
> {
const members = await cache.smembers(seenLocalesSetKey(provider));
const out: Array<{
locale: string;
country: string;
}> = [];
for (const member of members) {
const decoded = decodeLocaleMember(member);
if (decoded) out.push(decoded);
}
return out;
}
@@ -25,7 +25,6 @@ function createProvider(): IGifProvider {
getFeatured: async () => ({gifs: [], categories: []}), getFeatured: async () => ({gifs: [], categories: []}),
getTrendingGifs: async () => [], getTrendingGifs: async () => [],
suggest: async () => [], suggest: async () => [],
refreshFeaturedCategories: async () => undefined,
resolveByUrl: async () => null, resolveByUrl: async () => null,
buildShareUrl: (slug) => `https://klipy.example/${slug}`, buildShareUrl: (slug) => `https://klipy.example/${slug}`,
extractSlugFromUrl: () => null, extractSlugFromUrl: () => null,
@@ -47,7 +46,7 @@ function createApp(gifService?: GifService): Hono<HonoEnv> {
describe('GifProviderHeaderMiddleware', () => { describe('GifProviderHeaderMiddleware', () => {
it('emits active GIF provider metadata headers', async () => { it('emits active GIF provider metadata headers', async () => {
const gifService = new GifService({providers: [createProvider()], activeName: 'klipy'}); const gifService = new GifService(createProvider());
const response = await createApp(gifService).request('/probe'); const response = await createApp(gifService).request('/probe');
expect(response.headers.get(GIF_PROVIDER_HEADER)).toBe('klipy'); expect(response.headers.get(GIF_PROVIDER_HEADER)).toBe('klipy');
@@ -13,8 +13,8 @@ export const GifProviderHeaderMiddleware = createMiddleware<HonoEnv>(async (ctx,
await next(); await next();
const gifService = ctx.get('gifService') as GifService | undefined; const gifService = ctx.get('gifService') as GifService | undefined;
if (!gifService) return; if (!gifService) return;
const provider = gifService.getByName(await gifService.getActiveName()); const provider = await gifService.getActive().catch(() => null);
if (!provider || !(await provider.isAvailable())) return; if (!provider) return;
ctx.header(GIF_PROVIDER_HEADER, provider.meta.name); ctx.header(GIF_PROVIDER_HEADER, provider.meta.name);
ctx.header(GIF_PROVIDER_DISPLAY_NAME_HEADER, provider.meta.displayName); ctx.header(GIF_PROVIDER_DISPLAY_NAME_HEADER, provider.meta.displayName);
ctx.header(GIF_PROVIDER_ATTRIBUTION_HEADER, provider.meta.attributionRequired ? 'true' : 'false'); ctx.header(GIF_PROVIDER_ATTRIBUTION_HEADER, provider.meta.attributionRequired ? 'true' : 'false');
@@ -0,0 +1,82 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {ServiceUnavailableError} from '@fluxer/errors/src/domains/core/ServiceUnavailableError';
import {describe, expect, it} from 'vitest';
import {tryExtractGifProviderSlug} from './GifProviderUtils';
import type {IGifProvider} from './IGifProvider';
function createProvider(overrides: Partial<IGifProvider> = {}): IGifProvider {
return {
meta: {
name: 'klipy',
displayName: 'KLIPY',
attributionRequired: true,
},
isAvailable: async () => true,
search: async () => [],
registerShare: async () => undefined,
getFeatured: async () => ({gifs: [], categories: []}),
getTrendingGifs: async () => [],
suggest: async () => [],
resolveByUrl: async () => null,
buildShareUrl: (slug) => `https://klipy.example/${slug}`,
extractSlugFromUrl: () => 'slug',
...overrides,
};
}
describe('GifProviderUtils', () => {
it('returns a trimmed slug from an available provider', async () => {
await expect(
tryExtractGifProviderSlug(
createProvider({
extractSlugFromUrl: () => ' goatplaybanjo-chat-4 ',
}),
'https://klipy.com/gifs/goatplaybanjo-chat-4',
),
).resolves.toBe('goatplaybanjo-chat-4');
});
it('treats unavailable optional provider failures as no match', async () => {
await expect(
tryExtractGifProviderSlug(
createProvider({
extractSlugFromUrl: () => {
throw new ServiceUnavailableError();
},
}),
'https://klipy.com/gifs/goatplaybanjo-chat-4',
),
).resolves.toBeNull();
});
it('does not check availability when a provider cannot extract a slug', async () => {
let availabilityChecks = 0;
await expect(
tryExtractGifProviderSlug(
createProvider({
isAvailable: async () => {
availabilityChecks += 1;
return true;
},
extractSlugFromUrl: () => null,
}),
'https://example.com/media.gif',
),
).resolves.toBeNull();
expect(availabilityChecks).toBe(0);
});
it('returns null when the configured provider is unavailable', async () => {
await expect(
tryExtractGifProviderSlug(
createProvider({
isAvailable: async () => false,
extractSlugFromUrl: () => 'matched',
}),
'https://klipy.com/gifs/matched',
),
).resolves.toBeNull();
});
});
@@ -0,0 +1,30 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';
import {ServiceUnavailableError} from '@fluxer/errors/src/domains/core/ServiceUnavailableError';
import type {IGifProvider} from './IGifProvider';
export type ResolvedGifProviderSlug = {
provider: IGifProvider;
slug: string;
};
export function isOptionalGifProviderError(error: unknown): boolean {
return error instanceof FeatureTemporarilyDisabledError || error instanceof ServiceUnavailableError;
}
export async function tryExtractGifProviderSlug(provider: IGifProvider, value: string): Promise<string | null> {
try {
const slug = provider.extractSlugFromUrl(value);
const trimmed = slug?.trim() ?? '';
if (trimmed.length === 0) {
return null;
}
return (await provider.isAvailable()) ? trimmed : null;
} catch (error) {
if (isOptionalGifProviderError(error)) {
return null;
}
throw error;
}
}
@@ -0,0 +1,49 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {GeoipResult} from '@pkgs/geoip/src/GeoipLookup';
import {beforeEach, describe, expect, it, vi} from 'vitest';
import {resolveGifRequestCountry} from './GifRequestCountry';
const {lookupGeoipMock} = vi.hoisted(() => ({
lookupGeoipMock: vi.fn(),
}));
vi.mock('../utils/IpUtils', () => ({
lookupGeoip: lookupGeoipMock,
}));
function geoip(countryCode: string | null): GeoipResult {
return {
countryCode,
normalizedIp: '203.0.113.10',
city: null,
region: null,
countryName: null,
};
}
describe('resolveGifRequestCountry', () => {
const req = new Request('https://fluxer.test/gifs/search');
beforeEach(() => {
lookupGeoipMock.mockReset();
});
it('uses the GeoIP country code for GIF provider requests', async () => {
lookupGeoipMock.mockResolvedValue(geoip('se'));
await expect(resolveGifRequestCountry(req)).resolves.toBe('SE');
});
it('falls back to US when GeoIP has no country', async () => {
lookupGeoipMock.mockResolvedValue(geoip(null));
await expect(resolveGifRequestCountry(req)).resolves.toBe('US');
});
it('falls back to US when GeoIP lookup fails', async () => {
lookupGeoipMock.mockRejectedValue(new Error('geoip unavailable'));
await expect(resolveGifRequestCountry(req)).resolves.toBe('US');
});
});
@@ -0,0 +1,14 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {lookupGeoip} from '../utils/IpUtils';
const KLIPY_DEFAULT_COUNTRY = 'US';
export async function resolveGifRequestCountry(req: Request): Promise<string> {
try {
const geoip = await lookupGeoip(req);
return geoip.countryCode?.trim().toUpperCase() || KLIPY_DEFAULT_COUNTRY;
} catch {
return KLIPY_DEFAULT_COUNTRY;
}
}
+10 -29
View File
@@ -1,46 +1,27 @@
// SPDX-License-Identifier: AGPL-3.0-or-later // SPDX-License-Identifier: AGPL-3.0-or-later
import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError'; import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';
import {Logger} from '../Logger';
import type {IGifProvider} from './IGifProvider'; import type {IGifProvider} from './IGifProvider';
export class GifService { export class GifService {
private readonly providersByName: Map<string, IGifProvider>; private readonly provider: IGifProvider;
private readonly activeName: string | (() => Promise<string>);
constructor(params: { constructor(provider: IGifProvider) {
providers: ReadonlyArray<IGifProvider>; this.provider = provider;
activeName: string | (() => Promise<string>); }
}) {
this.providersByName = new Map(params.providers.map((p) => [p.meta.name, p])); getProvider(): IGifProvider {
this.activeName = params.activeName; return this.provider;
if (typeof params.activeName === 'string' && !this.providersByName.has(params.activeName)) {
Logger.warn(
{activeName: params.activeName, registered: Array.from(this.providersByName.keys())},
'Active GIF provider is not registered; /gifs requests will fail until configuration is fixed',
);
}
} }
async getActive(): Promise<IGifProvider> { async getActive(): Promise<IGifProvider> {
const activeName = await this.getActiveName(); if (!(await this.provider.isAvailable())) {
const provider = this.providersByName.get(activeName);
if (!provider || !(await provider.isAvailable())) {
Logger.debug({activeName}, 'Active GIF provider unavailable');
throw new FeatureTemporarilyDisabledError(); throw new FeatureTemporarilyDisabledError();
} }
return provider; return this.provider;
}
async getActiveName(): Promise<string> {
return typeof this.activeName === 'string' ? this.activeName : this.activeName();
} }
getByName(name: string): IGifProvider | null { getByName(name: string): IGifProvider | null {
return this.providersByName.get(name) ?? null; return name === this.provider.meta.name ? this.provider : null;
}
listProviders(): ReadonlyArray<IGifProvider> {
return Array.from(this.providersByName.values());
} }
} }
-1
View File
@@ -19,7 +19,6 @@ export interface IGifProvider {
}>; }>;
getTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>>; getTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>>;
suggest(params: {q: string; locale: string}): Promise<Array<string>>; suggest(params: {q: string; locale: string}): Promise<Array<string>>;
refreshFeaturedCategories(params: {locale: string; country: string}): Promise<void>;
resolveByUrl(params: {url: string; locale: string; country: string}): Promise<GifResponse | null>; resolveByUrl(params: {url: string; locale: string; country: string}): Promise<GifResponse | null>;
buildShareUrl(slug: string): string; buildShareUrl(slug: string): string;
extractSlugFromUrl(url: string): string | null; extractSlugFromUrl(url: string): string | null;
-563
View File
@@ -1,563 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {FLUXER_USER_AGENT} from '@fluxer/constants/src/Core';
import type {GifCategoryTagResponse, GifMediaFormat, GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
import type {ICacheService} from '@pkgs/cache/src/ICacheService';
import {ms} from 'itty-time';
import {Config} from '../Config';
import type {IMediaService} from '../infrastructure/IMediaService';
import {Logger} from '../Logger';
import {getWorkerService} from '../middleware/ServiceRegistry';
import {EXTERNAL_RESPONSE_LIMITS} from '../utils/ExternalResponseLimits';
import * as FetchUtils from '../utils/FetchUtils';
import {isJsonRecord, parseJsonUnknown} from '../utils/JsonBoundaryUtils';
import {
REFRESH_LOCK_TTL_SECONDS,
readEnrichedCategoriesCache,
refreshLockKey,
trackSeenLocale,
writeEnrichedCategoriesCache,
} from './GifFeaturedCategoriesCache';
import type {GifProviderMeta, IGifProvider} from './IGifProvider';
const KLIPY_BASE_URL = 'https://api.klipy.com/v2';
const DEFAULT_CONTENT_FILTER = 'low';
const CLIENT_KEY = 'fluxer';
const MAX_RETRIES = 3;
const BACKOFF_BASE_DELAY = ms('1 second');
const CACHE_EXPIRATION_TIME = ms('5 minutes');
const KLIPY_PROVIDER_META: GifProviderMeta = {
name: 'klipy',
displayName: 'KLIPY',
attributionRequired: true,
};
interface KlipyFileEntry {
url?: string;
width?: number;
height?: number;
size?: number;
}
type KlipyFormatKey = 'gif' | 'webp' | 'mp4' | 'webm';
type KlipySizeKey = 'hd' | 'md' | 'sm' | 'xs';
type KlipyFileBucket = Partial<Record<KlipyFormatKey, KlipyFileEntry>>;
interface KlipyGif {
id: string;
slug?: string;
title: string;
itemurl: string;
file?: Partial<Record<KlipySizeKey, KlipyFileBucket>>;
media_formats?: {
webm?: {
url: string;
dims: [number, number];
};
};
}
const KLIPY_SIZE_PREFERENCE: ReadonlyArray<KlipySizeKey> = ['hd', 'md', 'sm', 'xs'];
const KLIPY_FORMAT_KEYS: ReadonlyArray<KlipyFormatKey> = ['webm', 'mp4', 'webp', 'gif'];
const KLIPY_PUBLIC_FORMAT_KEYS: Record<KlipySizeKey, Record<KlipyFormatKey, string>> = {
hd: {webm: 'webm', mp4: 'mp4', webp: 'webp', gif: 'gif'},
md: {webm: 'mediumwebm', mp4: 'mediummp4', webp: 'mediumwebp', gif: 'mediumgif'},
sm: {webm: 'tinywebm', mp4: 'tinymp4', webp: 'tinywebp', gif: 'tinygif'},
xs: {webm: 'nanowebm', mp4: 'nanomp4', webp: 'nanowebp', gif: 'nanogif'},
};
interface KlipyCategoryTag {
searchterm: string;
}
function isKlipyFileEntry(value: unknown): value is KlipyFileEntry {
return (
isJsonRecord(value) &&
(value.url === undefined || typeof value.url === 'string') &&
(value.width === undefined || typeof value.width === 'number') &&
(value.height === undefined || typeof value.height === 'number') &&
(value.size === undefined || typeof value.size === 'number')
);
}
function isKlipyFileBucket(value: unknown): value is KlipyFileBucket {
return isJsonRecord(value) && Object.values(value).every(isKlipyFileEntry);
}
function isKlipyFallbackMediaFormat(value: unknown): value is NonNullable<KlipyGif['media_formats']>['webm'] {
if (!isJsonRecord(value) || typeof value.url !== 'string' || !Array.isArray(value.dims)) return false;
return value.dims.length === 2 && value.dims.every((dimension) => typeof dimension === 'number');
}
function isKlipyGif(value: unknown): value is KlipyGif {
if (
!isJsonRecord(value) ||
typeof value.id !== 'string' ||
typeof value.title !== 'string' ||
typeof value.itemurl !== 'string'
) {
return false;
}
return (
(value.slug === undefined || typeof value.slug === 'string') &&
(value.file === undefined || (isJsonRecord(value.file) && Object.values(value.file).every(isKlipyFileBucket))) &&
(value.media_formats === undefined ||
(isJsonRecord(value.media_formats) &&
(value.media_formats.webm === undefined || isKlipyFallbackMediaFormat(value.media_formats.webm))))
);
}
function isKlipyCategoryTag(value: unknown): value is KlipyCategoryTag {
return isJsonRecord(value) && typeof value.searchterm === 'string';
}
function readResultsArray(value: unknown): Array<unknown> {
if (!isJsonRecord(value) || !Array.isArray(value.results)) {
throw new Error('KLIPY API response did not include a results array');
}
return value.results;
}
function readTagsArray(value: unknown): Array<unknown> {
if (!isJsonRecord(value) || !Array.isArray(value.tags)) {
throw new Error('KLIPY API response did not include a tags array');
}
return value.tags;
}
type CacheEntry<T> = {
data: T;
timestamp: number;
};
interface KlipyPath {
type: 'gif' | 'clip';
slug: string;
}
type GifApiKeyResolver = () => Promise<string | null>;
export class KlipyGifProvider implements IGifProvider {
readonly meta = KLIPY_PROVIDER_META;
private readonly FEATURED_CACHE_KEY = 'klipy:featured';
private readonly TRENDING_CACHE_KEY = 'klipy:trending';
private refreshingKeys: Map<string, boolean> = new Map();
constructor(
private cacheService: ICacheService,
private mediaService: IMediaService,
private apiKeyResolver: GifApiKeyResolver = async () => Config.klipy.apiKey || null,
) {}
async isAvailable(): Promise<boolean> {
return Boolean(await this.apiKeyResolver());
}
private async getApiKey(): Promise<string> {
const apiKey = await this.apiKeyResolver();
if (!apiKey) {
throw new Error('KLIPY API key is not configured');
}
return apiKey;
}
private createURL({endpoint, params}: {endpoint: string; params: Record<string, string | number | undefined>}): URL {
const url = new URL(`${KLIPY_BASE_URL}/${endpoint}`);
const defaultParams = {
client_key: CLIENT_KEY,
contentfilter: DEFAULT_CONTENT_FILTER,
...params,
};
for (const [key, value] of Object.entries(defaultParams)) {
if (value !== undefined) {
url.searchParams.append(key, value.toString());
}
}
return url;
}
private async fetchKlipyData(url: URL): Promise<unknown> {
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
try {
const response = await fetch(url.toString(), {
headers: {'User-Agent': FLUXER_USER_AGENT},
signal: AbortSignal.timeout(ms('30 seconds')),
});
if (!response.ok) {
throw new Error(`Failed to fetch KLIPY data: ${response.statusText}`);
}
const responseText = await FetchUtils.streamToStringWithLimit(response.body, {
maxBytes: EXTERNAL_RESPONSE_LIMITS.klipyApiBytes,
headers: response.headers,
url: response.url,
description: 'KLIPY API response',
});
return parseJsonUnknown(responseText);
} catch (error) {
if (attempt < MAX_RETRIES - 1) {
const delay = BACKOFF_BASE_DELAY * 2 ** attempt;
await new Promise((resolve) => setTimeout(resolve, delay));
} else {
throw error;
}
}
}
throw new Error('Exceeded maximum retries');
}
private async fetchAndTransformGifs(url: URL): Promise<Array<GifResponse>> {
const results = readResultsArray(await this.fetchKlipyData(url)).filter(isKlipyGif);
return results.map((gif) => this.transformKlipyGif(gif)).filter((gif): gif is GifResponse => gif !== null);
}
private async getCache<T>(key: string): Promise<{
data: T;
isStale: boolean;
} | null> {
const cached = await this.cacheService.get<CacheEntry<T>>(key);
if (!cached) return null;
const age = Date.now() - cached.timestamp;
const isStale = age > CACHE_EXPIRATION_TIME;
return {data: cached.data, isStale};
}
private async setCache<T>(key: string, data: T): Promise<void> {
const cacheEntry: CacheEntry<T> = {
data,
timestamp: Date.now(),
};
await this.cacheService.set(key, cacheEntry);
}
private triggerBackgroundRefresh<T>(key: string, refreshFn: () => Promise<T>): void {
if (this.refreshingKeys.get(key)) {
return;
}
this.refreshingKeys.set(key, true);
setImmediate(async () => {
try {
const freshData = await refreshFn();
await this.setCache(key, freshData);
} catch (error) {
Logger.debug({key, error}, `Background refresh failed for key ${key}`);
} finally {
this.refreshingKeys.delete(key);
}
});
}
async search(params: {q: string; locale: string; country: string}): Promise<Array<GifResponse>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'search',
params: {
key: apiKey,
q: params.q,
country: params.country,
locale: params.locale,
limit: 50,
},
});
return this.fetchAndTransformGifs(url);
}
async registerShare(params: {id: string; q: string; locale: string; country: string}): Promise<void> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'registershare',
params: {
key: apiKey,
id: params.id,
country: params.country,
locale: params.locale,
q: params.q,
},
});
await fetch(url.toString(), {
headers: {'User-Agent': FLUXER_USER_AGENT},
signal: AbortSignal.timeout(ms('30 seconds')),
});
}
async getFeatured(params: {locale: string; country: string}): Promise<{
gifs: Array<GifResponse>;
categories: Array<GifCategoryTagResponse>;
}> {
const cached = await this.getCache<{
gifs: Array<GifResponse>;
categories: Array<GifCategoryTagResponse>;
}>(this.FEATURED_CACHE_KEY);
if (cached) {
if (cached.isStale) {
this.triggerBackgroundRefresh(this.FEATURED_CACHE_KEY, () => this.fetchFeaturedData(params));
}
return cached.data;
}
const data = await this.fetchFeaturedData(params);
await this.setCache(this.FEATURED_CACHE_KEY, data);
return data;
}
private async fetchFeaturedData(params: {locale: string; country: string}): Promise<{
gifs: Array<GifResponse>;
categories: Array<GifCategoryTagResponse>;
}> {
const [gifs, categories] = await Promise.all([this.getFeaturedGifs(params), this.getFeaturedCategories(params)]);
return {gifs, categories};
}
async getTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
const cached = await this.getCache<Array<GifResponse>>(this.TRENDING_CACHE_KEY);
if (cached) {
if (cached.isStale) {
this.triggerBackgroundRefresh(this.TRENDING_CACHE_KEY, () => this.fetchTrendingGifs(params));
}
return cached.data;
}
const gifs = await this.fetchTrendingGifs(params);
await this.setCache(this.TRENDING_CACHE_KEY, gifs);
return gifs;
}
private async fetchTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'featured',
params: {
key: apiKey,
country: params.country,
locale: params.locale,
limit: 50,
},
});
return this.fetchAndTransformGifs(url);
}
async suggest(params: {q: string; locale: string}): Promise<Array<string>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'autocomplete',
params: {
key: apiKey,
q: params.q,
locale: params.locale,
},
});
return readResultsArray(await this.fetchKlipyData(url)).filter(
(result): result is string => typeof result === 'string',
);
}
async resolveByUrl(params: {url: string; locale: string; country: string}): Promise<GifResponse | null> {
const slug = this.extractSlugFromUrl(params.url);
if (!slug) return null;
const results = await this.search({q: slug, locale: params.locale, country: params.country});
return results.find((gif) => gif.slug === slug || this.extractSlugFromUrl(gif.url) === slug) ?? results[0] ?? null;
}
private async getFeaturedGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'featured',
params: {
key: apiKey,
country: params.country,
locale: params.locale,
limit: 1,
},
});
return this.fetchAndTransformGifs(url);
}
private async getFeaturedCategories(params: {
locale: string;
country: string;
}): Promise<Array<GifCategoryTagResponse>> {
trackSeenLocale(this.cacheService, this.meta.name, params.locale, params.country).catch((error) => {
Logger.debug({err: error, ...params}, 'Failed to track seen GIF locale');
});
const rawTags = await this.fetchRawCategoryTags(params);
const cached = await readEnrichedCategoriesCache(this.cacheService, this.meta.name, params.locale, params.country);
if (cached) {
if (cached.isStale) {
this.scheduleEnrichmentRefresh(params);
}
const byName = new Map(cached.data.map((entry) => [entry.name, entry]));
return rawTags.map((tag) => byName.get(tag.name) ?? tag);
}
this.scheduleEnrichmentRefresh(params);
return rawTags;
}
private async fetchRawCategoryTags(params: {
locale: string;
country: string;
}): Promise<Array<GifCategoryTagResponse>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'categories',
params: {
key: apiKey,
country: params.country,
locale: params.locale,
type: 'featured',
},
});
const tags = readTagsArray(await this.fetchKlipyData(url)).filter(isKlipyCategoryTag);
return tags
.filter((tag) => Boolean(tag.searchterm))
.map((tag) => ({
name: tag.searchterm,
src: '',
proxy_src: '',
gif: null,
}));
}
private scheduleEnrichmentRefresh(params: {locale: string; country: string}): void {
(async () => {
try {
const workerService = getWorkerService();
await workerService.addJob('refreshGifFeaturedCategories', {
provider: this.meta.name,
locale: params.locale,
country: params.country,
});
} catch (error) {
Logger.debug({err: error, ...params}, 'Failed to enqueue GIF featured-categories refresh');
}
})();
}
async refreshFeaturedCategories(params: {locale: string; country: string}): Promise<void> {
const lockKey = refreshLockKey(this.meta.name, params.locale, params.country);
const token = await this.cacheService.acquireLock(lockKey, REFRESH_LOCK_TTL_SECONDS);
if (!token) {
Logger.debug(params, 'Skipping enriched GIF categories refresh; another worker holds the lock');
return;
}
try {
const rawTags = await this.fetchRawCategoryTags(params);
const enriched = await Promise.all(
rawTags.map(async (tag) => {
try {
const [gif] = await this.search({q: tag.name, locale: params.locale, country: params.country});
if (!gif) return tag;
return {
...tag,
src: gif.src,
proxy_src: gif.proxy_src,
gif,
};
} catch (error) {
Logger.debug({err: error, tag: tag.name, ...params}, 'Failed to enrich GIF category');
return tag;
}
}),
);
await writeEnrichedCategoriesCache(this.cacheService, this.meta.name, params.locale, params.country, enriched);
} catch (error) {
Logger.warn({err: error, ...params}, 'Failed to refresh enriched GIF categories');
throw error;
} finally {
await this.cacheService.releaseLock(lockKey, token).catch(() => undefined);
}
}
private parseKlipyPath(url: string): KlipyPath | null {
try {
const parsed = new URL(url);
const hostname = parsed.hostname.toLowerCase();
if (hostname !== 'klipy.com' && hostname !== 'www.klipy.com') return null;
const match = parsed.pathname.match(/^\/(gif|gifs|clip|clips)\/([^/]+)/i);
if (!match?.[1] || !match[2]) return null;
const type = match[1].toLowerCase().startsWith('clip') ? 'clip' : 'gif';
const slug = decodeURIComponent(match[2]).trim();
return slug ? {type, slug} : null;
} catch {
return null;
}
}
private buildKlipyPageUrl(path: KlipyPath): string {
const basePath = path.type === 'clip' ? 'clips' : 'gifs';
return `https://klipy.com/${basePath}/${encodeURIComponent(path.slug)}`;
}
private toMediaFormat(entry: KlipyFileEntry | undefined): GifMediaFormat | null {
if (!entry?.url) return null;
const width = typeof entry.width === 'number' && entry.width > 0 ? entry.width : 0;
const height = typeof entry.height === 'number' && entry.height > 0 ? entry.height : 0;
if (!width || !height) return null;
return {
src: entry.url,
proxy_src: this.mediaService.getExternalMediaProxyURL(entry.url),
width,
height,
};
}
private collectKlipyMedia(input: KlipyGif): {
media: Record<string, GifMediaFormat>;
preferred: GifMediaFormat | null;
} {
const media: Record<string, GifMediaFormat> = {};
let preferred: GifMediaFormat | null = null;
for (const size of KLIPY_SIZE_PREFERENCE) {
const bucket = input.file?.[size];
if (!bucket) continue;
for (const format of KLIPY_FORMAT_KEYS) {
const entry = this.toMediaFormat(bucket[format]);
if (!entry) continue;
const publicKey = KLIPY_PUBLIC_FORMAT_KEYS[size][format];
media[publicKey] = entry;
if (!preferred) preferred = entry;
}
}
if (Object.keys(media).length === 0 && input.media_formats?.webm) {
const webm = input.media_formats.webm;
const fallback: GifMediaFormat = {
src: webm.url,
proxy_src: this.mediaService.getExternalMediaProxyURL(webm.url),
width: webm.dims[0],
height: webm.dims[1],
};
media.webm = fallback;
preferred = fallback;
}
return {media, preferred};
}
private transformKlipyGif(input: KlipyGif): GifResponse | null {
const parsedPath = this.parseKlipyPath(input.itemurl);
const explicitSlug = input.slug?.trim();
const normalizedSlug = explicitSlug || parsedPath?.slug || input.id;
const normalizedType = parsedPath?.type ?? 'gif';
const normalizedUrl =
parsedPath || explicitSlug ? this.buildKlipyPageUrl({type: normalizedType, slug: normalizedSlug}) : input.itemurl;
const {media, preferred} = this.collectKlipyMedia(input);
const top = media.webm ?? preferred;
if (!top) return null;
return {
id: normalizedSlug,
slug: normalizedSlug,
provider: this.meta.name,
title: input.title,
url: normalizedUrl,
src: top.src,
proxy_src: top.proxy_src,
width: top.width,
height: top.height,
media,
};
}
extractSlugFromUrl(url: string): string | null {
return this.parseKlipyPath(url)?.slug ?? null;
}
buildShareUrl(slug: string): string {
const trimmed = slug.trim();
if (!trimmed) return 'https://klipy.com/gifs';
return this.buildKlipyPageUrl({type: 'gif', slug: trimmed});
}
}
@@ -0,0 +1,24 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {describe, expect, it} from 'vitest';
import {buildKlipyShareUrl, extractKlipySlugFromUrl} from './NatsGifProvider';
describe('NatsGifProvider KLIPY URL helpers', () => {
it('extracts GIF and clip slugs from KLIPY share URLs', () => {
expect(extractKlipySlugFromUrl('https://klipy.com/gifs/goatplaybanjo-chat-4')).toBe('goatplaybanjo-chat-4');
expect(extractKlipySlugFromUrl('https://www.klipy.com/gif/funny-123')).toBe('funny-123');
expect(extractKlipySlugFromUrl('https://klipy.com/clips/clip-123')).toBe('clip-123');
});
it('rejects non-KLIPY and unsupported URLs', () => {
expect(extractKlipySlugFromUrl('https://notklipy.com/gifs/funny-123')).toBeNull();
expect(extractKlipySlugFromUrl('https://klipy.com/search/funny-123')).toBeNull();
expect(extractKlipySlugFromUrl('not a url')).toBeNull();
});
it('builds canonical GIF share URLs locally', () => {
expect(buildKlipyShareUrl('goatplaybanjo-chat-4')).toBe('https://klipy.com/gifs/goatplaybanjo-chat-4');
expect(buildKlipyShareUrl(' ')).toBe('https://klipy.com/gifs');
expect(buildKlipyShareUrl('a slug')).toBe('https://klipy.com/gifs/a%20slug');
});
});
+290
View File
@@ -0,0 +1,290 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';
import {ServiceUnavailableError} from '@fluxer/errors/src/domains/core/ServiceUnavailableError';
import {
type GifCategoryTagResponse,
GifCategoryTagResponse as GifCategoryTagResponseSchema,
type GifResponse,
GifResponse as GifResponseSchema,
} from '@fluxer/schema/src/domains/gif/GifSchemas';
import type {INatsConnectionManager} from '@pkgs/nats/src/INatsConnectionManager';
import {NatsConnectionManager} from '@pkgs/nats/src/NatsConnectionManager';
import {StringCodec} from 'nats';
import {Config} from '../Config';
import {Logger} from '../Logger';
import {isJsonRecord, parseJsonUnknown} from '../utils/JsonBoundaryUtils';
import type {GifProviderMeta, IGifProvider} from './IGifProvider';
const GIF_SERVICE_SUBJECT = process.env.FLUXER_GIF_SERVICE_SUBJECT || 'svc.gifs';
const DEFAULT_GIF_SERVICE_TIMEOUT_MS = 12_000;
const DEFAULT_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS = 3_000;
const GIF_PROVIDER_META: GifProviderMeta = {
name: 'klipy',
displayName: 'KLIPY',
attributionRequired: true,
};
const KLIPY_SHARE_ORIGIN = 'https://klipy.com';
const KLIPY_SHARE_HOSTS = new Set(['klipy.com', 'www.klipy.com']);
type GifApiKeyResolver = () => Promise<string | null>;
type NatsGifRequest =
| {op: 'IsAvailable'; api_key: string | null}
| {op: 'Search'; api_key: string; q: string; locale: string; country: string}
| {op: 'GetFeatured'; api_key: string; locale: string; country: string}
| {op: 'GetTrendingGifs'; api_key: string; locale: string; country: string}
| {op: 'Suggest'; api_key: string; q: string; locale: string}
| {op: 'RegisterShare'; api_key: string; id: string; q: string; locale: string; country: string}
| {op: 'ResolveByUrl'; api_key: string; url: string; locale: string; country: string};
export function extractKlipySlugFromUrl(rawUrl: string): string | null {
let parsed: URL;
try {
parsed = new URL(rawUrl);
} catch {
return null;
}
if (!KLIPY_SHARE_HOSTS.has(parsed.hostname.toLowerCase())) {
return null;
}
const segments = parsed.pathname.split('/').filter(Boolean);
const kind = segments[0]?.toLowerCase();
const slug = segments[1]?.trim();
if (!slug) {
return null;
}
switch (kind) {
case 'gif':
case 'gifs':
case 'clip':
case 'clips':
return slug;
default:
return null;
}
}
export function buildKlipyShareUrl(slug: string): string {
const trimmed = slug.trim();
if (!trimmed) {
return `${KLIPY_SHARE_ORIGIN}/gifs`;
}
return `${KLIPY_SHARE_ORIGIN}/gifs/${encodeURIComponent(trimmed)}`;
}
function readPositiveIntegerEnv(name: string, fallback: number): number {
const value = process.env[name];
if (!value) return fallback;
const parsed = Number(value);
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
}
function readFailedMessage(value: unknown): string | null {
if (!isJsonRecord(value)) return null;
if (typeof value.error === 'string') return value.error;
if (!('Failed' in value)) return null;
const failed = value.Failed;
return isJsonRecord(failed) && typeof failed.message === 'string' ? failed.message : 'GIF service failed';
}
function readVariant(value: unknown, variant: string): unknown {
if (!isJsonRecord(value) || !(variant in value)) {
throw new ServiceUnavailableError({message: `GIF service returned an unexpected ${variant} response`});
}
return value[variant];
}
function readGifList(value: unknown, variant: string): Array<GifResponse> {
const parsed = GifResponseSchema.array().safeParse(readVariant(value, variant));
if (!parsed.success) {
throw new ServiceUnavailableError({message: `GIF service returned invalid ${variant} data`});
}
return parsed.data;
}
function readFeatured(value: unknown): {
gifs: Array<GifResponse>;
categories: Array<GifCategoryTagResponse>;
} {
const featured = readVariant(value, 'Featured');
if (!isJsonRecord(featured)) {
throw new ServiceUnavailableError({message: 'GIF service returned invalid featured data'});
}
const gifs = GifResponseSchema.array().safeParse(featured.gifs);
const categories = GifCategoryTagResponseSchema.array().safeParse(featured.categories);
if (!gifs.success || !categories.success) {
throw new ServiceUnavailableError({message: 'GIF service returned invalid featured data'});
}
return {gifs: gifs.data, categories: categories.data};
}
function readSuggestions(value: unknown): Array<string> {
const suggestions = readVariant(value, 'Suggestions');
if (!Array.isArray(suggestions) || !suggestions.every((suggestion) => typeof suggestion === 'string')) {
throw new ServiceUnavailableError({message: 'GIF service returned invalid suggestions'});
}
return suggestions;
}
function readResolved(value: unknown): GifResponse | null {
const resolved = readVariant(value, 'Resolved');
if (!isJsonRecord(resolved)) {
throw new ServiceUnavailableError({message: 'GIF service returned invalid resolved data'});
}
if (resolved.gif === null || resolved.gif === undefined) return null;
const parsed = GifResponseSchema.safeParse(resolved.gif);
if (!parsed.success) {
throw new ServiceUnavailableError({message: 'GIF service returned invalid resolved GIF'});
}
return parsed.data;
}
class NatsGifProvider implements IGifProvider {
readonly meta = GIF_PROVIDER_META;
private readonly codec = StringCodec();
constructor(
private readonly connectionManager: INatsConnectionManager,
private readonly apiKeyResolver: GifApiKeyResolver,
private readonly requestTimeoutMs = readPositiveIntegerEnv(
'FLUXER_GIF_SERVICE_TIMEOUT_MS',
DEFAULT_GIF_SERVICE_TIMEOUT_MS,
),
private readonly registerShareTimeoutMs = readPositiveIntegerEnv(
'FLUXER_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS',
DEFAULT_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS,
),
private readonly subject = GIF_SERVICE_SUBJECT,
) {}
async isAvailable(): Promise<boolean> {
return Boolean((await this.apiKeyResolver())?.trim());
}
async search(params: {q: string; locale: string; country: string}): Promise<Array<GifResponse>> {
const response = await this.request({
op: 'Search',
api_key: await this.getApiKey(),
q: params.q,
locale: params.locale,
country: params.country,
});
return readGifList(response, 'SearchResults');
}
async registerShare(params: {id: string; q: string; locale: string; country: string}): Promise<void> {
const response = await this.request(
{
op: 'RegisterShare',
api_key: await this.getApiKey(),
id: params.id,
q: params.q,
locale: params.locale,
country: params.country,
},
this.registerShareTimeoutMs,
);
if (response !== 'Registered') {
throw new ServiceUnavailableError({message: 'GIF service returned an unexpected register-share response'});
}
}
async getFeatured(params: {locale: string; country: string}): Promise<{
gifs: Array<GifResponse>;
categories: Array<GifCategoryTagResponse>;
}> {
return readFeatured(
await this.request({
op: 'GetFeatured',
api_key: await this.getApiKey(),
locale: params.locale,
country: params.country,
}),
);
}
async getTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
const response = await this.request({
op: 'GetTrendingGifs',
api_key: await this.getApiKey(),
locale: params.locale,
country: params.country,
});
return readGifList(response, 'TrendingResults');
}
async suggest(params: {q: string; locale: string}): Promise<Array<string>> {
return readSuggestions(
await this.request({
op: 'Suggest',
api_key: await this.getApiKey(),
q: params.q,
locale: params.locale,
}),
);
}
async resolveByUrl(params: {url: string; locale: string; country: string}): Promise<GifResponse | null> {
return readResolved(
await this.request({
op: 'ResolveByUrl',
api_key: await this.getApiKey(),
url: params.url,
locale: params.locale,
country: params.country,
}),
);
}
buildShareUrl(slug: string): string {
return buildKlipyShareUrl(slug);
}
extractSlugFromUrl(url: string): string | null {
return extractKlipySlugFromUrl(url);
}
private async getApiKey(): Promise<string> {
const apiKey = (await this.apiKeyResolver())?.trim();
if (!apiKey) {
throw new FeatureTemporarilyDisabledError();
}
return apiKey;
}
private async request(payload: NatsGifRequest, timeout = this.requestTimeoutMs): Promise<unknown> {
try {
if (this.connectionManager.isClosed()) {
await this.connectionManager.connect();
}
const connection = this.connectionManager.getConnection();
const response = await connection.request(this.subject, this.codec.encode(JSON.stringify(payload)), {timeout});
const decoded = this.codec.decode(response.data);
const parsed = parseJsonUnknown(decoded);
const failedMessage = readFailedMessage(parsed);
if (failedMessage) {
throw new ServiceUnavailableError({message: failedMessage});
}
return parsed;
} catch (error) {
Logger.warn({error, op: payload.op}, '[gif-service] request failed');
if (error instanceof FeatureTemporarilyDisabledError || error instanceof ServiceUnavailableError) {
throw error;
}
throw new ServiceUnavailableError({message: 'GIF service is temporarily unavailable'});
}
}
}
export function createNatsGifProvider(apiKeyResolver: GifApiKeyResolver): NatsGifProvider {
const manager = new NatsConnectionManager({
url: Config.nats.coreUrl,
token: Config.nats.authToken || undefined,
name: process.env.FLUXER_GIF_SERVICE_NATS_CLIENT_NAME || 'fluxer-api-gifs',
});
void manager.connect().catch((error) => {
Logger.warn({error}, '[gif-service] Failed to establish NATS connection');
});
return new NatsGifProvider(manager, apiKeyResolver);
}
-523
View File
@@ -1,523 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {FLUXER_USER_AGENT} from '@fluxer/constants/src/Core';
import type {GifCategoryTagResponse, GifMediaFormat, GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
import type {ICacheService} from '@pkgs/cache/src/ICacheService';
import {ms} from 'itty-time';
import {Config} from '../Config';
import type {IMediaService} from '../infrastructure/IMediaService';
import {Logger} from '../Logger';
import {getWorkerService} from '../middleware/ServiceRegistry';
import {EXTERNAL_RESPONSE_LIMITS} from '../utils/ExternalResponseLimits';
import * as FetchUtils from '../utils/FetchUtils';
import {isJsonRecord, parseJsonUnknown} from '../utils/JsonBoundaryUtils';
import {
REFRESH_LOCK_TTL_SECONDS,
readEnrichedCategoriesCache,
refreshLockKey,
trackSeenLocale,
writeEnrichedCategoriesCache,
} from './GifFeaturedCategoriesCache';
import type {GifProviderMeta, IGifProvider} from './IGifProvider';
const TENOR_BASE_URL = 'https://tenor.googleapis.com/v2';
const DEFAULT_MEDIA_FILTER = 'webm,mp4,webp,gif,tinywebm,tinymp4,tinygif,nanogif';
const PUBLIC_MEDIA_FORMATS = ['webm', 'mp4', 'webp', 'gif', 'tinywebm', 'tinymp4', 'tinygif', 'nanogif'] as const;
const DEFAULT_CONTENT_FILTER = 'low';
const CLIENT_KEY = 'fluxer';
const MAX_RETRIES = 3;
const BACKOFF_BASE_DELAY = ms('1 second');
const CACHE_EXPIRATION_TIME = ms('5 minutes');
const TENOR_PROVIDER_META: GifProviderMeta = {
name: 'tenor',
displayName: 'Tenor',
attributionRequired: false,
};
type GifApiKeyResolver = () => Promise<string | null>;
interface TenorMediaFormat {
url: string;
dims: [number, number];
}
interface TenorGif {
id: string;
title?: string;
content_description?: string;
media_formats?: Record<string, TenorMediaFormat>;
itemurl?: string;
url?: string;
}
interface TenorCategoryTag {
searchterm: string;
image: string;
}
function isTenorMediaFormat(value: unknown): value is TenorMediaFormat {
if (!isJsonRecord(value) || typeof value.url !== 'string' || !Array.isArray(value.dims)) return false;
return value.dims.length === 2 && value.dims.every((dimension) => typeof dimension === 'number');
}
function isTenorGif(value: unknown): value is TenorGif {
if (!isJsonRecord(value) || typeof value.id !== 'string') return false;
const mediaFormats = value.media_formats;
return (
(value.title === undefined || typeof value.title === 'string') &&
(value.content_description === undefined || typeof value.content_description === 'string') &&
(mediaFormats === undefined ||
(isJsonRecord(mediaFormats) && Object.values(mediaFormats).every(isTenorMediaFormat))) &&
(value.itemurl === undefined || typeof value.itemurl === 'string') &&
(value.url === undefined || typeof value.url === 'string')
);
}
function isTenorCategoryTag(value: unknown): value is TenorCategoryTag {
return isJsonRecord(value) && typeof value.searchterm === 'string' && typeof value.image === 'string';
}
function readResultsArray(value: unknown): Array<unknown> {
if (!isJsonRecord(value) || !Array.isArray(value.results)) {
throw new Error('Tenor API response did not include a results array');
}
return value.results;
}
function readTagsArray(value: unknown): Array<unknown> {
if (!isJsonRecord(value) || !Array.isArray(value.tags)) {
throw new Error('Tenor API response did not include a tags array');
}
return value.tags;
}
type CacheEntry<T> = {
data: T;
timestamp: number;
};
export class TenorGifProvider implements IGifProvider {
readonly meta = TENOR_PROVIDER_META;
private refreshingKeys: Map<string, boolean> = new Map();
private featuredCacheKey(locale: string, country: string): string {
return `tenor:featured:${locale}:${country}`;
}
private trendingCacheKey(locale: string, country: string): string {
return `tenor:trending:${locale}:${country}`;
}
constructor(
private cacheService: ICacheService,
private mediaService: IMediaService,
private apiKeyResolver: GifApiKeyResolver = async () => Config.tenor.apiKey || null,
) {}
async isAvailable(): Promise<boolean> {
return Boolean(await this.apiKeyResolver());
}
private async getApiKey(): Promise<string> {
const apiKey = await this.apiKeyResolver();
if (!apiKey) {
throw new Error('Tenor API key is not configured');
}
return apiKey;
}
private createURL({endpoint, params}: {endpoint: string; params: Record<string, string | number | undefined>}): URL {
const url = new URL(`${TENOR_BASE_URL}/${endpoint}`);
for (const [key, value] of Object.entries(params)) {
if (value !== undefined && value !== '') {
url.searchParams.append(key, value.toString());
}
}
return url;
}
private async fetchTenorData(url: URL): Promise<unknown> {
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
try {
const response = await fetch(url.toString(), {
headers: {'User-Agent': FLUXER_USER_AGENT},
signal: AbortSignal.timeout(ms('30 seconds')),
});
if (!response.ok) {
throw new Error(`Failed to fetch Tenor data: ${response.statusText}`);
}
const responseText = await FetchUtils.streamToStringWithLimit(response.body, {
maxBytes: EXTERNAL_RESPONSE_LIMITS.tenorApiBytes,
headers: response.headers,
url: response.url,
description: 'Tenor API response',
});
return parseJsonUnknown(responseText);
} catch (error) {
if (attempt < MAX_RETRIES - 1) {
const delay = BACKOFF_BASE_DELAY * 2 ** attempt;
await new Promise((resolve) => setTimeout(resolve, delay));
} else {
throw error;
}
}
}
throw new Error('Exceeded maximum retries');
}
private async fetchAndTransformGifs(url: URL): Promise<Array<GifResponse>> {
const results = readResultsArray(await this.fetchTenorData(url)).filter(isTenorGif);
return results.map((gif) => this.transformTenorGif(gif)).filter((gif): gif is GifResponse => gif !== null);
}
private async getCache<T>(key: string): Promise<{
data: T;
isStale: boolean;
} | null> {
const cached = await this.cacheService.get<CacheEntry<T>>(key);
if (!cached) return null;
const age = Date.now() - cached.timestamp;
const isStale = age > CACHE_EXPIRATION_TIME;
return {data: cached.data, isStale};
}
private async setCache<T>(key: string, data: T): Promise<void> {
const cacheEntry: CacheEntry<T> = {
data,
timestamp: Date.now(),
};
await this.cacheService.set(key, cacheEntry);
}
private triggerBackgroundRefresh<T>(key: string, refreshFn: () => Promise<T>): void {
if (this.refreshingKeys.get(key)) {
return;
}
this.refreshingKeys.set(key, true);
setImmediate(async () => {
try {
const freshData = await refreshFn();
await this.setCache(key, freshData);
} catch (error) {
Logger.debug({key, error}, `Background refresh failed for key ${key}`);
} finally {
this.refreshingKeys.delete(key);
}
});
}
async search(params: {q: string; locale: string; country: string}): Promise<Array<GifResponse>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'search',
params: {
key: apiKey,
client_key: CLIENT_KEY,
q: params.q,
country: params.country,
locale: params.locale,
contentfilter: DEFAULT_CONTENT_FILTER,
media_filter: DEFAULT_MEDIA_FILTER,
limit: 50,
},
});
return this.fetchAndTransformGifs(url);
}
async registerShare(params: {id: string; q: string; locale: string; country: string}): Promise<void> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'registershare',
params: {
key: apiKey,
client_key: CLIENT_KEY,
id: params.id,
country: params.country,
locale: params.locale,
q: params.q,
},
});
await fetch(url.toString(), {
headers: {'User-Agent': FLUXER_USER_AGENT},
signal: AbortSignal.timeout(ms('30 seconds')),
});
}
async getFeatured(params: {locale: string; country: string}): Promise<{
gifs: Array<GifResponse>;
categories: Array<GifCategoryTagResponse>;
}> {
const cacheKey = this.featuredCacheKey(params.locale, params.country);
const cached = await this.getCache<{
gifs: Array<GifResponse>;
categories: Array<GifCategoryTagResponse>;
}>(cacheKey);
if (cached) {
if (cached.isStale) {
this.triggerBackgroundRefresh(cacheKey, () => this.fetchFeaturedData(params));
}
return cached.data;
}
const data = await this.fetchFeaturedData(params);
await this.setCache(cacheKey, data);
return data;
}
private async fetchFeaturedData(params: {locale: string; country: string}): Promise<{
gifs: Array<GifResponse>;
categories: Array<GifCategoryTagResponse>;
}> {
const [gifs, categories] = await Promise.all([this.getFeaturedGifs(params), this.getFeaturedCategories(params)]);
return {gifs, categories};
}
async getTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
const cacheKey = this.trendingCacheKey(params.locale, params.country);
const cached = await this.getCache<Array<GifResponse>>(cacheKey);
if (cached) {
if (cached.isStale) {
this.triggerBackgroundRefresh(cacheKey, () => this.fetchTrendingGifs(params));
}
return cached.data;
}
const gifs = await this.fetchTrendingGifs(params);
await this.setCache(cacheKey, gifs);
return gifs;
}
private async fetchTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'featured',
params: {
key: apiKey,
client_key: CLIENT_KEY,
country: params.country,
locale: params.locale,
contentfilter: DEFAULT_CONTENT_FILTER,
media_filter: DEFAULT_MEDIA_FILTER,
limit: 50,
},
});
return this.fetchAndTransformGifs(url);
}
async suggest(params: {q: string; locale: string}): Promise<Array<string>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'search_suggestions',
params: {
key: apiKey,
client_key: CLIENT_KEY,
q: params.q,
locale: params.locale,
limit: 20,
},
});
return readResultsArray(await this.fetchTenorData(url)).filter(
(result): result is string => typeof result === 'string',
);
}
async resolveByUrl(params: {url: string; locale: string; country: string}): Promise<GifResponse | null> {
const slug = this.extractSlugFromUrl(params.url);
if (!slug) return null;
const id = this.extractIdFromSlug(slug);
if (!id) return null;
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'posts',
params: {
key: apiKey,
client_key: CLIENT_KEY,
ids: id,
country: params.country,
locale: params.locale,
media_filter: DEFAULT_MEDIA_FILTER,
},
});
const [gif] = await this.fetchAndTransformGifs(url);
return gif ?? null;
}
private async getFeaturedGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'featured',
params: {
key: apiKey,
client_key: CLIENT_KEY,
country: params.country,
locale: params.locale,
contentfilter: DEFAULT_CONTENT_FILTER,
media_filter: DEFAULT_MEDIA_FILTER,
limit: 1,
},
});
return this.fetchAndTransformGifs(url);
}
private async getFeaturedCategories(params: {
locale: string;
country: string;
}): Promise<Array<GifCategoryTagResponse>> {
trackSeenLocale(this.cacheService, this.meta.name, params.locale, params.country).catch((error) => {
Logger.debug({err: error, ...params}, 'Failed to track seen GIF locale');
});
const rawTags = await this.fetchRawCategoryTags(params);
const cached = await readEnrichedCategoriesCache(this.cacheService, this.meta.name, params.locale, params.country);
if (cached) {
if (cached.isStale) {
this.scheduleEnrichmentRefresh(params);
}
const byName = new Map(cached.data.map((entry) => [entry.name, entry]));
return rawTags.map((tag) => byName.get(tag.name) ?? tag);
}
this.scheduleEnrichmentRefresh(params);
return rawTags;
}
private async fetchRawCategoryTags(params: {
locale: string;
country: string;
}): Promise<Array<GifCategoryTagResponse>> {
const apiKey = await this.getApiKey();
const url = this.createURL({
endpoint: 'categories',
params: {
key: apiKey,
client_key: CLIENT_KEY,
country: params.country,
locale: params.locale,
contentfilter: DEFAULT_CONTENT_FILTER,
type: 'featured',
},
});
const tags = readTagsArray(await this.fetchTenorData(url)).filter(isTenorCategoryTag);
return tags
.filter((tag) => Boolean(tag.searchterm) && Boolean(tag.image))
.map((tag) => ({
name: tag.searchterm,
src: tag.image,
proxy_src: this.mediaService.getExternalMediaProxyURL(tag.image),
gif: null,
}));
}
private scheduleEnrichmentRefresh(params: {locale: string; country: string}): void {
(async () => {
try {
const workerService = getWorkerService();
await workerService.addJob('refreshGifFeaturedCategories', {
provider: this.meta.name,
locale: params.locale,
country: params.country,
});
} catch (error) {
Logger.debug({err: error, ...params}, 'Failed to enqueue GIF featured-categories refresh');
}
})();
}
async refreshFeaturedCategories(params: {locale: string; country: string}): Promise<void> {
const lockKey = refreshLockKey(this.meta.name, params.locale, params.country);
const token = await this.cacheService.acquireLock(lockKey, REFRESH_LOCK_TTL_SECONDS);
if (!token) {
Logger.debug(params, 'Skipping enriched GIF categories refresh; another worker holds the lock');
return;
}
try {
const rawTags = await this.fetchRawCategoryTags(params);
const enriched = await Promise.all(
rawTags.map(async (tag) => {
try {
const [gif] = await this.search({q: tag.name, locale: params.locale, country: params.country});
return {...tag, gif: gif ?? null};
} catch (error) {
Logger.debug({err: error, tag: tag.name, ...params}, 'Failed to enrich GIF category');
return tag;
}
}),
);
await writeEnrichedCategoriesCache(this.cacheService, this.meta.name, params.locale, params.country, enriched);
} catch (error) {
Logger.warn({err: error, ...params}, 'Failed to refresh enriched GIF categories');
throw error;
} finally {
await this.cacheService.releaseLock(lockKey, token).catch(() => undefined);
}
}
private selectMediaFormat(mediaFormats: Record<string, TenorMediaFormat> | undefined): TenorMediaFormat | null {
if (!mediaFormats) return null;
const preferredKeys = ['webm', 'mp4', 'tinywebm', 'tinymp4', 'webp', 'gif', 'tinygif', 'nanogif'];
for (const key of preferredKeys) {
const candidate = mediaFormats[key];
if (candidate?.url && candidate.dims?.length === 2) return candidate;
}
for (const candidate of Object.values(mediaFormats)) {
if (candidate?.url && candidate.dims?.length === 2) return candidate;
}
return null;
}
private transformTenorGif(input: TenorGif): GifResponse | null {
const best = this.selectMediaFormat(input.media_formats);
if (!best) return null;
const title = input.title?.trim() || input.content_description?.trim() || '';
const fallbackUrl = `https://tenor.com/view/${encodeURIComponent(input.id)}`;
const url = input.itemurl?.trim() || input.url?.trim() || fallbackUrl;
const slug = this.extractSlugFromUrl(url) ?? `view/${input.id}`;
const media: Record<string, GifMediaFormat> = {};
for (const key of PUBLIC_MEDIA_FORMATS) {
const candidate = input.media_formats?.[key];
if (candidate?.url && candidate.dims?.length === 2) {
media[key] = {
src: candidate.url,
proxy_src: this.mediaService.getExternalMediaProxyURL(candidate.url),
width: candidate.dims[0],
height: candidate.dims[1],
};
}
}
return {
id: input.id,
slug,
provider: this.meta.name,
title,
url,
src: best.url,
proxy_src: this.mediaService.getExternalMediaProxyURL(best.url),
width: best.dims[0],
height: best.dims[1],
media,
};
}
extractSlugFromUrl(url: string): string | null {
try {
const parsed = new URL(url);
const hostname = parsed.hostname.toLowerCase();
if (hostname !== 'tenor.com' && hostname !== 'www.tenor.com') return null;
const match = parsed.pathname.match(/^\/(?:[a-z]{2}\/)?view\/([^/]+)/i);
if (!match?.[1]) return null;
const slug = decodeURIComponent(match[1]).trim();
return slug ? `view/${slug}` : null;
} catch {
return null;
}
}
private extractIdFromSlug(slug: string): string | null {
const normalized = slug.trim().replace(/^view\//i, '');
if (!normalized) return null;
const lastDashIndex = normalized.lastIndexOf('-');
const candidate = lastDashIndex === -1 ? normalized : normalized.slice(lastDashIndex + 1);
return candidate.trim() || null;
}
buildShareUrl(slug: string): string {
const trimmed = slug.trim().replace(/^\/+|\/+$/g, '');
const normalized = trimmed.toLowerCase().startsWith('view/') ? trimmed : `view/${trimmed}`;
return `https://tenor.com/${normalized}`;
}
}
@@ -70,6 +70,7 @@ describe('NatsUnfurlerService', () => {
bypass_cache: true, bypass_cache: true,
cache_only: false, cache_only: false,
youtube_api_key: null, youtube_api_key: null,
klipy_api_key: null,
}, },
timeout: 12000, timeout: 12000,
}, },
@@ -19,6 +19,7 @@ interface NatsUnfurlRequest {
bypass_cache: boolean; bypass_cache: boolean;
cache_only: boolean; cache_only: boolean;
youtube_api_key: string | null; youtube_api_key: string | null;
klipy_api_key: string | null;
} }
interface NatsUnfurlInnerResult { interface NatsUnfurlInnerResult {
@@ -59,6 +60,7 @@ export class NatsUnfurlerService extends IUnfurlerService {
constructor( constructor(
connectionManager: INatsConnectionManager, connectionManager: INatsConnectionManager,
private readonly resolveYoutubeApiKey: (() => Promise<string | null>) | null = null, private readonly resolveYoutubeApiKey: (() => Promise<string | null>) | null = null,
private readonly resolveKlipyApiKey: (() => Promise<string | null>) | null = null,
) { ) {
super(); super();
this.connectionManager = connectionManager; this.connectionManager = connectionManager;
@@ -77,6 +79,7 @@ export class NatsUnfurlerService extends IUnfurlerService {
bypass_cache: options.bypassCache === true, bypass_cache: options.bypassCache === true,
cache_only: options.cacheOnly === true, cache_only: options.cacheOnly === true,
youtube_api_key: this.resolveYoutubeApiKey ? await this.resolveYoutubeApiKey() : null, youtube_api_key: this.resolveYoutubeApiKey ? await this.resolveYoutubeApiKey() : null,
klipy_api_key: this.resolveKlipyApiKey ? await this.resolveKlipyApiKey() : null,
}; };
if (this.connectionManager.isClosed()) { if (this.connectionManager.isClosed()) {
await this.connectionManager.connect(); await this.connectionManager.connect();
@@ -99,13 +99,10 @@ interface InstanceServicesPublicConfig {
bluesky_enabled: boolean; bluesky_enabled: boolean;
} }
export type InstanceGifProvider = 'tenor' | 'klipy';
export type InstanceCaptchaProvider = 'hcaptcha' | 'turnstile' | 'none'; export type InstanceCaptchaProvider = 'hcaptcha' | 'turnstile' | 'none';
type InstanceEmailProvider = 'smtp' | 'none'; type InstanceEmailProvider = 'smtp' | 'none';
interface InstanceGifIntegrationConfig { interface InstanceGifIntegrationConfig {
provider: InstanceGifProvider | null;
tenor_api_key: string | null;
klipy_api_key: string | null; klipy_api_key: string | null;
} }
@@ -160,9 +157,7 @@ interface InstanceIntegrationsConfig {
bluesky: InstanceBlueskyIntegrationConfig; bluesky: InstanceBlueskyIntegrationConfig;
} }
export interface InstanceGifEffectiveConfig { interface InstanceGifEffectiveConfig {
provider: InstanceGifProvider;
tenor_api_key: string | null;
klipy_api_key: string | null; klipy_api_key: string | null;
active_api_key: string | null; active_api_key: string | null;
available: boolean; available: boolean;
@@ -179,9 +174,6 @@ export interface InstanceCaptchaEffectiveConfig {
interface InstanceIntegrationsAdminConfig { interface InstanceIntegrationsAdminConfig {
gif: { gif: {
provider: InstanceGifProvider | null;
effective_provider: InstanceGifProvider;
tenor_api_key_set: boolean;
klipy_api_key_set: boolean; klipy_api_key_set: boolean;
effective_available: boolean; effective_available: boolean;
}; };
@@ -438,8 +430,6 @@ function normalizeInstancePolicyConfig(value: unknown): InstancePolicyConfig {
const DEFAULT_INSTANCE_INTEGRATIONS_CONFIG: InstanceIntegrationsConfig = { const DEFAULT_INSTANCE_INTEGRATIONS_CONFIG: InstanceIntegrationsConfig = {
gif: { gif: {
provider: null,
tenor_api_key: null,
klipy_api_key: null, klipy_api_key: null,
}, },
youtube: { youtube: {
@@ -492,10 +482,6 @@ const DEFAULT_INSTANCE_MEDIA_CONFIG: InstanceMediaConfig = {
attachment_decay: DEFAULT_INSTANCE_ATTACHMENT_DECAY_CONFIG, attachment_decay: DEFAULT_INSTANCE_ATTACHMENT_DECAY_CONFIG,
}; };
function isGifProvider(value: unknown): value is InstanceGifProvider {
return value === 'tenor' || value === 'klipy';
}
function isCaptchaProvider(value: unknown): value is InstanceCaptchaProvider { function isCaptchaProvider(value: unknown): value is InstanceCaptchaProvider {
return value === 'hcaptcha' || value === 'turnstile' || value === 'none'; return value === 'hcaptcha' || value === 'turnstile' || value === 'none';
} }
@@ -561,8 +547,6 @@ function normalizeInstanceIntegrationsConfig(value: unknown): InstanceIntegratio
: defaults.bluesky.keys; : defaults.bluesky.keys;
return { return {
gif: { gif: {
provider: isGifProvider(gif.provider) ? gif.provider : defaults.gif.provider,
tenor_api_key: normalizeSecretString(gif.tenor_api_key),
klipy_api_key: normalizeSecretString(gif.klipy_api_key), klipy_api_key: normalizeSecretString(gif.klipy_api_key),
}, },
youtube: { youtube: {
@@ -1214,16 +1198,11 @@ export class InstanceConfigRepository {
async getEffectiveGifConfig(): Promise<InstanceGifEffectiveConfig> { async getEffectiveGifConfig(): Promise<InstanceGifEffectiveConfig> {
const integrations = await this.getInstanceIntegrationsConfig(); const integrations = await this.getInstanceIntegrationsConfig();
const provider = integrations.gif.provider ?? Config.gif.provider;
const tenorApiKey = integrations.gif.tenor_api_key ?? normalizeSecretString(Config.tenor.apiKey);
const klipyApiKey = integrations.gif.klipy_api_key ?? normalizeSecretString(Config.klipy.apiKey); const klipyApiKey = integrations.gif.klipy_api_key ?? normalizeSecretString(Config.klipy.apiKey);
const activeApiKey = provider === 'tenor' ? tenorApiKey : klipyApiKey;
return { return {
provider,
tenor_api_key: tenorApiKey,
klipy_api_key: klipyApiKey, klipy_api_key: klipyApiKey,
active_api_key: activeApiKey, active_api_key: klipyApiKey,
available: Boolean(activeApiKey), available: Boolean(klipyApiKey),
}; };
} }
@@ -1323,9 +1302,6 @@ export class InstanceConfigRepository {
]); ]);
return { return {
gif: { gif: {
provider: integrations.gif.provider,
effective_provider: gif.provider,
tenor_api_key_set: secretIsSet(integrations.gif.tenor_api_key) || secretIsSet(Config.tenor.apiKey),
klipy_api_key_set: secretIsSet(integrations.gif.klipy_api_key) || secretIsSet(Config.klipy.apiKey), klipy_api_key_set: secretIsSet(integrations.gif.klipy_api_key) || secretIsSet(Config.klipy.apiKey),
effective_available: gif.available, effective_available: gif.available,
}, },
@@ -12,7 +12,7 @@ import {RateLimitMiddleware} from '../middleware/RateLimitMiddleware';
import {OpenAPI} from '../middleware/ResponseTypeMiddleware'; import {OpenAPI} from '../middleware/ResponseTypeMiddleware';
import {RateLimitConfigs} from '../RateLimitConfig'; import {RateLimitConfigs} from '../RateLimitConfig';
import type {HonoEnv} from '../types/HonoEnv'; import type {HonoEnv} from '../types/HonoEnv';
import type {InstanceCaptchaEffectiveConfig, InstanceGifEffectiveConfig} from './InstanceConfigRepository'; import type {InstanceCaptchaEffectiveConfig} from './InstanceConfigRepository';
function buildDiscoveryStaticInput( function buildDiscoveryStaticInput(
gifService: GifService | undefined, gifService: GifService | undefined,
@@ -20,12 +20,11 @@ function buildDiscoveryStaticInput(
runtime: { runtime: {
captcha: InstanceCaptchaEffectiveConfig; captcha: InstanceCaptchaEffectiveConfig;
emailEnabled: boolean; emailEnabled: boolean;
gif: InstanceGifEffectiveConfig;
}, },
): DiscoveryStaticInput { ): DiscoveryStaticInput {
const apiClientEndpoint = Config.endpoints.apiClient; const apiClientEndpoint = Config.endpoints.apiClient;
const apiPublicEndpoint = Config.endpoints.apiPublic; const apiPublicEndpoint = Config.endpoints.apiPublic;
const activeGif = gifService?.getByName(runtime.gif.provider); const gifProvider = gifService?.getProvider();
return { return {
apiCodeVersion: API_CODE_VERSION, apiCodeVersion: API_CODE_VERSION,
endpoints: { endpoints: {
@@ -54,9 +53,9 @@ function buildDiscoveryStaticInput(
emails_enabled: runtime.emailEnabled, emails_enabled: runtime.emailEnabled,
}, },
gif: { gif: {
provider: runtime.gif.provider, provider: gifProvider?.meta.name ?? 'klipy',
display_name: activeGif?.meta.displayName ?? runtime.gif.provider, display_name: gifProvider?.meta.displayName ?? 'KLIPY',
attribution_required: activeGif?.meta.attributionRequired ?? false, attribution_required: gifProvider?.meta.attributionRequired ?? true,
}, },
push: { push: {
public_vapid_key: Config.push.publicVapidKey ?? null, public_vapid_key: Config.push.publicVapidKey ?? null,
@@ -86,14 +85,13 @@ export function InstanceController(app: Hono<HonoEnv>) {
const limits = limitConfigService?.getConfigWireFormat(); const limits = limitConfigService?.getConfigWireFormat();
const sso = await ctx.get('ssoService').getPublicStatus(); const sso = await ctx.get('ssoService').getPublicStatus();
const instanceConfigRepository = ctx.get('instanceConfigRepository'); const instanceConfigRepository = ctx.get('instanceConfigRepository');
const [registration, community, services, appPublicConfig, captcha, email, gif] = await Promise.all([ const [registration, community, services, appPublicConfig, captcha, email] = await Promise.all([
instanceConfigRepository.getRegistrationPublicConfig(), instanceConfigRepository.getRegistrationPublicConfig(),
instanceConfigRepository.getInstanceCommunityPublicConfig(), instanceConfigRepository.getInstanceCommunityPublicConfig(),
instanceConfigRepository.getResolvedServicesConfig(), instanceConfigRepository.getResolvedServicesConfig(),
instanceConfigRepository.getAppPublicConfig(), instanceConfigRepository.getAppPublicConfig(),
instanceConfigRepository.getEffectiveCaptchaConfig(), instanceConfigRepository.getEffectiveCaptchaConfig(),
instanceConfigRepository.getEffectiveEmailConfig(), instanceConfigRepository.getEffectiveEmailConfig(),
instanceConfigRepository.getEffectiveGifConfig(),
]); ]);
if (!limits) { if (!limits) {
throw new Error('limit_config_service is not bound'); throw new Error('limit_config_service is not bound');
@@ -111,7 +109,6 @@ export function InstanceController(app: Hono<HonoEnv>) {
{ {
captcha, captcha,
emailEnabled: email.enabled, emailEnabled: email.enabled,
gif,
}, },
), ),
{ {
@@ -2,6 +2,7 @@
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes'; import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
import {ForbiddenError} from '@fluxer/errors/src/domains/core/ForbiddenError'; import {ForbiddenError} from '@fluxer/errors/src/domains/core/ForbiddenError';
import {resolveClientIpHeaderName} from '@fluxer/ip_utils/src/ClientIp';
import {createMiddleware} from 'hono/factory'; import {createMiddleware} from 'hono/factory';
import {Config} from '../Config'; import {Config} from '../Config';
import {Logger} from '../Logger'; import {Logger} from '../Logger';
@@ -23,7 +24,7 @@ const defaultExemptPaths: Array<string> = [
export function RequireClientIpMiddleware({ export function RequireClientIpMiddleware({
exemptPaths = defaultExemptPaths, exemptPaths = defaultExemptPaths,
requiredHeaders = ['x-forwarded-for'], requiredHeaders = [resolveClientIpHeaderName(Config.proxy.client_ip_header)],
}: RequireClientIpOptions = {}) { }: RequireClientIpOptions = {}) {
return createMiddleware<HonoEnv>(async (ctx, next) => { return createMiddleware<HonoEnv>(async (ctx, next) => {
if (Config.dev.testModeEnabled) { if (Config.dev.testModeEnabled) {
@@ -82,6 +82,7 @@ import {UserChannelRequestService} from '../user/services/UserChannelRequestServ
import {UserContentRequestService} from '../user/services/UserContentRequestService'; import {UserContentRequestService} from '../user/services/UserContentRequestService';
import {UserRelationshipRequestService} from '../user/services/UserRelationshipRequestService'; import {UserRelationshipRequestService} from '../user/services/UserRelationshipRequestService';
import {UserService} from '../user/services/UserService'; import {UserService} from '../user/services/UserService';
import {resolveRequestClientIp} from '../utils/IpUtils';
import {VoicePresenceHeartbeatStore} from '../voice/VoicePresenceHeartbeatStore'; import {VoicePresenceHeartbeatStore} from '../voice/VoicePresenceHeartbeatStore';
import {VoiceService} from '../voice/VoiceService'; import {VoiceService} from '../voice/VoiceService';
import {WebhookRequestService} from '../webhook/WebhookRequestService'; import {WebhookRequestService} from '../webhook/WebhookRequestService';
@@ -364,7 +365,7 @@ function getLiveKitWebhookService(): LiveKitWebhookService | null {
export const ServiceMiddleware = createMiddleware<HonoEnv>(async (ctx, next) => { export const ServiceMiddleware = createMiddleware<HonoEnv>(async (ctx, next) => {
const apiContext = createApiContext({ const apiContext = createApiContext({
requestId: ctx.get('requestId') ?? crypto.randomUUID(), requestId: ctx.get('requestId') ?? crypto.randomUUID(),
clientIp: ctx.req.header('x-forwarded-for') ?? null, clientIp: resolveRequestClientIp(ctx.req.raw),
userAgent: ctx.req.header('user-agent') ?? null, userAgent: ctx.req.header('user-agent') ?? null,
}); });
ctx.set('apiContext', apiContext); ctx.set('apiContext', apiContext);
@@ -38,8 +38,7 @@ import {DownloadService} from '../download/DownloadService';
import {createEmailProvider} from '../email/EmailProviderFactory'; import {createEmailProvider} from '../email/EmailProviderFactory';
import {FavoriteMemeRepository} from '../favorite_meme/FavoriteMemeRepository'; import {FavoriteMemeRepository} from '../favorite_meme/FavoriteMemeRepository';
import {GifService} from '../gif/GifService'; import {GifService} from '../gif/GifService';
import {KlipyGifProvider} from '../gif/KlipyGifProvider'; import {createNatsGifProvider} from '../gif/NatsGifProvider';
import {TenorGifProvider} from '../gif/TenorGifProvider';
import {GuildAuditLogService} from '../guild/GuildAuditLogService'; import {GuildAuditLogService} from '../guild/GuildAuditLogService';
import {GuildDiscoveryRepository} from '../guild/repositories/GuildDiscoveryRepository'; import {GuildDiscoveryRepository} from '../guild/repositories/GuildDiscoveryRepository';
import {GuildRepository} from '../guild/repositories/GuildRepository'; import {GuildRepository} from '../guild/repositories/GuildRepository';
@@ -349,6 +348,7 @@ export function setInjectedUnfurlerService(service: IUnfurlerService | undefined
} }
const getDefaultUnfurlerService = singleton(() => { const getDefaultUnfurlerService = singleton(() => {
const instanceConfigRepository = getInstanceConfigRepository();
const manager = new NatsConnectionManager({ const manager = new NatsConnectionManager({
url: Config.nats.coreUrl, url: Config.nats.coreUrl,
token: Config.nats.authToken || undefined, token: Config.nats.authToken || undefined,
@@ -357,7 +357,11 @@ const getDefaultUnfurlerService = singleton(() => {
void manager.connect().catch((error) => { void manager.connect().catch((error) => {
Logger.error({error}, '[nats-unfurl] Failed to establish NATS connection'); Logger.error({error}, '[nats-unfurl] Failed to establish NATS connection');
}); });
return new NatsUnfurlerService(manager, async () => getInstanceConfigRepository().getEffectiveYoutubeApiKey()); return new NatsUnfurlerService(
manager,
async () => instanceConfigRepository.getEffectiveYoutubeApiKey(),
async () => (await instanceConfigRepository.getEffectiveGifConfig()).klipy_api_key,
);
}); });
export function getUnfurlerService(): IUnfurlerService { export function getUnfurlerService(): IUnfurlerService {
@@ -376,24 +380,10 @@ export const getBotMfaMirrorService = singleton(
() => new BotMfaMirrorService(getApplicationRepository(), getUserRepository(), getGatewayService()), () => new BotMfaMirrorService(getApplicationRepository(), getUserRepository(), getGatewayService()),
); );
export const getGifService = singleton(() => { export const getGifService = singleton(() => {
const cache = getCacheService();
const media = getMediaService();
const instanceConfigRepository = getInstanceConfigRepository(); const instanceConfigRepository = getInstanceConfigRepository();
return new GifService({ return new GifService(
providers: [ createNatsGifProvider(async () => (await instanceConfigRepository.getEffectiveGifConfig()).klipy_api_key),
new TenorGifProvider( );
cache,
media,
async () => (await instanceConfigRepository.getEffectiveGifConfig()).tenor_api_key,
),
new KlipyGifProvider(
cache,
media,
async () => (await instanceConfigRepository.getEffectiveGifConfig()).klipy_api_key,
),
],
activeName: async () => (await instanceConfigRepository.getEffectiveGifConfig()).provider,
});
}); });
export const getExpressionAssetPurger = singleton(() => new ExpressionAssetPurger(getAssetDeletionQueue())); export const getExpressionAssetPurger = singleton(() => new ExpressionAssetPurger(getAssetDeletionQueue()));
export const getGuildAuditLogService = singleton( export const getGuildAuditLogService = singleton(
+3 -6
View File
@@ -24808,10 +24808,7 @@
"gif": { "gif": {
"type": "object", "type": "object",
"properties": { "properties": {
"provider": { "provider": {"type": "string", "description": "Stable machine name of the active GIF provider."},
"type": "string",
"description": "Stable machine name of the active GIF provider (e.g. \"klipy\", \"tenor\")"
},
"display_name": {"type": "string", "description": "Human-readable provider name shown in the UI"}, "display_name": {"type": "string", "description": "Human-readable provider name shown in the UI"},
"attribution_required": { "attribution_required": {
"type": "boolean", "type": "boolean",
@@ -29258,7 +29255,7 @@
}, },
"gif_provider": { "gif_provider": {
"anyOf": [{"type": "string"}, {"type": "null"}], "anyOf": [{"type": "string"}, {"type": "null"}],
"description": "Stable name of the GIF provider that issued gif_slug (e.g. \"klipy\", \"tenor\"), if any" "description": "Stable name of the GIF provider that issued gif_slug, if any. Legacy records may contain older provider names."
}, },
"media": { "media": {
"anyOf": [ "anyOf": [
@@ -35767,7 +35764,7 @@
}, },
"gif_provider": { "gif_provider": {
"anyOf": [{"type": "string"}, {"type": "null"}], "anyOf": [{"type": "string"}, {"type": "null"}],
"description": "Stable name of the GIF provider that issued gif_slug (e.g. \"klipy\", \"tenor\")" "description": "Stable name of the GIF provider that issued gif_slug. New provider GIFs are sourced from KLIPY."
}, },
"media": { "media": {
"anyOf": [ "anyOf": [
@@ -12,6 +12,7 @@ import {Logger} from '../Logger';
import type {HonoApp} from '../types/HonoEnv'; import type {HonoApp} from '../types/HonoEnv';
import type {IUserRepository} from '../user/IUserRepository'; import type {IUserRepository} from '../user/IUserRepository';
import {mapUserToPrivateResponse} from '../user/UserMappers'; import {mapUserToPrivateResponse} from '../user/UserMappers';
import {resolveRequestClientIp} from '../utils/IpUtils';
interface TwilioInboundSmsWebhookContext { interface TwilioInboundSmsWebhookContext {
authToken: string; authToken: string;
@@ -29,7 +30,7 @@ export function installTwilioInboundSmsWebhook(app: HonoApp, ctx: TwilioInboundS
const params = parseFormUrlEncoded(rawBody); const params = parseFormUrlEncoded(rawBody);
const signature = c.req.header('x-twilio-signature') ?? ''; const signature = c.req.header('x-twilio-signature') ?? '';
if (!verifyTwilioSignature(ctx.authToken, ctx.publicWebhookUrl, params, signature)) { if (!verifyTwilioSignature(ctx.authToken, ctx.publicWebhookUrl, params, signature)) {
Logger.warn({ip: c.req.header('cf-connecting-ip') ?? '?'}, 'Twilio webhook signature failed; rejecting'); Logger.warn({ip: resolveRequestClientIp(c.req.raw) ?? '?'}, 'Twilio webhook signature failed; rejecting');
return c.text('forbidden', 403); return c.text('forbidden', 403);
} }
const fromPhone = params.get('From') ?? ''; const fromPhone = params.get('From') ?? '';
@@ -18,8 +18,6 @@ export const EXTERNAL_RESPONSE_LIMITS = {
blueskyProfileBytes: 256 * 1024, blueskyProfileBytes: 256 * 1024,
wikipediaSummaryBytes: 256 * 1024, wikipediaSummaryBytes: 256 * 1024,
oEmbedBytes: 256 * 1024, oEmbedBytes: 256 * 1024,
klipyApiBytes: 512 * 1024,
tenorApiBytes: 512 * 1024,
pwnedPasswordsBytes: 1024 * 1024, pwnedPasswordsBytes: 1024 * 1024,
rdapBytes: 512 * 1024, rdapBytes: 512 * 1024,
externalTemplateBytes: 512 * 1024, externalTemplateBytes: 512 * 1024,
+8 -7
View File
@@ -16,16 +16,17 @@ interface GetIpAddressReverseOptions {
cacheTtlSeconds?: number; cacheTtlSeconds?: number;
} }
export function resolveRequestClientIp(req: Request): string | null {
return extractClientIp(req, {
trustClientIpHeader: Config.proxy.trust_client_ip_header,
clientIpHeaderName: Config.proxy.client_ip_header,
});
}
export async function lookupGeoip(req: Request): Promise<GeoipResult>; export async function lookupGeoip(req: Request): Promise<GeoipResult>;
export async function lookupGeoip(ip: string): Promise<GeoipResult>; export async function lookupGeoip(ip: string): Promise<GeoipResult>;
export async function lookupGeoip(input: string | Request): Promise<GeoipResult> { export async function lookupGeoip(input: string | Request): Promise<GeoipResult> {
const ip = const ip = typeof input === 'string' ? input : resolveRequestClientIp(input);
typeof input === 'string'
? input
: extractClientIp(input, {
trustClientIpHeader: Config.proxy.trust_client_ip_header,
clientIpHeaderName: Config.proxy.client_ip_header,
});
if (!ip) { if (!ip) {
return {countryCode: null, normalizedIp: null, city: null, region: null, countryName: null}; return {countryCode: null, normalizedIp: null, city: null, region: null, countryName: null};
} }
@@ -63,7 +63,6 @@ const LANE_CONFIG = {
batch: { batch: {
consumerName: 'workers_batch', consumerName: 'workers_batch',
tasks: [ tasks: [
'enqueueGifFeaturedCategoriesRefresh',
'expireAttachments', 'expireAttachments',
'indexChannelMessages', 'indexChannelMessages',
'indexGuildMembers', 'indexGuildMembers',
@@ -74,7 +73,6 @@ const LANE_CONFIG = {
'processPendingBulkMessageDeletions', 'processPendingBulkMessageDeletions',
'processPremiumStateReconciliationQueue', 'processPremiumStateReconciliationQueue',
'prunePostgresKvTtl', 'prunePostgresKvTtl',
'refreshGifFeaturedCategories',
'refreshSearchIndex', 'refreshSearchIndex',
'syncDiscoveryIndex', 'syncDiscoveryIndex',
'syncDisposableEmailDomains', 'syncDisposableEmailDomains',
-1
View File
@@ -52,7 +52,6 @@ function registerCronJobs(cron: CronScheduler): void {
cron.upsert('prunePostgresKvTtl', 'prunePostgresKvTtl', {}, '0 */5 * * * *'); cron.upsert('prunePostgresKvTtl', 'prunePostgresKvTtl', {}, '0 */5 * * * *');
cron.upsert('syncDiscoveryIndex', 'syncDiscoveryIndex', {}, '0 */15 * * * *'); cron.upsert('syncDiscoveryIndex', 'syncDiscoveryIndex', {}, '0 */15 * * * *');
cron.upsert('syncDisposableEmailDomains', 'syncDisposableEmailDomains', {}, '0 */30 * * * *'); cron.upsert('syncDisposableEmailDomains', 'syncDisposableEmailDomains', {}, '0 */30 * * * *');
cron.upsert('enqueueGifFeaturedCategoriesRefresh', 'enqueueGifFeaturedCategoriesRefresh', {}, '0 */30 * * * *');
cron.upsert('syncUrlBlocklists', 'syncUrlBlocklists', {}, '0 0 */6 * * *'); cron.upsert('syncUrlBlocklists', 'syncUrlBlocklists', {}, '0 0 */6 * * *');
cron.upsert('syncFileShaBlocklists', 'syncFileShaBlocklists', {}, '0 0 */12 * * *'); cron.upsert('syncFileShaBlocklists', 'syncFileShaBlocklists', {}, '0 0 */12 * * *');
cron.upsert('flushUserActivityBuffer', 'flushUserActivityBuffer', {}, '*/10 * * * * *'); cron.upsert('flushUserActivityBuffer', 'flushUserActivityBuffer', {}, '*/10 * * * * *');
@@ -13,7 +13,6 @@ import bulkDeleteSelfMessagesImmediate from './tasks/BulkDeleteSelfMessagesImmed
import bulkDeleteUserMessages from './tasks/BulkDeleteUserMessages'; import bulkDeleteUserMessages from './tasks/BulkDeleteUserMessages';
import bulkDeleteUserMessagesScoped from './tasks/BulkDeleteUserMessagesScoped'; import bulkDeleteUserMessagesScoped from './tasks/BulkDeleteUserMessagesScoped';
import deleteUserMessagesInGuildByTime from './tasks/DeleteUserMessagesInGuildByTime'; import deleteUserMessagesInGuildByTime from './tasks/DeleteUserMessagesInGuildByTime';
import enqueueGifFeaturedCategoriesRefresh from './tasks/EnqueueGifFeaturedCategoriesRefresh';
import expireAttachments from './tasks/ExpireAttachments'; import expireAttachments from './tasks/ExpireAttachments';
import extractEmbeds from './tasks/ExtractEmbeds'; import extractEmbeds from './tasks/ExtractEmbeds';
import finalizeNcmecAttachmentReport from './tasks/FinalizeNcmecAttachmentReport'; import finalizeNcmecAttachmentReport from './tasks/FinalizeNcmecAttachmentReport';
@@ -34,7 +33,6 @@ import processPremiumStateReconciliationQueue from './tasks/ProcessPremiumStateR
import processStripeWebhook from './tasks/ProcessStripeWebhook'; import processStripeWebhook from './tasks/ProcessStripeWebhook';
import prunePostgresKvTtl from './tasks/PrunePostgresKvTtl'; import prunePostgresKvTtl from './tasks/PrunePostgresKvTtl';
import reconcileUserPayments from './tasks/ReconcileUserPayments'; import reconcileUserPayments from './tasks/ReconcileUserPayments';
import refreshGifFeaturedCategories from './tasks/RefreshGifFeaturedCategories';
import refreshSearchIndex from './tasks/RefreshSearchIndex'; import refreshSearchIndex from './tasks/RefreshSearchIndex';
import revalidateUserConnections from './tasks/RevalidateUserConnections'; import revalidateUserConnections from './tasks/RevalidateUserConnections';
import {sendScheduledMessage} from './tasks/SendScheduledMessage'; import {sendScheduledMessage} from './tasks/SendScheduledMessage';
@@ -60,7 +58,6 @@ export const workerTasks: Record<WorkerTaskName, WorkerTaskHandler> = {
bulkUpdateSuspiciousActivityFlags: bulkUpdateSuspiciousActivityFlags, bulkUpdateSuspiciousActivityFlags: bulkUpdateSuspiciousActivityFlags,
bulkUpdateUserFlags: bulkUpdateUserFlags, bulkUpdateUserFlags: bulkUpdateUserFlags,
deleteUserMessagesInGuildByTime, deleteUserMessagesInGuildByTime,
enqueueGifFeaturedCategoriesRefresh,
expireAttachments, expireAttachments,
extractEmbeds, extractEmbeds,
finalizeNcmecAttachmentReport, finalizeNcmecAttachmentReport,
@@ -80,7 +77,6 @@ export const workerTasks: Record<WorkerTaskName, WorkerTaskHandler> = {
processPremiumStateReconciliationQueue, processPremiumStateReconciliationQueue,
reconcileUserPayments, reconcileUserPayments,
prunePostgresKvTtl, prunePostgresKvTtl,
refreshGifFeaturedCategories,
refreshSearchIndex, refreshSearchIndex,
revalidateUserConnections, revalidateUserConnections,
sendScheduledMessage, sendScheduledMessage,
@@ -1,39 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
import {listSeenLocales} from '../../gif/GifFeaturedCategoriesCache';
import {getGifService} from '../../middleware/ServiceSingletons';
import {getWorkerDependencies} from '../WorkerContext';
const enqueueGifFeaturedCategoriesRefresh: WorkerTaskHandler = async (_payload, helpers) => {
const {cacheService, workerService} = getWorkerDependencies();
const gifService = getGifService();
for (const provider of gifService.listProviders()) {
if (!(await provider.isAvailable())) continue;
const locales = await listSeenLocales(cacheService, provider.meta.name);
if (locales.length === 0) {
helpers.logger.debug({provider: provider.meta.name}, 'No seen locales for GIF provider; nothing to refresh');
continue;
}
helpers.logger.debug(
{provider: provider.meta.name, count: locales.length},
'Fanning out enriched GIF categories refresh',
);
for (const {locale, country} of locales) {
try {
await workerService.addJob('refreshGifFeaturedCategories', {
provider: provider.meta.name,
locale,
country,
});
} catch (error) {
helpers.logger.warn(
{err: error, provider: provider.meta.name, locale, country},
'Failed to enqueue per-locale GIF categories refresh',
);
}
}
}
};
export default enqueueGifFeaturedCategoriesRefresh;
@@ -1,33 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
import {getGifService} from '../../middleware/ServiceSingletons';
interface RefreshPayload {
provider?: string;
locale?: string;
country?: string;
}
const refreshGifFeaturedCategories: WorkerTaskHandler = async (rawPayload, helpers) => {
const payload = (rawPayload ?? {}) as RefreshPayload;
const {provider, locale, country} = payload;
if (!provider || !locale || !country) {
helpers.logger.warn({payload}, 'refreshGifFeaturedCategories called without required fields');
return;
}
const gifService = getGifService();
const target = gifService.getByName(provider);
if (!target) {
helpers.logger.warn({provider}, 'refreshGifFeaturedCategories: unknown provider');
return;
}
if (!(await target.isAvailable())) {
helpers.logger.debug({provider}, 'refreshGifFeaturedCategories: provider not configured, skipping');
return;
}
helpers.logger.debug({provider, locale, country}, 'Refreshing enriched GIF featured categories');
await target.refreshFeaturedCategories({locale, country});
};
export default refreshGifFeaturedCategories;
+22
View File
@@ -0,0 +1,22 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
[package]
name = "fluxer-gifs"
version = "0.1.0"
edition.workspace = true
license.workspace = true
[dependencies]
anyhow = "1.0.102"
base64 = "0.22.1"
fluxer-svc = { path = "../fluxer_svc", default-features = false }
hmac = "0.13.0"
moka = { version = "0.12.15", features = ["future", "sync"] }
reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls"] }
serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.150"
sha2 = "0.11.0"
tokio = { version = "1.52.3", features = ["macros", "rt-multi-thread", "signal", "sync", "time"] }
tracing = "0.1.44"
url = "2.5"
urlencoding = "2.1"
+28
View File
@@ -0,0 +1,28 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
FROM rust:1-bookworm AS builder
WORKDIR /usr/src/app
COPY . .
RUN cargo build --release -p fluxer-gifs
FROM debian:bookworm-slim
ARG BUILD_VERSION=""
WORKDIR /usr/local/bin
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && \
rm -rf /var/lib/apt/lists/*
COPY --from=builder /usr/src/app/target/release/fluxer-gifs /usr/local/bin/fluxer-gifs
ENV BUILD_VERSION="${BUILD_VERSION}"
USER 65532:65532
EXPOSE 8090
CMD ["/usr/local/bin/fluxer-gifs"]
+721
View File
@@ -0,0 +1,721 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::media_proxy::MediaProxyUrlBuilder;
use crate::types::{GifCategoryTag, GifItem, GifMediaFormat};
use anyhow::Context;
use reqwest::Url;
use serde::Deserialize;
use serde_json::Value;
use std::collections::{BTreeMap, HashSet};
use std::time::Duration;
use tokio::time::sleep;
const KLIPY_BASE_URL: &str = "https://api.klipy.com/v2";
const KLIPY_DIRECT_BASE_URL: &str = "https://api.klipy.com/api/v1";
const DEFAULT_CONTENT_FILTER: &str = "low";
const CLIENT_KEY: &str = "fluxer";
const MAX_RETRIES: usize = 3;
const BACKOFF_BASE_DELAY: Duration = Duration::from_secs(1);
const KLIPY_RESPONSE_LIMIT_BYTES: usize = 512 * 1024;
const FLUXER_USER_AGENT: &str = "Fluxerbot/1.0 (+https://fluxer.app)";
const KLIPY_PROVIDER_NAME: &str = "klipy";
const KLIPY_FEATURED_CATEGORY_REFRESH_COUNTRY: &str = "US";
const SIZE_PREFERENCE: [&str; 4] = ["hd", "md", "sm", "xs"];
const FORMAT_PREFERENCE: [&str; 4] = ["webm", "mp4", "webp", "gif"];
#[derive(Clone)]
pub struct KlipyClient {
http_client: reqwest::Client,
media_proxy: MediaProxyUrlBuilder,
}
#[derive(Debug, Deserialize)]
struct ResultsResponse {
results: Vec<Value>,
}
#[derive(Debug, Deserialize)]
struct TagsResponse {
tags: Vec<Value>,
}
#[derive(Debug, Deserialize)]
struct KlipyGif {
id: Value,
#[serde(default)]
slug: Option<String>,
#[serde(default)]
title: String,
#[serde(default)]
itemurl: Option<String>,
#[serde(default)]
file: Option<BTreeMap<String, BTreeMap<String, KlipyFileEntry>>>,
#[serde(default)]
media_formats: Option<KlipyMediaFormats>,
}
#[derive(Debug, Deserialize)]
struct KlipyMediaFormats {
#[serde(default)]
webm: Option<KlipyFallbackMediaFormat>,
}
#[derive(Debug, Deserialize)]
struct KlipyFallbackMediaFormat {
url: String,
dims: [i32; 2],
}
#[derive(Debug, Deserialize)]
struct KlipyFileEntry {
#[serde(default)]
url: Option<String>,
#[serde(default)]
width: Option<i32>,
#[serde(default)]
height: Option<i32>,
}
#[derive(Debug, Deserialize)]
struct KlipyCategoryTag {
searchterm: String,
}
#[derive(Debug, Deserialize)]
struct DirectGifResponse {
#[serde(default)]
data: Option<KlipyGif>,
}
enum KlipyJsonFetch<T> {
Found(T),
NotFound,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct KlipyPath {
path_type: KlipyPathType,
slug: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum KlipyPathType {
Gif,
Clip,
}
impl KlipyClient {
pub fn new(media_proxy: MediaProxyUrlBuilder) -> anyhow::Result<Self> {
let http_client = reqwest::Client::builder()
.user_agent(FLUXER_USER_AGENT)
.timeout(Duration::from_secs(30))
.build()
.context("failed to build KLIPY HTTP client")?;
Ok(Self {
http_client,
media_proxy,
})
}
pub async fn search(
&self,
api_key: &str,
q: &str,
locale: &str,
country: &str,
limit: u32,
) -> anyhow::Result<Vec<GifItem>> {
let locale = normalize_locale(locale);
let limit = limit.to_string();
self.fetch_gifs(
"search",
&[
("key", api_key),
("q", q),
("country", country),
("locale", &locale),
("limit", &limit),
],
)
.await
}
pub async fn featured_gifs(
&self,
api_key: &str,
locale: &str,
country: &str,
) -> anyhow::Result<Vec<GifItem>> {
let locale = normalize_locale(locale);
self.fetch_gifs(
"featured",
&[
("key", api_key),
("country", country),
("locale", &locale),
("limit", "1"),
],
)
.await
}
pub async fn trending_gifs(
&self,
api_key: &str,
locale: &str,
country: &str,
) -> anyhow::Result<Vec<GifItem>> {
let locale = normalize_locale(locale);
self.fetch_gifs(
"featured",
&[
("key", api_key),
("country", country),
("locale", &locale),
("limit", "50"),
],
)
.await
}
pub async fn suggestions(
&self,
api_key: &str,
q: &str,
locale: &str,
) -> anyhow::Result<Vec<String>> {
let locale = normalize_locale(locale);
let response: ResultsResponse = self
.fetch_json(
"autocomplete",
&[("key", api_key), ("q", q), ("locale", &locale)],
)
.await?;
Ok(response
.results
.into_iter()
.filter_map(|value| value.as_str().map(ToOwned::to_owned))
.collect())
}
pub async fn register_share(
&self,
api_key: &str,
id: &str,
q: &str,
locale: &str,
country: &str,
) -> anyhow::Result<()> {
let locale = normalize_locale(locale);
let url = self.create_url(
"registershare",
&[
("key", api_key),
("id", id),
("country", country),
("locale", &locale),
("q", q),
],
)?;
let response = self.http_client.get(url).send().await?;
if !response.status().is_success() {
anyhow::bail!(
"KLIPY registershare failed with status {}",
response.status()
);
}
Ok(())
}
pub async fn resolve_by_url(
&self,
api_key: &str,
url: &str,
_locale: &str,
_country: &str,
) -> anyhow::Result<Option<GifItem>> {
let Some(path) = parse_klipy_path(url) else {
return Ok(None);
};
self.fetch_direct_gif(api_key, &path).await
}
pub async fn featured_categories(
&self,
api_key: &str,
locale: &str,
) -> anyhow::Result<Vec<GifCategoryTag>> {
let normalized_locale = normalize_locale(locale);
let response: TagsResponse = self
.fetch_json(
"categories",
&[
("key", api_key),
("country", KLIPY_FEATURED_CATEGORY_REFRESH_COUNTRY),
("locale", &normalized_locale),
("type", "featured"),
],
)
.await?;
let mut seen = HashSet::new();
let search_terms = response
.tags
.into_iter()
.filter_map(|value| serde_json::from_value::<KlipyCategoryTag>(value).ok())
.map(|tag| tag.searchterm.trim().to_owned())
.filter(|term| !term.is_empty())
.filter(|term| seen.insert(term.clone()))
.collect::<Vec<_>>();
let mut categories = Vec::with_capacity(search_terms.len());
for search_term in search_terms {
let gif = match self
.search(
api_key,
&search_term,
&normalized_locale,
KLIPY_FEATURED_CATEGORY_REFRESH_COUNTRY,
1,
)
.await
{
Ok(mut gifs) => gifs.drain(..).next(),
Err(err) => {
tracing::debug!(
error = %err,
search_term = %search_term,
locale = %normalized_locale,
"failed to fetch KLIPY category preview GIF"
);
None
}
};
categories.push(category_response(search_term, gif));
}
Ok(categories)
}
async fn fetch_gifs(
&self,
endpoint: &str,
params: &[(&str, &str)],
) -> anyhow::Result<Vec<GifItem>> {
let response: ResultsResponse = self.fetch_json(endpoint, params).await?;
Ok(response
.results
.into_iter()
.filter_map(|value| serde_json::from_value::<KlipyGif>(value).ok())
.filter_map(|gif| self.transform_gif(gif))
.collect())
}
async fn fetch_direct_gif(
&self,
api_key: &str,
path: &KlipyPath,
) -> anyhow::Result<Option<GifItem>> {
let url = self.create_direct_url(api_key, path)?;
let mut last_error = None;
for attempt in 0..MAX_RETRIES {
match self.fetch_direct_gif_once(url.clone(), path).await {
Ok(value) => return Ok(value),
Err(error) if attempt + 1 < MAX_RETRIES => {
last_error = Some(error);
sleep(BACKOFF_BASE_DELAY * 2_u32.pow(attempt as u32)).await;
}
Err(error) => return Err(error),
}
}
Err(last_error.unwrap_or_else(|| anyhow::anyhow!("exceeded KLIPY retry limit")))
}
async fn fetch_direct_gif_once(
&self,
url: Url,
path: &KlipyPath,
) -> anyhow::Result<Option<GifItem>> {
match self.fetch_json_response::<DirectGifResponse>(url).await? {
KlipyJsonFetch::NotFound => Ok(None),
KlipyJsonFetch::Found(response) => Ok(response
.data
.and_then(|gif| self.transform_gif_with_path(gif, Some(path)))),
}
}
async fn fetch_json<T>(&self, endpoint: &str, params: &[(&str, &str)]) -> anyhow::Result<T>
where
T: serde::de::DeserializeOwned,
{
let url = self.create_url(endpoint, params)?;
let mut last_error = None;
for attempt in 0..MAX_RETRIES {
match self.fetch_json_once(url.clone()).await {
Ok(value) => return Ok(value),
Err(error) if attempt + 1 < MAX_RETRIES => {
last_error = Some(error);
sleep(BACKOFF_BASE_DELAY * 2_u32.pow(attempt as u32)).await;
}
Err(error) => return Err(error),
}
}
Err(last_error.unwrap_or_else(|| anyhow::anyhow!("exceeded KLIPY retry limit")))
}
async fn fetch_json_once<T>(&self, url: Url) -> anyhow::Result<T>
where
T: serde::de::DeserializeOwned,
{
match self.fetch_json_response(url).await? {
KlipyJsonFetch::Found(value) => Ok(value),
KlipyJsonFetch::NotFound => anyhow::bail!("KLIPY request returned not found"),
}
}
async fn fetch_json_response<T>(&self, url: Url) -> anyhow::Result<KlipyJsonFetch<T>>
where
T: serde::de::DeserializeOwned,
{
let response = self.http_client.get(url.clone()).send().await?;
if response.status() == reqwest::StatusCode::NOT_FOUND {
return Ok(KlipyJsonFetch::NotFound);
}
if !response.status().is_success() {
anyhow::bail!("KLIPY request failed with status {}", response.status());
}
if response
.content_length()
.is_some_and(|len| len > KLIPY_RESPONSE_LIMIT_BYTES as u64)
{
anyhow::bail!("KLIPY response declared more than {KLIPY_RESPONSE_LIMIT_BYTES} bytes");
}
let bytes = response.bytes().await?;
if bytes.len() > KLIPY_RESPONSE_LIMIT_BYTES {
anyhow::bail!("KLIPY response exceeded {KLIPY_RESPONSE_LIMIT_BYTES} bytes");
}
serde_json::from_slice(&bytes)
.with_context(|| format!("failed to parse KLIPY response from {url}"))
.map(KlipyJsonFetch::Found)
}
fn create_url(&self, endpoint: &str, params: &[(&str, &str)]) -> anyhow::Result<Url> {
let mut url = Url::parse(&format!("{KLIPY_BASE_URL}/{endpoint}"))?;
{
let mut query = url.query_pairs_mut();
query.append_pair("client_key", CLIENT_KEY);
query.append_pair("contentfilter", DEFAULT_CONTENT_FILTER);
for (key, value) in params {
query.append_pair(key, value);
}
}
Ok(url)
}
fn create_direct_url(&self, api_key: &str, path: &KlipyPath) -> anyhow::Result<Url> {
let mut url = Url::parse(&format!("{KLIPY_DIRECT_BASE_URL}/"))?;
{
let mut segments = url
.path_segments_mut()
.map_err(|_| anyhow::anyhow!("KLIPY direct base URL cannot be a base"))?;
segments
.push(api_key)
.push(klipy_resource(path.path_type))
.push(&path.slug);
}
Ok(url)
}
fn transform_gif(&self, input: KlipyGif) -> Option<GifItem> {
self.transform_gif_with_path(input, None)
}
fn transform_gif_with_path(
&self,
input: KlipyGif,
fallback_path: Option<&KlipyPath>,
) -> Option<GifItem> {
let parsed_path = input.itemurl.as_deref().and_then(parse_klipy_path);
let resolved_path = parsed_path.as_ref().or(fallback_path);
let explicit_slug = input
.slug
.as_deref()
.map(str::trim)
.filter(|slug| !slug.is_empty());
let fallback_id = klipy_id_as_string(&input.id)?;
let normalized_slug = explicit_slug
.or_else(|| resolved_path.map(|path| path.slug.as_str()))
.unwrap_or(fallback_id.as_str())
.to_owned();
let normalized_type = resolved_path
.map(|path| path.path_type)
.unwrap_or(KlipyPathType::Gif);
let normalized_url = if resolved_path.is_some() || explicit_slug.is_some() {
build_share_url_with_type(normalized_type, &normalized_slug)
} else {
input
.itemurl
.clone()
.unwrap_or_else(|| build_share_url_with_type(normalized_type, &normalized_slug))
};
let (media, preferred) = self.collect_media(&input);
let top = media.get("webm").cloned().or(preferred)?;
Some(GifItem {
id: normalized_slug.clone(),
slug: normalized_slug,
provider: KLIPY_PROVIDER_NAME.to_owned(),
title: input.title,
url: normalized_url,
src: top.src.clone(),
proxy_src: top.proxy_src.clone(),
width: top.width,
height: top.height,
media,
placeholder: None,
})
}
fn collect_media(
&self,
input: &KlipyGif,
) -> (BTreeMap<String, GifMediaFormat>, Option<GifMediaFormat>) {
let mut media = BTreeMap::new();
let mut preferred = None;
for size in SIZE_PREFERENCE {
let Some(bucket) = input.file.as_ref().and_then(|files| files.get(size)) else {
continue;
};
for format in FORMAT_PREFERENCE {
let Some(entry) = bucket.get(format) else {
continue;
};
let Some(media_format) = self.to_media_format(entry) else {
continue;
};
let public_key = public_format_key(size, format);
media.insert(public_key, media_format.clone());
if preferred.is_none() {
preferred = Some(media_format);
}
}
}
if media.is_empty()
&& let Some(webm) = input
.media_formats
.as_ref()
.and_then(|formats| formats.webm.as_ref())
&& webm.dims[0] > 0
&& webm.dims[1] > 0
&& let Some(proxy_src) = self.media_proxy.external_proxy_url(&webm.url)
{
let fallback = GifMediaFormat {
src: webm.url.clone(),
proxy_src,
width: webm.dims[0],
height: webm.dims[1],
};
media.insert("webm".to_owned(), fallback.clone());
preferred = Some(fallback);
}
(media, preferred)
}
fn to_media_format(&self, entry: &KlipyFileEntry) -> Option<GifMediaFormat> {
let src = entry.url.as_ref()?;
let width = entry.width.filter(|width| *width > 0)?;
let height = entry.height.filter(|height| *height > 0)?;
let proxy_src = self.media_proxy.external_proxy_url(src)?;
Some(GifMediaFormat {
src: src.clone(),
proxy_src,
width,
height,
})
}
}
pub fn normalize_locale(locale: &str) -> String {
locale.replace('-', "_")
}
pub fn build_share_url(slug: &str) -> String {
let trimmed = slug.trim();
if trimmed.is_empty() {
return "https://klipy.com/gifs".to_owned();
}
build_share_url_with_type(KlipyPathType::Gif, trimmed)
}
pub fn extract_slug_from_url(url: &str) -> Option<String> {
parse_klipy_path(url).map(|path| path.slug)
}
fn klipy_id_as_string(value: &Value) -> Option<String> {
match value {
Value::String(value) => {
let trimmed = value.trim();
(!trimmed.is_empty()).then(|| trimmed.to_owned())
}
Value::Number(value) => Some(value.to_string()),
_ => None,
}
}
fn parse_klipy_path(raw_url: &str) -> Option<KlipyPath> {
let parsed = Url::parse(raw_url).ok()?;
let hostname = parsed.host_str()?.to_ascii_lowercase();
if hostname != "klipy.com" && hostname != "www.klipy.com" {
return None;
}
let mut segments = parsed.path_segments()?;
let kind = segments.next()?.to_ascii_lowercase();
let slug = segments.next()?.trim().to_owned();
if slug.is_empty() {
return None;
}
let path_type = match kind.as_str() {
"gif" | "gifs" => KlipyPathType::Gif,
"clip" | "clips" => KlipyPathType::Clip,
_ => return None,
};
Some(KlipyPath { path_type, slug })
}
fn build_share_url_with_type(path_type: KlipyPathType, slug: &str) -> String {
let base_path = match path_type {
KlipyPathType::Gif => "gifs",
KlipyPathType::Clip => "clips",
};
let encoded_slug = urlencoding::encode(slug);
format!("https://klipy.com/{base_path}/{encoded_slug}")
}
fn klipy_resource(path_type: KlipyPathType) -> &'static str {
match path_type {
KlipyPathType::Gif => "gifs",
KlipyPathType::Clip => "clips",
}
}
fn public_format_key(size: &str, format: &str) -> String {
match (size, format) {
("hd", "webm") => "webm",
("hd", "mp4") => "mp4",
("hd", "webp") => "webp",
("hd", "gif") => "gif",
("md", "webm") => "mediumwebm",
("md", "mp4") => "mediummp4",
("md", "webp") => "mediumwebp",
("md", "gif") => "mediumgif",
("sm", "webm") => "tinywebm",
("sm", "mp4") => "tinymp4",
("sm", "webp") => "tinywebp",
("sm", "gif") => "tinygif",
("xs", "webm") => "nanowebm",
("xs", "mp4") => "nanomp4",
("xs", "webp") => "nanowebp",
("xs", "gif") => "nanogif",
_ => format,
}
.to_owned()
}
fn category_response(name: String, gif: Option<GifItem>) -> GifCategoryTag {
GifCategoryTag {
src: gif.as_ref().map(|gif| gif.src.clone()).unwrap_or_default(),
proxy_src: gif
.as_ref()
.map(|gif| gif.proxy_src.clone())
.unwrap_or_default(),
gif,
name,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn locale_uses_klipy_underscore_form() {
assert_eq!(normalize_locale("en-US"), "en_US");
assert_eq!(normalize_locale("sv_SE"), "sv_SE");
}
#[test]
fn extracts_klipy_slug_only_from_klipy_hosts() {
assert_eq!(
extract_slug_from_url("https://klipy.com/gifs/funny-123").as_deref(),
Some("funny-123")
);
assert_eq!(
extract_slug_from_url("https://www.klipy.com/clip/abc").as_deref(),
Some("abc")
);
assert_eq!(
extract_slug_from_url("https://notklipy.com/gifs/funny"),
None
);
}
#[test]
fn build_share_url_uses_gifs_path() {
assert_eq!(build_share_url("hello"), "https://klipy.com/gifs/hello");
assert_eq!(build_share_url(" "), "https://klipy.com/gifs");
assert_eq!(
build_share_url_with_type(KlipyPathType::Clip, "hello"),
"https://klipy.com/clips/hello"
);
}
#[test]
fn stringifies_numeric_klipy_ids() {
assert_eq!(
klipy_id_as_string(&serde_json::json!(2484942301552561_i64)).as_deref(),
Some("2484942301552561")
);
assert_eq!(
klipy_id_as_string(&serde_json::json!(" abc ")).as_deref(),
Some("abc")
);
assert_eq!(klipy_id_as_string(&serde_json::json!(" ")), None);
}
#[test]
fn maps_provider_format_keys() {
assert_eq!(public_format_key("hd", "webm"), "webm");
assert_eq!(public_format_key("sm", "gif"), "tinygif");
assert_eq!(public_format_key("xs", "webp"), "nanowebp");
}
#[tokio::test]
#[ignore]
async fn live_resolves_klipy_url_with_direct_lookup() {
let api_key = std::env::var("FLUXER_KLIPY_API_KEY")
.or_else(|_| std::env::var("KLIPY_API_KEY"))
.expect("FLUXER_KLIPY_API_KEY or KLIPY_API_KEY set");
let client =
KlipyClient::new(MediaProxyUrlBuilder::from_env().expect("media proxy env configured"))
.expect("KLIPY client");
let gif = client
.resolve_by_url(
&api_key,
"https://klipy.com/gifs/goatplaybanjo-chat-4",
"en-US",
"US",
)
.await
.expect("KLIPY direct lookup")
.expect("resolved GIF");
assert_eq!(gif.slug, "goatplaybanjo-chat-4");
assert_eq!(gif.provider, KLIPY_PROVIDER_NAME);
assert_eq!(gif.url, "https://klipy.com/gifs/goatplaybanjo-chat-4");
assert!(gif.width > 0);
assert!(gif.height > 0);
assert!(gif.media.contains_key("webm") || gif.media.contains_key("mp4"));
assert!(gif.proxy_src.starts_with("http"));
}
}
+39
View File
@@ -0,0 +1,39 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
mod klipy;
mod media_proxy;
mod router_impl;
mod shard_impl;
mod types;
use fluxer_svc::config::{Mode, ServiceConfig};
use fluxer_svc::transport::NatsTransport;
use router_impl::GifsRouter;
use shard_impl::GifsShard;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
fluxer_svc::init_tracing();
let config = ServiceConfig::from_env()?;
let transport = NatsTransport::connect(&config.nats_url).await?;
tracing::info!(
service = config.service_name,
mode = ?config.mode,
shard_id = config.shard_id,
shard_count = config.shard_count,
listen_addr = %config.listen_addr,
"starting gifs service"
);
match config.mode {
Mode::Router => {
let router = GifsRouter::new(config.cache_max_entries, config.cache_ttl);
fluxer_svc::router::run_router(&config, router, transport).await
}
Mode::Shard => {
let shard = GifsShard::new(&config)?;
fluxer_svc::shard::run_shard(&config, shard, transport).await
}
}
}
+216
View File
@@ -0,0 +1,216 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use base64::prelude::*;
use hmac::{Hmac, KeyInit, Mac};
use sha2::Sha256;
use url::Url;
const V2_PATH_PREFIX: &str = "v2/";
#[derive(Clone)]
pub struct MediaProxyUrlBuilder {
endpoint: String,
endpoint_host: Option<String>,
secret_key: String,
}
impl MediaProxyUrlBuilder {
pub fn from_env() -> anyhow::Result<Self> {
let endpoint = std::env::var("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT")
.or_else(|_| std::env::var("FLUXER_MEDIA_ENDPOINT"))
.unwrap_or_default();
if endpoint.trim().is_empty() {
anyhow::bail!(
"gifs shard requires FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT or FLUXER_MEDIA_ENDPOINT"
);
}
let secret_key = std::env::var("FLUXER_MEDIA_PROXY_SECRET_KEY").unwrap_or_default();
if secret_key.trim().is_empty() {
anyhow::bail!("gifs shard requires FLUXER_MEDIA_PROXY_SECRET_KEY");
}
let endpoint = endpoint.trim_end_matches('/').to_owned();
let endpoint_host = Url::parse(&endpoint)
.ok()
.and_then(|parsed| parsed.host_str().map(ToOwned::to_owned));
Ok(Self {
endpoint,
endpoint_host,
secret_key,
})
}
pub fn external_proxy_url(&self, input_url: &str) -> Option<String> {
let parsed = Url::parse(input_url).ok()?;
if self
.endpoint_host
.as_deref()
.is_some_and(|host| parsed.host_str() == Some(host))
{
return Some(input_url.to_owned());
}
let proxy_path = build_external_media_proxy_path(parsed.as_str());
let signature = create_signature(&proxy_path, &self.secret_key);
Some(format!(
"{}/external/{signature}/{proxy_path}",
self.endpoint
))
}
}
fn build_external_media_proxy_path(input_url: &str) -> String {
format!(
"{V2_PATH_PREFIX}{}",
BASE64_URL_SAFE_NO_PAD.encode(input_url)
)
}
fn create_signature(input: &str, secret: &str) -> String {
let mut mac = Hmac::<Sha256>::new_from_slice(secret.as_bytes()).expect("HMAC accepts any key");
mac.update(input.as_bytes());
BASE64_URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes())
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
static ENV_LOCK: Mutex<()> = Mutex::new(());
fn with_media_proxy_env(
vars: &[(&str, Option<&str>)],
test: impl FnOnce() -> anyhow::Result<()>,
) -> anyhow::Result<()> {
let _guard = ENV_LOCK.lock().unwrap();
let keys = [
"FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT",
"FLUXER_MEDIA_ENDPOINT",
"FLUXER_MEDIA_PROXY_ENDPOINT",
"FLUXER_MEDIA_PROXY_SECRET_KEY",
];
let saved = keys
.iter()
.map(|key| (*key, std::env::var(key).ok()))
.collect::<Vec<_>>();
for key in keys {
unsafe {
std::env::remove_var(key);
}
}
for (key, value) in vars {
if let Some(value) = value {
unsafe {
std::env::set_var(key, value);
}
}
}
let result = test();
for (key, value) in saved {
match value {
Some(value) => unsafe {
std::env::set_var(key, value);
},
None => unsafe {
std::env::remove_var(key);
},
}
}
result
}
#[test]
fn external_proxy_url_builds_v2_signed_url() {
let builder = MediaProxyUrlBuilder {
endpoint: "https://media.example.test".to_owned(),
endpoint_host: Some("media.example.test".to_owned()),
secret_key: "secret".to_owned(),
};
let url = builder
.external_proxy_url("https://img.klipy.com/a.webp?x=1")
.expect("proxy url");
assert!(url.starts_with("https://media.example.test/external/"));
assert!(url.contains("/v2/"));
assert_eq!(
builder.external_proxy_url("https://media.example.test/external/existing"),
Some("https://media.example.test/external/existing".to_owned())
);
}
#[test]
fn from_env_uses_public_endpoint_when_internal_proxy_endpoint_is_set() -> anyhow::Result<()> {
with_media_proxy_env(
&[
(
"FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT",
Some("https://media.example.test/"),
),
(
"FLUXER_MEDIA_PROXY_ENDPOINT",
Some("http://media-proxy:8080"),
),
("FLUXER_MEDIA_PROXY_SECRET_KEY", Some("secret")),
],
|| {
let builder = MediaProxyUrlBuilder::from_env()?;
assert_eq!(builder.endpoint, "https://media.example.test");
Ok(())
},
)
}
#[test]
fn from_env_accepts_legacy_public_media_endpoint() -> anyhow::Result<()> {
with_media_proxy_env(
&[
(
"FLUXER_MEDIA_ENDPOINT",
Some("https://media.example.test/media"),
),
(
"FLUXER_MEDIA_PROXY_ENDPOINT",
Some("http://media-proxy:8080"),
),
("FLUXER_MEDIA_PROXY_SECRET_KEY", Some("secret")),
],
|| {
let builder = MediaProxyUrlBuilder::from_env()?;
assert_eq!(builder.endpoint, "https://media.example.test/media");
Ok(())
},
)
}
#[test]
fn from_env_rejects_internal_proxy_endpoint_without_public_endpoint() -> anyhow::Result<()> {
with_media_proxy_env(
&[
(
"FLUXER_MEDIA_PROXY_ENDPOINT",
Some("http://media-proxy:8080"),
),
("FLUXER_MEDIA_PROXY_SECRET_KEY", Some("secret")),
],
|| {
let err = MediaProxyUrlBuilder::from_env()
.err()
.expect("internal endpoint must not be accepted as public endpoint")
.to_string();
assert!(err.contains("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT"));
Ok(())
},
)
}
}
+183
View File
@@ -0,0 +1,183 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::types::{GifRequest, GifServiceResponse};
use fluxer_svc::router::RouterService;
use moka::sync::Cache;
use std::time::Duration;
pub struct GifsRouter {
l1: Cache<String, GifServiceResponse>,
}
impl GifsRouter {
pub fn new(max_entries: u64, ttl: Duration) -> Self {
Self {
l1: Cache::builder()
.max_capacity(max_entries)
.time_to_live(ttl)
.build(),
}
}
}
impl RouterService for GifsRouter {
type Request = GifRequest;
type Response = GifServiceResponse;
fn service_name(&self) -> &str {
"gifs"
}
fn route_key(req: &GifRequest) -> String {
request_key(req).unwrap_or_else(|| "uncached".to_owned())
}
fn coalesce_key(req: &GifRequest) -> Option<String> {
request_key(req)
}
fn l1_lookup(&self, req: &GifRequest) -> Option<GifServiceResponse> {
request_l1_key(req).and_then(|key| self.l1.get(&key))
}
fn l1_insert(&self, req: &GifRequest, resp: &GifServiceResponse) {
if response_is_cacheable(resp)
&& let Some(key) = request_l1_key(req)
{
self.l1.insert(key, resp.clone());
}
}
fn l1_invalidate(&self, key: &str) {
self.l1.invalidate(key);
}
}
fn request_l1_key(req: &GifRequest) -> Option<String> {
match req {
GifRequest::Search { .. }
| GifRequest::GetFeatured { .. }
| GifRequest::GetTrendingGifs { .. }
| GifRequest::Suggest { .. }
| GifRequest::ResolveByUrl { .. }
| GifRequest::BuildShareUrl { .. }
| GifRequest::ExtractSlugFromUrl { .. } => request_key(req),
GifRequest::IsAvailable { .. } | GifRequest::RegisterShare { .. } => None,
}
}
fn request_key(req: &GifRequest) -> Option<String> {
match req {
GifRequest::IsAvailable { .. } => None,
GifRequest::Search {
q, locale, country, ..
} => Some(format!("search:{locale}:{country}:{q}")),
GifRequest::GetFeatured {
locale, country, ..
} => Some(format!("featured:{locale}:{country}")),
GifRequest::GetTrendingGifs {
locale, country, ..
} => Some(format!("trending:{locale}:{country}")),
GifRequest::Suggest { q, locale, .. } => Some(format!("suggest:{locale}:{q}")),
GifRequest::RegisterShare { .. } => None,
GifRequest::ResolveByUrl {
url,
locale,
country,
..
} => Some(format!("resolve:{locale}:{country}:{url}")),
GifRequest::BuildShareUrl { slug } => Some(format!("share-url:{slug}")),
GifRequest::ExtractSlugFromUrl { url } => Some(format!("extract-slug:{url}")),
}
}
fn response_is_cacheable(resp: &GifServiceResponse) -> bool {
!matches!(
resp,
GifServiceResponse::Available { .. }
| GifServiceResponse::Registered
| GifServiceResponse::Failed { .. }
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn coalesce_key_excludes_api_key() {
let first = GifRequest::Search {
api_key: "a".to_owned(),
q: "wave".to_owned(),
locale: "en_US".to_owned(),
country: "US".to_owned(),
};
let second = GifRequest::Search {
api_key: "b".to_owned(),
q: "wave".to_owned(),
locale: "en_US".to_owned(),
country: "US".to_owned(),
};
assert_eq!(
GifsRouter::coalesce_key(&first),
GifsRouter::coalesce_key(&second)
);
}
#[test]
fn trending_key_is_locale_country_scoped_and_api_key_free() {
let base = GifRequest::GetTrendingGifs {
api_key: "a".to_owned(),
locale: "en_US".to_owned(),
country: "US".to_owned(),
};
let different_api_key = GifRequest::GetTrendingGifs {
api_key: "b".to_owned(),
locale: "en_US".to_owned(),
country: "US".to_owned(),
};
let different_locale = GifRequest::GetTrendingGifs {
api_key: "a".to_owned(),
locale: "sv_SE".to_owned(),
country: "US".to_owned(),
};
let different_country = GifRequest::GetTrendingGifs {
api_key: "a".to_owned(),
locale: "en_US".to_owned(),
country: "SE".to_owned(),
};
assert_eq!(
GifsRouter::coalesce_key(&base),
Some("trending:en_US:US".to_owned())
);
assert_eq!(
GifsRouter::coalesce_key(&base),
GifsRouter::coalesce_key(&different_api_key)
);
assert_ne!(
GifsRouter::coalesce_key(&base),
GifsRouter::coalesce_key(&different_locale)
);
assert_ne!(
GifsRouter::coalesce_key(&base),
GifsRouter::coalesce_key(&different_country)
);
assert_eq!(request_l1_key(&base), Some("trending:en_US:US".to_owned()));
}
#[test]
fn register_share_is_not_cached_or_coalesced() {
let request = GifRequest::RegisterShare {
api_key: "key".to_owned(),
id: "gif".to_owned(),
q: "wave".to_owned(),
locale: "en_US".to_owned(),
country: "US".to_owned(),
};
assert_eq!(GifsRouter::coalesce_key(&request), None);
assert_eq!(request_l1_key(&request), None);
}
}
+410
View File
@@ -0,0 +1,410 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::klipy::{KlipyClient, build_share_url, extract_slug_from_url};
use crate::media_proxy::MediaProxyUrlBuilder;
use crate::types::{GifCategoryTag, GifItem, GifRequest, GifServiceResponse};
use fluxer_svc::config::ServiceConfig;
use fluxer_svc::shard::ShardService;
use moka::future::Cache;
use std::collections::HashSet;
use std::future::Future;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::Mutex;
const SEARCH_SOFT_TTL: Duration = Duration::from_secs(30);
const SEARCH_HARD_TTL: Duration = Duration::from_secs(5 * 60);
const SUGGEST_SOFT_TTL: Duration = Duration::from_secs(60);
const SUGGEST_HARD_TTL: Duration = Duration::from_secs(10 * 60);
const FEATURED_GIFS_SOFT_TTL: Duration = Duration::from_secs(5 * 60);
const FEATURED_GIFS_HARD_TTL: Duration = Duration::from_secs(30 * 60);
const CATEGORIES_SOFT_TTL: Duration = Duration::from_secs(24 * 60 * 60);
const CATEGORIES_HARD_TTL: Duration = Duration::from_secs(48 * 60 * 60);
const RESOLVE_SOFT_TTL: Duration = Duration::from_secs(30 * 60);
const RESOLVE_HARD_TTL: Duration = Duration::from_secs(2 * 60 * 60);
#[derive(Clone)]
pub struct GifsShard {
inner: Arc<GifsShardInner>,
}
struct GifsShardInner {
klipy: KlipyClient,
gif_lists: Cache<String, Cached<Vec<GifItem>>>,
categories: Cache<String, Cached<Vec<GifCategoryTag>>>,
suggestions: Cache<String, Cached<Vec<String>>>,
resolved: Cache<String, Cached<Option<GifItem>>>,
refreshing: Mutex<HashSet<String>>,
}
#[derive(Debug, Clone)]
struct Cached<T> {
data: T,
stored_at: Instant,
}
#[derive(Debug, Clone, Copy)]
struct CachePolicy {
soft_ttl: Duration,
hard_ttl: Duration,
}
impl CachePolicy {
const fn new(soft_ttl: Duration, hard_ttl: Duration) -> Self {
Self { soft_ttl, hard_ttl }
}
}
impl<T> Cached<T> {
fn new(data: T) -> Self {
Self {
data,
stored_at: Instant::now(),
}
}
fn age(&self) -> Duration {
self.stored_at.elapsed()
}
}
impl GifsShard {
pub fn new(config: &ServiceConfig) -> anyhow::Result<Self> {
let media_proxy = MediaProxyUrlBuilder::from_env()?;
let klipy = KlipyClient::new(media_proxy)?;
let max_capacity = config.cache_max_entries;
let max_cache_ttl = CATEGORIES_HARD_TTL;
Ok(Self {
inner: Arc::new(GifsShardInner {
klipy,
gif_lists: build_cache(max_capacity, max_cache_ttl),
categories: build_cache(max_capacity, max_cache_ttl),
suggestions: build_cache(max_capacity, max_cache_ttl),
resolved: build_cache(max_capacity, max_cache_ttl),
refreshing: Mutex::new(HashSet::new()),
}),
})
}
async fn get_cached<T, Fetch, Fut>(
&self,
cache: Cache<String, Cached<T>>,
key: String,
policy: CachePolicy,
fetch: Fetch,
) -> anyhow::Result<T>
where
T: Clone + Send + Sync + 'static,
Fetch: Fn() -> Fut + Clone + Send + Sync + 'static,
Fut: Future<Output = anyhow::Result<T>> + Send + 'static,
{
if let Some(cached) = cache.get(&key).await {
let age = cached.age();
if age <= policy.soft_ttl {
return Ok(cached.data);
}
if age <= policy.hard_ttl {
self.trigger_background_refresh(cache, key, policy, fetch);
return Ok(cached.data);
}
cache.invalidate(&key).await;
}
let fetch_for_load = fetch.clone();
let cached = cache
.try_get_with(key, async move {
let data = fetch_for_load().await?;
Ok::<Cached<T>, anyhow::Error>(Cached::new(data))
})
.await
.map_err(|error| anyhow::anyhow!("{}", error.as_ref()))?;
Ok(cached.data)
}
fn trigger_background_refresh<T, Fetch, Fut>(
&self,
cache: Cache<String, Cached<T>>,
key: String,
_policy: CachePolicy,
fetch: Fetch,
) where
T: Clone + Send + Sync + 'static,
Fetch: Fn() -> Fut + Clone + Send + Sync + 'static,
Fut: Future<Output = anyhow::Result<T>> + Send + 'static,
{
let this = self.clone();
tokio::spawn(async move {
{
let mut refreshing = this.inner.refreshing.lock().await;
if !refreshing.insert(key.clone()) {
return;
}
}
let result = fetch().await;
match result {
Ok(data) => {
cache.insert(key.clone(), Cached::new(data)).await;
}
Err(error) => {
tracing::debug!(error = %error, cache_key = %key, "background GIF cache refresh failed");
}
}
let mut refreshing = this.inner.refreshing.lock().await;
refreshing.remove(&key);
});
}
async fn handle_available(&self, api_key: Option<String>) -> GifServiceResponse {
GifServiceResponse::Available {
available: api_key.as_deref().is_some_and(|key| !key.trim().is_empty()),
}
}
async fn handle_search(
&self,
api_key: String,
q: String,
locale: String,
country: String,
) -> anyhow::Result<GifServiceResponse> {
let key = format!("search:{locale}:{country}:{q}");
let this = self.clone();
let gifs = self
.get_cached(
self.inner.gif_lists.clone(),
key,
CachePolicy::new(SEARCH_SOFT_TTL, SEARCH_HARD_TTL),
move || {
let this = this.clone();
let api_key = api_key.clone();
let q = q.clone();
let locale = locale.clone();
let country = country.clone();
async move {
this.inner
.klipy
.search(&api_key, &q, &locale, &country, 50)
.await
}
},
)
.await?;
Ok(GifServiceResponse::SearchResults(gifs))
}
async fn handle_featured(
&self,
api_key: String,
locale: String,
country: String,
) -> anyhow::Result<GifServiceResponse> {
let gifs_key = format!("featured_gifs:{locale}:{country}");
let categories_key = format!("featured_categories:{locale}");
let gifs_this = self.clone();
let categories_this = self.clone();
let api_key_for_gifs = api_key.clone();
let locale_for_gifs = locale.clone();
let country_for_gifs = country.clone();
let api_key_for_categories = api_key;
let locale_for_categories = locale;
let gifs_future = self.get_cached(
self.inner.gif_lists.clone(),
gifs_key,
CachePolicy::new(FEATURED_GIFS_SOFT_TTL, FEATURED_GIFS_HARD_TTL),
move || {
let this = gifs_this.clone();
let api_key = api_key_for_gifs.clone();
let locale = locale_for_gifs.clone();
let country = country_for_gifs.clone();
async move {
this.inner
.klipy
.featured_gifs(&api_key, &locale, &country)
.await
}
},
);
let categories_future = self.get_cached(
self.inner.categories.clone(),
categories_key,
CachePolicy::new(CATEGORIES_SOFT_TTL, CATEGORIES_HARD_TTL),
move || {
let this = categories_this.clone();
let api_key = api_key_for_categories.clone();
let locale = locale_for_categories.clone();
async move {
this.inner
.klipy
.featured_categories(&api_key, &locale)
.await
}
},
);
let (gifs, categories) = tokio::try_join!(gifs_future, categories_future)?;
Ok(GifServiceResponse::Featured { gifs, categories })
}
async fn handle_trending(
&self,
api_key: String,
locale: String,
country: String,
) -> anyhow::Result<GifServiceResponse> {
let key = format!("trending:{locale}:{country}");
let this = self.clone();
let gifs = self
.get_cached(
self.inner.gif_lists.clone(),
key,
CachePolicy::new(FEATURED_GIFS_SOFT_TTL, FEATURED_GIFS_HARD_TTL),
move || {
let this = this.clone();
let api_key = api_key.clone();
let locale = locale.clone();
let country = country.clone();
async move {
this.inner
.klipy
.trending_gifs(&api_key, &locale, &country)
.await
}
},
)
.await?;
Ok(GifServiceResponse::TrendingResults(gifs))
}
async fn handle_suggest(
&self,
api_key: String,
q: String,
locale: String,
) -> anyhow::Result<GifServiceResponse> {
let key = format!("suggest:{locale}:{q}");
let this = self.clone();
let suggestions = self
.get_cached(
self.inner.suggestions.clone(),
key,
CachePolicy::new(SUGGEST_SOFT_TTL, SUGGEST_HARD_TTL),
move || {
let this = this.clone();
let api_key = api_key.clone();
let q = q.clone();
let locale = locale.clone();
async move { this.inner.klipy.suggestions(&api_key, &q, &locale).await }
},
)
.await?;
Ok(GifServiceResponse::Suggestions(suggestions))
}
async fn handle_resolve_by_url(
&self,
api_key: String,
url: String,
locale: String,
country: String,
) -> anyhow::Result<GifServiceResponse> {
let key = format!("resolve:{locale}:{country}:{url}");
let this = self.clone();
let gif = self
.get_cached(
self.inner.resolved.clone(),
key,
CachePolicy::new(RESOLVE_SOFT_TTL, RESOLVE_HARD_TTL),
move || {
let this = this.clone();
let api_key = api_key.clone();
let url = url.clone();
let locale = locale.clone();
let country = country.clone();
async move {
this.inner
.klipy
.resolve_by_url(&api_key, &url, &locale, &country)
.await
}
},
)
.await?;
Ok(GifServiceResponse::Resolved { gif })
}
}
impl ShardService for GifsShard {
type Request = GifRequest;
type Response = GifServiceResponse;
fn service_name(&self) -> &str {
"gifs"
}
async fn handle(&self, request: GifRequest) -> anyhow::Result<GifServiceResponse> {
let response = match request {
GifRequest::IsAvailable { api_key } => self.handle_available(api_key).await,
GifRequest::Search {
api_key,
q,
locale,
country,
} => self.handle_search(api_key, q, locale, country).await?,
GifRequest::GetFeatured {
api_key,
locale,
country,
} => self.handle_featured(api_key, locale, country).await?,
GifRequest::GetTrendingGifs {
api_key,
locale,
country,
} => self.handle_trending(api_key, locale, country).await?,
GifRequest::Suggest { api_key, q, locale } => {
self.handle_suggest(api_key, q, locale).await?
}
GifRequest::RegisterShare {
api_key,
id,
q,
locale,
country,
} => {
self.inner
.klipy
.register_share(&api_key, &id, &q, &locale, &country)
.await?;
GifServiceResponse::Registered
}
GifRequest::ResolveByUrl {
api_key,
url,
locale,
country,
} => {
self.handle_resolve_by_url(api_key, url, locale, country)
.await?
}
GifRequest::BuildShareUrl { slug } => GifServiceResponse::ShareUrl {
url: build_share_url(&slug),
},
GifRequest::ExtractSlugFromUrl { url } => GifServiceResponse::ExtractedSlug {
slug: extract_slug_from_url(&url),
},
};
Ok(response)
}
}
fn build_cache<T>(max_capacity: u64, time_to_live: Duration) -> Cache<String, Cached<T>>
where
T: Clone + Send + Sync + 'static,
{
Cache::builder()
.max_capacity(max_capacity)
.time_to_live(time_to_live)
.build()
}
+111
View File
@@ -0,0 +1,111 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "op", rename_all_fields = "snake_case")]
pub enum GifRequest {
IsAvailable {
api_key: Option<String>,
},
Search {
api_key: String,
q: String,
locale: String,
country: String,
},
GetFeatured {
api_key: String,
locale: String,
country: String,
},
GetTrendingGifs {
api_key: String,
locale: String,
country: String,
},
Suggest {
api_key: String,
q: String,
locale: String,
},
RegisterShare {
api_key: String,
id: String,
q: String,
locale: String,
country: String,
},
ResolveByUrl {
api_key: String,
url: String,
locale: String,
country: String,
},
BuildShareUrl {
slug: String,
},
ExtractSlugFromUrl {
url: String,
},
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum GifServiceResponse {
Available {
available: bool,
},
SearchResults(Vec<GifItem>),
Featured {
gifs: Vec<GifItem>,
categories: Vec<GifCategoryTag>,
},
TrendingResults(Vec<GifItem>),
Suggestions(Vec<String>),
Registered,
Resolved {
gif: Option<GifItem>,
},
ShareUrl {
url: String,
},
ExtractedSlug {
slug: Option<String>,
},
Failed {
message: String,
},
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GifMediaFormat {
pub src: String,
pub proxy_src: String,
pub width: i32,
pub height: i32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GifItem {
pub id: String,
pub slug: String,
pub provider: String,
pub title: String,
pub url: String,
pub src: String,
pub proxy_src: String,
pub width: i32,
pub height: i32,
pub media: BTreeMap<String, GifMediaFormat>,
#[serde(skip_serializing_if = "Option::is_none")]
pub placeholder: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GifCategoryTag {
pub name: String,
pub src: String,
pub proxy_src: String,
pub gif: Option<GifItem>,
}
-2
View File
@@ -13,11 +13,9 @@ entities = "1.0.1"
fluxer-svc = { path = "../fluxer_svc", default-features = false } fluxer-svc = { path = "../fluxer_svc", default-features = false }
hmac = "0.13.0" hmac = "0.13.0"
infer = "0.19.0" infer = "0.19.0"
mime_guess = "2.0.5"
moka = { version = "0.12.15", features = ["future", "sync"] } moka = { version = "0.12.15", features = ["future", "sync"] }
regex = "1.12" regex = "1.12"
reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls", "stream"] } reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls", "stream"] }
rmp-serde = "1.3"
scraper = "0.27" scraper = "0.27"
serde = { version = "1.0.228", features = ["derive", "rc"] } serde = { version = "1.0.228", features = ["derive", "rc"] }
serde_json = "1.0.150" serde_json = "1.0.150"
@@ -81,6 +81,7 @@ async fn resolve_html(ctx: &ResolveContext<'_>) -> anyhow::Result<ResolverResult
media_proxy: ctx.media_proxy, media_proxy: ctx.media_proxy,
static_cdn_endpoint: ctx.static_cdn_endpoint, static_cdn_endpoint: ctx.static_cdn_endpoint,
youtube_api_key: ctx.youtube_api_key.clone(), youtube_api_key: ctx.youtube_api_key.clone(),
klipy_api_key: ctx.klipy_api_key.clone(),
}; };
if let Some(embeds) = activity_pub::try_resolve( if let Some(embeds) = activity_pub::try_resolve(
+1
View File
@@ -1075,6 +1075,7 @@ mod tests {
media_proxy, media_proxy,
static_cdn_endpoint: "https://static.example.test", static_cdn_endpoint: "https://static.example.test",
youtube_api_key: None, youtube_api_key: None,
klipy_api_key: None,
} }
} }
+300 -132
View File
@@ -9,7 +9,12 @@ use std::pin::Pin;
use std::time::Duration; use std::time::Duration;
use url::Url; use url::Url;
const KLIPY_FLIGHT_CHUNK_MAX_BYTES: usize = 512 * 1024; const KLIPY_API_V1_BASE_URL: &str = "https://api.klipy.com/api/v1";
const KLIPY_API_MAX_BYTES: usize = 512 * 1024;
const KLIPY_API_TIMEOUT: Duration = Duration::from_secs(10);
const KLIPY_SIZE_PREFERENCE: &[&str] = &["hd", "md", "sm", "xs"];
const KLIPY_THUMBNAIL_FORMATS: &[&str] = &["webp", "gif"];
const KLIPY_VIDEO_FORMATS: &[&str] = &["webm", "mp4"];
pub struct KlipyResolver; pub struct KlipyResolver;
@@ -28,36 +33,13 @@ struct KlipyMediaFormats {
impl Resolver for KlipyResolver { impl Resolver for KlipyResolver {
fn matches(&self, url: &Url) -> bool { fn matches(&self, url: &Url) -> bool {
url.host_str() is_klipy_host(url)
.is_some_and(|h| h.eq_ignore_ascii_case("klipy.com"))
} }
fn transform_url(&self, url: &Url) -> Option<Url> { fn transform_url(&self, url: &Url) -> Option<Url> {
if !url let (kind, slug) = klipy_path(url)?;
.host_str() let resource = klipy_resource(&kind);
.is_some_and(|h| h.eq_ignore_ascii_case("klipy.com")) Url::parse(&format!("https://klipy.com/{resource}/{slug}/player")).ok()
{
return None;
}
let path = url.path();
static PATH_RE: std::sync::LazyLock<regex::Regex> = std::sync::LazyLock::new(|| {
regex::Regex::new(r"^/(gif|gifs|clip|clips)/([^/]+)").expect("valid regex")
});
let caps = PATH_RE.captures(path)?;
let kind = caps.get(1)?.as_str();
let slug = caps.get(2)?.as_str();
let normalized_kind = if kind.starts_with("clip") {
"clips"
} else {
"gifs"
};
Url::parse(&format!(
"https://klipy.com/{normalized_kind}/{slug}/player"
))
.ok()
} }
fn resolve<'a>( fn resolve<'a>(
@@ -65,26 +47,21 @@ impl Resolver for KlipyResolver {
ctx: &'a ResolveContext<'_>, ctx: &'a ResolveContext<'_>,
) -> Pin<Box<dyn Future<Output = anyhow::Result<ResolverResult>> + Send + 'a>> { ) -> Pin<Box<dyn Future<Output = anyhow::Result<ResolverResult>> + Send + 'a>> {
Box::pin(async move { Box::pin(async move {
let result = http_fetch::fetch_url( let Some(api_key) = ctx.klipy_api_key.clone().or_else(klipy_api_key) else {
&ctx.http_client,
ctx.url.as_str(),
http_fetch::DEFAULT_HTML_MAX_BYTES,
Duration::from_secs(10),
)
.await?;
if result.status != 200 {
return Ok(ResolverResult { embeds: vec![] }); return Ok(ResolverResult { embeds: vec![] });
} };
let formats = match resolve_media_via_api(ctx, &api_key).await {
let html = String::from_utf8_lossy(&result.bytes); Ok(formats) => formats,
let formats = match extract_klipy_media(&html) { Err(err) => {
Some(formats) => formats, tracing::warn!(error = %err, "KLIPY API resolution failed");
None => { None
return Ok(ResolverResult { embeds: vec![] });
} }
}; };
let Some(formats) = formats else {
return Ok(ResolverResult { embeds: vec![] });
};
let mut embed = MessageEmbed::new("gifv"); let mut embed = MessageEmbed::new("gifv");
embed.url = Some(ctx.original_url.to_string()); embed.url = Some(ctx.original_url.to_string());
embed.provider = Some(EmbedProvider { embed.provider = Some(EmbedProvider {
@@ -108,6 +85,12 @@ impl Resolver for KlipyResolver {
} }
} }
fn is_klipy_host(url: &Url) -> bool {
url.host_str().is_some_and(|h| {
h.eq_ignore_ascii_case("klipy.com") || h.eq_ignore_ascii_case("www.klipy.com")
})
}
async fn resolve_klipy_media( async fn resolve_klipy_media(
ctx: &ResolveContext<'_>, ctx: &ResolveContext<'_>,
format: &KlipyMediaFormat, format: &KlipyMediaFormat,
@@ -130,6 +113,129 @@ async fn resolve_klipy_media(
)) ))
} }
fn klipy_path(url: &Url) -> Option<(String, String)> {
if !is_klipy_host(url) {
return None;
}
static PATH_RE: std::sync::LazyLock<regex::Regex> = std::sync::LazyLock::new(|| {
regex::Regex::new(r"^/(gif|gifs|clip|clips)/([^/]+)").expect("valid regex")
});
let caps = PATH_RE.captures(url.path())?;
Some((
caps.get(1)?.as_str().to_owned(),
caps.get(2)?.as_str().to_owned(),
))
}
fn klipy_resource(kind: &str) -> &'static str {
if kind.starts_with("clip") {
"clips"
} else {
"gifs"
}
}
fn klipy_api_key() -> Option<String> {
std::env::var("FLUXER_KLIPY_API_KEY")
.ok()
.filter(|key| !key.is_empty())
.or_else(|| {
std::env::var("KLIPY_API_KEY")
.ok()
.filter(|key| !key.is_empty())
})
}
async fn resolve_media_via_api(
ctx: &ResolveContext<'_>,
api_key: &str,
) -> anyhow::Result<Option<KlipyMediaFormats>> {
let Some((kind, slug)) = klipy_path(&ctx.original_url) else {
return Ok(None);
};
let resource = klipy_resource(&kind);
let url = klipy_direct_url(api_key, resource, &slug)?;
let response = http_fetch::fetch_url(
&ctx.http_client,
url.as_str(),
KLIPY_API_MAX_BYTES,
KLIPY_API_TIMEOUT,
)
.await?;
if response.status != 200 {
tracing::warn!(
status = response.status,
"KLIPY direct API lookup returned non-200 status"
);
return Ok(None);
}
let payload: serde_json::Value = serde_json::from_slice(&response.bytes)?;
Ok(payload.get("data").and_then(extract_klipy_api_media))
}
fn klipy_direct_url(api_key: &str, resource: &str, slug: &str) -> anyhow::Result<Url> {
Ok(Url::parse(&format!(
"{KLIPY_API_V1_BASE_URL}/{api_key}/{resource}/{slug}"
))?)
}
fn extract_klipy_api_media(item: &serde_json::Value) -> Option<KlipyMediaFormats> {
let file = item.get("file");
let thumbnail = file.and_then(|file| pick_klipy_file_format(file, KLIPY_THUMBNAIL_FORMATS));
let video = file
.and_then(|file| pick_klipy_file_format(file, KLIPY_VIDEO_FORMATS))
.or_else(|| extract_klipy_fallback_webm(item.pointer("/media_formats/webm")));
if thumbnail.is_none() && video.is_none() {
return None;
}
Some(KlipyMediaFormats { thumbnail, video })
}
fn pick_klipy_file_format(file: &serde_json::Value, formats: &[&str]) -> Option<KlipyMediaFormat> {
for size in KLIPY_SIZE_PREFERENCE {
for media_format in formats {
if let Some(media) =
extract_media_format(file.pointer(&format!("/{size}/{media_format}")))
{
return Some(media);
}
}
}
for media_format in formats {
if let Some(media) = extract_media_format(file.get(*media_format)) {
return Some(media);
}
}
None
}
fn extract_klipy_fallback_webm(value: Option<&serde_json::Value>) -> Option<KlipyMediaFormat> {
let value = value?;
let url = value
.get("url")
.and_then(|v| v.as_str())
.filter(|url| !url.is_empty())?;
let dims = value.get("dims")?.as_array()?;
let width = dims
.first()
.and_then(|value| value.as_i64())
.filter(|value| *value > 0)
.and_then(|value| u32::try_from(value).ok());
let height = dims
.get(1)
.and_then(|value| value.as_i64())
.filter(|value| *value > 0)
.and_then(|value| u32::try_from(value).ok());
Some(KlipyMediaFormat {
url: Some(url.to_owned()),
width,
height,
})
}
fn resolve_relative_url(base_url: &Url, media_url: &str) -> Option<String> { fn resolve_relative_url(base_url: &Url, media_url: &str) -> Option<String> {
let url = base_url.join(media_url).ok()?; let url = base_url.join(media_url).ok()?;
if matches!(url.scheme(), "http" | "https") { if matches!(url.scheme(), "http" | "https") {
@@ -158,55 +264,21 @@ fn build_embed_media_payload(
} }
} }
fn extract_klipy_media(html: &str) -> Option<KlipyMediaFormats> {
static FLIGHT_RE: std::sync::LazyLock<regex::Regex> = std::sync::LazyLock::new(|| {
regex::Regex::new(r#"(?s)self\.__next_f\.push\(\[1,"(.*?)"\]\)"#).expect("valid regex")
});
for cap in FLIGHT_RE.captures_iter(html) {
let encoded = cap.get(1)?.as_str();
if encoded.len() > KLIPY_FLIGHT_CHUNK_MAX_BYTES {
continue;
}
if let Some(media) = parse_next_flight_data(encoded) {
return Some(media);
}
}
None
}
fn parse_next_flight_data(encoded: &str) -> Option<KlipyMediaFormats> {
let unescaped = serde_json::from_str::<String>(&format!("\"{encoded}\"")).ok()?;
let colon_idx = unescaped.find(':')?;
let json_str = &unescaped[colon_idx + 1..];
if json_str.len() > KLIPY_FLIGHT_CHUNK_MAX_BYTES {
return None;
}
let arr: Vec<serde_json::Value> = serde_json::from_str(json_str).ok()?;
for item in &arr {
if let Some(media) = item.get("media")
&& media.get("file").is_some()
{
return Some(KlipyMediaFormats {
thumbnail: extract_media_format(media.pointer("/file/hd/webp")),
video: extract_media_format(media.pointer("/file/hd/mp4")),
});
}
}
None
}
fn extract_media_format(value: Option<&serde_json::Value>) -> Option<KlipyMediaFormat> { fn extract_media_format(value: Option<&serde_json::Value>) -> Option<KlipyMediaFormat> {
let value = value?; let value = value?;
if let Some(url) = value.as_str().filter(|url| !url.is_empty()) {
return Some(KlipyMediaFormat {
url: Some(url.to_owned()),
width: None,
height: None,
});
}
let url = value
.get("url")
.and_then(|v| v.as_str())
.filter(|url| !url.is_empty())?;
Some(KlipyMediaFormat { Some(KlipyMediaFormat {
url: value url: Some(url.to_owned()),
.get("url")
.and_then(|v| v.as_str())
.map(|url| url.to_owned()),
width: value width: value
.get("width") .get("width")
.and_then(|v| v.as_u64()) .and_then(|v| v.as_u64())
@@ -222,35 +294,6 @@ fn extract_media_format(value: Option<&serde_json::Value>) -> Option<KlipyMediaF
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn parse_next_flight_data_extracts_media_formats() {
let encoded = r#"0:[{\"media\":{\"file\":{\"hd\":{\"webp\":{\"url\":\"https://img.example/a.webp\",\"width\":320,\"height\":180},\"mp4\":{\"url\":\"https://img.example/a.mp4\",\"width\":640,\"height\":360}}}}}]"#;
assert_eq!(
parse_next_flight_data(encoded),
Some(KlipyMediaFormats {
thumbnail: Some(KlipyMediaFormat {
url: Some("https://img.example/a.webp".to_owned()),
width: Some(320),
height: Some(180),
}),
video: Some(KlipyMediaFormat {
url: Some("https://img.example/a.mp4".to_owned()),
width: Some(640),
height: Some(360),
}),
})
);
}
#[test]
fn extract_klipy_media_skips_oversized_flight_chunks() {
let html = format!(
r#"self.__next_f.push([1,"{}"])"#,
"x".repeat(KLIPY_FLIGHT_CHUNK_MAX_BYTES + 1)
);
assert_eq!(extract_klipy_media(&html), None);
}
#[test] #[test]
fn resolve_relative_url_uses_original_url_like_ts() { fn resolve_relative_url_uses_original_url_like_ts() {
let base = Url::parse("https://klipy.com/gifs/funny").unwrap(); let base = Url::parse("https://klipy.com/gifs/funny").unwrap();
@@ -332,27 +375,152 @@ mod tests {
} }
#[test] #[test]
fn extract_klipy_media_from_realistic_html() { fn klipy_path_extracts_kind_and_slug() {
let html = r#" let (kind, slug) =
<script>self.__next_f.push([1,"0:[{\"media\":{\"file\":{\"hd\":{\"webp\":{\"url\":\"https://img.klipy.com/hd.webp\",\"width\":640,\"height\":360},\"mp4\":{\"url\":\"https://img.klipy.com/hd.mp4\",\"width\":1280,\"height\":720}}}}}]"])</script> klipy_path(&Url::parse("https://www.klipy.com/gifs/funny-cat-123").unwrap()).unwrap();
"#; assert_eq!(kind, "gifs");
let result = extract_klipy_media(html); assert_eq!(slug, "funny-cat-123");
assert!(result.is_some()); assert!(klipy_path(&Url::parse("https://klipy.com/about").unwrap()).is_none());
let formats = result.unwrap(); assert!(klipy_path(&Url::parse("https://notklipy.com/gifs/x").unwrap()).is_none());
}
#[test]
fn klipy_resource_maps_kind_to_api_segment() {
assert_eq!(klipy_resource("gif"), "gifs");
assert_eq!(klipy_resource("gifs"), "gifs");
assert_eq!(klipy_resource("clip"), "clips");
assert_eq!(klipy_resource("clips"), "clips");
}
#[test]
fn pick_klipy_file_format_prefers_hd_and_format_order() {
let file = serde_json::json!({
"hd": {
"webp": {"url": "https://img.klipy.com/hd.webp", "width": 254, "height": 450},
"webm": {"url": "https://img.klipy.com/hd.webm", "width": 254, "height": 450},
"mp4": {"url": "https://img.klipy.com/hd.mp4", "width": 254, "height": 450}
},
"sm": {
"webp": {"url": "https://img.klipy.com/sm.webp", "width": 165, "height": 294}
}
});
let thumbnail = pick_klipy_file_format(&file, KLIPY_THUMBNAIL_FORMATS).unwrap();
assert_eq!( assert_eq!(
formats.thumbnail.as_ref().unwrap().url.as_deref(), thumbnail.url.as_deref(),
Some("https://img.klipy.com/hd.webp") Some("https://img.klipy.com/hd.webp")
); );
assert_eq!(thumbnail.width, Some(254));
assert_eq!(thumbnail.height, Some(450));
assert_eq!( assert_eq!(
formats.video.as_ref().unwrap().url.as_deref(), pick_klipy_file_format(&file, KLIPY_VIDEO_FORMATS)
Some("https://img.klipy.com/hd.mp4") .unwrap()
.url
.as_deref(),
Some("https://img.klipy.com/hd.webm")
); );
} }
#[test] #[test]
fn extract_klipy_media_returns_none_for_non_media_chunks() { fn pick_klipy_file_format_handles_string_shape() {
let html = r#"<script>self.__next_f.push([1,"0:[{\"status\":\"ok\"}]"])</script>"#; let file = serde_json::json!({
assert!(extract_klipy_media(html).is_none()); "mp4": "https://img.klipy.com/c.mp4",
"gif": "https://img.klipy.com/c.gif",
"webp": "https://img.klipy.com/c.webp"
});
let thumbnail = pick_klipy_file_format(&file, KLIPY_THUMBNAIL_FORMATS).unwrap();
assert_eq!(
thumbnail.url.as_deref(),
Some("https://img.klipy.com/c.webp")
);
assert_eq!(thumbnail.width, None);
assert_eq!(
pick_klipy_file_format(&file, KLIPY_VIDEO_FORMATS)
.unwrap()
.url
.as_deref(),
Some("https://img.klipy.com/c.mp4")
);
}
#[test]
fn extract_klipy_api_media_uses_fallback_webm_shape() {
let item = serde_json::json!({
"media_formats": {
"webm": {
"url": "https://img.klipy.com/fallback.webm",
"dims": [320, 180]
}
}
});
let media = extract_klipy_api_media(&item).unwrap();
let video = media.video.unwrap();
assert_eq!(
video.url.as_deref(),
Some("https://img.klipy.com/fallback.webm")
);
assert_eq!(video.width, Some(320));
assert_eq!(video.height, Some(180));
assert!(media.thumbnail.is_none());
}
#[tokio::test]
#[ignore = "hits the live KLIPY API and local media proxy"]
async fn live_klipy_embed_resolves_real_media() {
let api_key = std::env::var("FLUXER_KLIPY_API_KEY").expect("FLUXER_KLIPY_API_KEY set");
let media_proxy_endpoint =
std::env::var("FLUXER_MEDIA_PROXY_ENDPOINT").expect("FLUXER_MEDIA_PROXY_ENDPOINT set");
let media_proxy_secret = std::env::var("FLUXER_MEDIA_PROXY_SECRET_KEY")
.expect("FLUXER_MEDIA_PROXY_SECRET_KEY set");
let media_proxy_public_endpoint = std::env::var("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT").ok();
let raw_url = std::env::var("FLUXER_KLIPY_LIVE_URL")
.unwrap_or_else(|_| "https://klipy.com/gifs/goatplaybanjo-chat-4".to_owned());
let resolver = KlipyResolver;
let original_url = Url::parse(&raw_url).expect("valid live KLIPY URL");
let url = resolver
.transform_url(&original_url)
.unwrap_or_else(|| original_url.clone());
let media_proxy = crate::media_proxy::MediaProxyClient::new_with_public_endpoint(
&media_proxy_endpoint,
&media_proxy_secret,
media_proxy_public_endpoint.as_deref(),
reqwest::Client::new(),
);
let ctx = ResolveContext {
url,
original_url: original_url.clone(),
http_client: reqwest::Client::new(),
nsfw_mode: crate::types::NsfwMode::Allow,
media_proxy: &media_proxy,
static_cdn_endpoint: "",
youtube_api_key: None,
klipy_api_key: Some(api_key),
};
let result = resolver.resolve(&ctx).await.expect("resolve KLIPY embed");
assert_eq!(result.embeds.len(), 1);
let embed = &result.embeds[0];
assert_eq!(embed.embed_type, "gifv");
assert_eq!(embed.url.as_deref(), Some(original_url.as_str()));
assert_eq!(
embed
.provider
.as_ref()
.and_then(|provider| provider.name.as_deref()),
Some("KLIPY")
);
let video = embed.video.as_ref().expect("video media resolved");
assert!(
video
.url
.as_deref()
.is_some_and(|url| url.starts_with("https://"))
);
assert!(video.width.is_some_and(|width| width > 0));
assert!(video.height.is_some_and(|height| height > 0));
assert!(video.content_type.as_deref().is_some_and(|content_type| {
content_type.starts_with("video/") || content_type == "image/gif"
}));
} }
#[test] #[test]
+1
View File
@@ -27,6 +27,7 @@ pub struct ResolveContext<'mp> {
pub media_proxy: &'mp MediaProxyClient, pub media_proxy: &'mp MediaProxyClient,
pub static_cdn_endpoint: &'mp str, pub static_cdn_endpoint: &'mp str,
pub youtube_api_key: Option<String>, pub youtube_api_key: Option<String>,
pub klipy_api_key: Option<String>,
} }
impl ResolveContext<'_> { impl ResolveContext<'_> {
+1
View File
@@ -125,6 +125,7 @@ mod tests {
bypass_cache: false, bypass_cache: false,
cache_only, cache_only,
youtube_api_key: None, youtube_api_key: None,
klipy_api_key: None,
} }
} }
+9 -1
View File
@@ -91,6 +91,7 @@ impl UnfurlShard {
url_str: &str, url_str: &str,
nsfw_mode: NsfwMode, nsfw_mode: NsfwMode,
youtube_api_key: Option<&str>, youtube_api_key: Option<&str>,
klipy_api_key: Option<&str>,
) -> anyhow::Result<UnfurlResult> { ) -> anyhow::Result<UnfurlResult> {
let parsed = Url::parse(url_str)?; let parsed = Url::parse(url_str)?;
@@ -104,6 +105,7 @@ impl UnfurlShard {
media_proxy: &self.media_proxy, media_proxy: &self.media_proxy,
static_cdn_endpoint: &self.static_cdn_endpoint, static_cdn_endpoint: &self.static_cdn_endpoint,
youtube_api_key: youtube_api_key.map(str::to_owned), youtube_api_key: youtube_api_key.map(str::to_owned),
klipy_api_key: klipy_api_key.map(str::to_owned),
}; };
if let Some(idx) = matched_resolver_idx { if let Some(idx) = matched_resolver_idx {
@@ -204,6 +206,7 @@ impl ShardService for UnfurlShard {
bypass_cache, bypass_cache,
cache_only, cache_only,
ref youtube_api_key, ref youtube_api_key,
ref klipy_api_key,
} => { } => {
let nsfw = nsfw_mode.unwrap_or_default(); let nsfw = nsfw_mode.unwrap_or_default();
let cache_key = unfurl_cache_key(url, nsfw); let cache_key = unfurl_cache_key(url, nsfw);
@@ -219,7 +222,12 @@ impl ShardService for UnfurlShard {
} }
let result = match self let result = match self
.resolve_url(url, nsfw, youtube_api_key.as_deref()) .resolve_url(
url,
nsfw,
youtube_api_key.as_deref(),
klipy_api_key.as_deref(),
)
.await .await
{ {
Ok(r) => Arc::new(r), Ok(r) => Arc::new(r),
+4
View File
@@ -15,6 +15,8 @@ pub enum UnfurlRequest {
cache_only: bool, cache_only: bool,
#[serde(default)] #[serde(default)]
youtube_api_key: Option<String>, youtube_api_key: Option<String>,
#[serde(default)]
klipy_api_key: Option<String>,
}, },
Invalidate { Invalidate {
url: String, url: String,
@@ -203,12 +205,14 @@ mod tests {
bypass_cache, bypass_cache,
cache_only, cache_only,
youtube_api_key, youtube_api_key,
klipy_api_key,
} => { } => {
assert_eq!(url, "https://fxtwitter.com/example/status/1"); assert_eq!(url, "https://fxtwitter.com/example/status/1");
assert_eq!(nsfw_mode, Some(NsfwMode::Block)); assert_eq!(nsfw_mode, Some(NsfwMode::Block));
assert!(!bypass_cache); assert!(!bypass_cache);
assert!(!cache_only); assert!(!cache_only);
assert!(youtube_api_key.is_none()); assert!(youtube_api_key.is_none());
assert!(klipy_api_key.is_none());
} }
UnfurlRequest::Invalidate { .. } => panic!("expected unfurl request"), UnfurlRequest::Invalidate { .. } => panic!("expected unfurl request"),
} }
-7
View File
@@ -219,15 +219,9 @@ function defaultConfig(): MasterConfig {
port: 3310, port: 3310,
fail_open: false, fail_open: false,
}, },
gif: {
provider: 'tenor',
},
klipy: { klipy: {
api_key: '', api_key: '',
}, },
tenor: {
api_key: '',
},
youtube: { youtube: {
api_key: '', api_key: '',
}, },
@@ -399,7 +393,6 @@ function normalizeConfig(config: MasterConfig): MasterConfig {
assertOneOf(config.integrations.email.provider, ['smtp', 'none'], 'FLUXER_EMAIL_PROVIDER'); assertOneOf(config.integrations.email.provider, ['smtp', 'none'], 'FLUXER_EMAIL_PROVIDER');
assertOneOf(config.integrations.captcha.provider, ['hcaptcha', 'turnstile', 'none'], 'FLUXER_CAPTCHA_PROVIDER'); assertOneOf(config.integrations.captcha.provider, ['hcaptcha', 'turnstile', 'none'], 'FLUXER_CAPTCHA_PROVIDER');
assertOneOf(config.integrations.search.engine, ['elasticsearch', 'meilisearch'], 'FLUXER_SEARCH_ENGINE'); assertOneOf(config.integrations.search.engine, ['elasticsearch', 'meilisearch'], 'FLUXER_SEARCH_ENGINE');
assertOneOf(config.integrations.gif.provider, ['tenor', 'klipy'], 'FLUXER_GIF_PROVIDER');
assertOneOf( assertOneOf(
config.instance.abuse_policy.direct_contact_spam.action, config.instance.abuse_policy.direct_contact_spam.action,
['flag_spammer', 'suppress_delivery'], ['flag_spammer', 'suppress_delivery'],
-6
View File
@@ -259,15 +259,9 @@ export interface MasterConfig {
port: number; port: number;
fail_open: boolean; fail_open: boolean;
}; };
gif: {
provider: 'tenor' | 'klipy';
};
klipy: { klipy: {
api_key: string; api_key: string;
}; };
tenor: {
api_key: string;
};
youtube: { youtube: {
api_key: string; api_key: string;
}; };
@@ -278,9 +278,7 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
FLUXER_CLAMAV_HOST: {path: ['integrations', 'clamav', 'host']}, FLUXER_CLAMAV_HOST: {path: ['integrations', 'clamav', 'host']},
FLUXER_CLAMAV_PORT: {path: ['integrations', 'clamav', 'port'], parse: parseEnvValue}, FLUXER_CLAMAV_PORT: {path: ['integrations', 'clamav', 'port'], parse: parseEnvValue},
FLUXER_CLAMAV_FAIL_OPEN: {path: ['integrations', 'clamav', 'fail_open'], parse: parseEnvValue}, FLUXER_CLAMAV_FAIL_OPEN: {path: ['integrations', 'clamav', 'fail_open'], parse: parseEnvValue},
FLUXER_GIF_PROVIDER: {path: ['integrations', 'gif', 'provider']},
FLUXER_KLIPY_API_KEY: {path: ['integrations', 'klipy', 'api_key']}, FLUXER_KLIPY_API_KEY: {path: ['integrations', 'klipy', 'api_key']},
FLUXER_TENOR_API_KEY: {path: ['integrations', 'tenor', 'api_key']},
FLUXER_YOUTUBE_API_KEY: {path: ['integrations', 'youtube', 'api_key']}, FLUXER_YOUTUBE_API_KEY: {path: ['integrations', 'youtube', 'api_key']},
FLUXER_BUNNY_PURGE_ENABLED: {path: ['integrations', 'bunny', 'purge_enabled'], parse: parseEnvValue}, FLUXER_BUNNY_PURGE_ENABLED: {path: ['integrations', 'bunny', 'purge_enabled'], parse: parseEnvValue},
FLUXER_BUNNY_API_KEY: {path: ['integrations', 'bunny', 'api_key']}, FLUXER_BUNNY_API_KEY: {path: ['integrations', 'bunny', 'api_key']},
@@ -21,13 +21,11 @@ describe('extractClientIp', () => {
it('prioritises configured header', () => { it('prioritises configured header', () => {
const request = new Request('http://example.com', { const request = new Request('http://example.com', {
headers: { headers: {
'Cf-Connecting-Ip': '203.0.113.40', 'X-Real-Ip': '203.0.113.40',
'X-Forwarded-For': '203.0.113.60', 'X-Forwarded-For': '203.0.113.60',
}, },
}); });
expect(extractClientIp(request, {trustClientIpHeader: true, clientIpHeaderName: 'cf-connecting-ip'})).toBe( expect(extractClientIp(request, {trustClientIpHeader: true, clientIpHeaderName: 'x-real-ip'})).toBe('203.0.113.40');
'203.0.113.40',
);
}); });
it('uses default header name (x-forwarded-for) when clientIpHeaderName is not specified', () => { it('uses default header name (x-forwarded-for) when clientIpHeaderName is not specified', () => {
const request = new Request('http://example.com', { const request = new Request('http://example.com', {
@@ -39,7 +37,7 @@ describe('extractClientIp', () => {
const request = new Request('http://example.com', { const request = new Request('http://example.com', {
headers: {'X-Forwarded-For': '192.168.1.3'}, headers: {'X-Forwarded-For': '192.168.1.3'},
}); });
expect(extractClientIp(request, {trustClientIpHeader: true, clientIpHeaderName: 'cf-connecting-ip'})).toBeNull(); expect(extractClientIp(request, {trustClientIpHeader: true, clientIpHeaderName: 'x-real-ip'})).toBeNull();
}); });
it('extracts first hop from x-forwarded-for', () => { it('extracts first hop from x-forwarded-for', () => {
const request = new Request('http://example.com', { const request = new Request('http://example.com', {
@@ -108,7 +106,7 @@ describe('resolveClientIpHeaderName', () => {
expect(resolveClientIpHeaderName()).toBe('x-forwarded-for'); expect(resolveClientIpHeaderName()).toBe('x-forwarded-for');
}); });
it('returns the configured header name normalised to lowercase', () => { it('returns the configured header name normalised to lowercase', () => {
expect(resolveClientIpHeaderName('cf-connecting-ip')).toBe('cf-connecting-ip'); expect(resolveClientIpHeaderName('x-client-ip')).toBe('x-client-ip');
expect(resolveClientIpHeaderName('X-Real-Ip')).toBe('x-real-ip'); expect(resolveClientIpHeaderName('X-Real-Ip')).toBe('x-real-ip');
}); });
}); });
@@ -526,7 +526,6 @@ const InstancePolicyResponse = z.object({
}), }),
}); });
const GifProviderSchema = z.enum(['tenor', 'klipy']);
const CaptchaProviderSchema = z.enum(['hcaptcha', 'turnstile', 'none']); const CaptchaProviderSchema = z.enum(['hcaptcha', 'turnstile', 'none']);
const EmailProviderSchema = z.enum(['smtp', 'none']); const EmailProviderSchema = z.enum(['smtp', 'none']);
@@ -559,9 +558,6 @@ const InstanceMediaResponse = z.object({
const InstanceIntegrationsResponse = z.object({ const InstanceIntegrationsResponse = z.object({
gif: z.object({ gif: z.object({
provider: GifProviderSchema.nullable(),
effective_provider: GifProviderSchema,
tenor_api_key_set: z.boolean(),
klipy_api_key_set: z.boolean(), klipy_api_key_set: z.boolean(),
effective_available: z.boolean(), effective_available: z.boolean(),
}), }),
@@ -648,8 +644,6 @@ export const InstanceConfigUpdateRequest = z.object({
.object({ .object({
gif: z gif: z
.object({ .object({
provider: GifProviderSchema.nullish(),
tenor_api_key: z.string().trim().max(4096).nullish(),
klipy_api_key: z.string().trim().max(4096).nullish(), klipy_api_key: z.string().trim().max(4096).nullish(),
}) })
.nullish(), .nullish(),
@@ -9,7 +9,7 @@ export const GIF_PROVIDER_DISPLAY_NAME_HEADER = 'X-Fluxer-GIF-Provider-Display-N
export const GIF_PROVIDER_ATTRIBUTION_HEADER = 'X-Fluxer-GIF-Provider-Attribution-Required'; export const GIF_PROVIDER_ATTRIBUTION_HEADER = 'X-Fluxer-GIF-Provider-Attribution-Required';
const LocaleType = LocaleSchema.default('en-US').transform((v) => v.replace('-', '_')); const LocaleType = LocaleSchema.default('en-US').transform((v) => v.replace('-', '_'));
const GifProviderName = createStringType(1, 32).describe( const GifProviderName = createStringType(1, 32).describe(
'Identifier of the active GIF provider (e.g. "klipy", "tenor"). Vendor names are opaque to clients.', 'Identifier of the active GIF provider. KLIPY is currently the only supported provider.',
); );
export const GifSearchQuery = z.object({ export const GifSearchQuery = z.object({
@@ -69,10 +69,10 @@ export type GifResponse = z.infer<typeof GifResponse>;
export const GifCategoryTagResponse = z.object({ export const GifCategoryTagResponse = z.object({
name: z.string().describe('Category search term (locale-translated label suitable for display).'), name: z.string().describe('Category search term (locale-translated label suitable for display).'),
src: z.string().describe('URL to the category preview image (legacy / fallback).'), src: z.string().describe('Category preview image URL from the top GIF for this category search term.'),
proxy_src: z.string().describe('Proxied URL to the category preview image (legacy / fallback).'), proxy_src: z.string().describe('Proxied category preview image URL from the top GIF for this category search term.'),
gif: GifResponse.nullable().describe( gif: GifResponse.nullable().describe(
'Full enriched GIF for the category (first search result for `name`). Null until the background cache is populated for this locale; fall back to `src` / `proxy_src` in that case.', 'Enriched category preview GIF from the top search result for this category. Null only when no preview GIF was available.',
), ),
}); });
@@ -91,7 +91,7 @@ export const WellKnownFluxerResponse = z.object({
.describe('Feature flags for this instance'), .describe('Feature flags for this instance'),
gif: z gif: z
.object({ .object({
provider: z.string().describe('Stable machine name of the active GIF provider (e.g. "klipy", "tenor")'), provider: z.string().describe('Stable machine name of the active GIF provider.'),
display_name: z.string().describe('Human-readable provider name shown in the UI'), display_name: z.string().describe('Human-readable provider name shown in the UI'),
attribution_required: z attribution_required: z
.boolean() .boolean()
@@ -30,7 +30,7 @@ export const CreateFavoriteMemeFromUrlBodySchema = FavoriteMemeBase.extend({
.describe('Provider-issued slug or slug-id token for the GIF, when sourced from a provider'), .describe('Provider-issued slug or slug-id token for the GIF, when sourced from a provider'),
gif_provider: createStringType(1, 32) gif_provider: createStringType(1, 32)
.nullish() .nullish()
.describe('Stable name of the GIF provider that issued gif_slug (e.g. "klipy", "tenor")'), .describe('Stable name of the GIF provider that issued gif_slug. New provider GIFs are sourced from KLIPY.'),
media: z media: z
.record(z.string(), GifMediaFormat) .record(z.string(), GifMediaFormat)
.nullish() .nullish()
@@ -77,7 +77,9 @@ export const FavoriteMemeResponse = z.object({
gif_provider: z gif_provider: z
.string() .string()
.nullish() .nullish()
.describe('Stable name of the GIF provider that issued gif_slug (e.g. "klipy", "tenor"), if any'), .describe(
'Stable name of the GIF provider that issued gif_slug, if any. Legacy records may contain older provider names.',
),
media: z media: z
.record(z.string(), GifMediaFormat) .record(z.string(), GifMediaFormat)
.nullish() .nullish()
+6
View File
@@ -189,6 +189,12 @@ pub struct RustServiceSpec {
pub fn rust_services() -> Vec<RustServiceSpec> { pub fn rust_services() -> Vec<RustServiceSpec> {
vec![ vec![
RustServiceSpec {
name: "gifs",
package: "fluxer-gifs",
path: ROOT.join("fluxer_gifs"),
port_base: 8110,
},
RustServiceSpec { RustServiceSpec {
name: "messages", name: "messages",
package: "fluxer-messages", package: "fluxer-messages",