mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-07 19:22:14 +09:00
fix: migrate gifs to klipy service (#1220)
This commit is contained in:
@@ -0,0 +1,32 @@
|
|||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
name: build gifs
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
build-version:
|
||||||
|
description: "Explicit Fluxer CalVer build version (YYYY.MDD.MICRO, UTC HHMMSS without leading zeroes) to use instead of automatic UTC clock allocation"
|
||||||
|
type: string
|
||||||
|
required: false
|
||||||
|
default: ""
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
build-version:
|
||||||
|
description: "Explicit Fluxer CalVer build version (YYYY.MDD.MICRO, UTC HHMMSS without leading zeroes) to use instead of automatic UTC clock allocation"
|
||||||
|
type: string
|
||||||
|
required: false
|
||||||
|
default: ""
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
actions: read
|
||||||
|
contents: read
|
||||||
|
packages: write
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
image:
|
||||||
|
uses: ./.github/workflows/_build-image.yaml
|
||||||
|
with:
|
||||||
|
image: fluxer-gifs
|
||||||
|
dockerfile: fluxer_gifs/Dockerfile
|
||||||
|
build-version: ${{ inputs.build-version }}
|
||||||
|
secrets: inherit
|
||||||
@@ -309,7 +309,7 @@ jobs:
|
|||||||
*worker*)
|
*worker*)
|
||||||
EXTRA_SETS+=(--set-string "workerRealtime.tag=${HELM_TEST_BUILD_VERSION}" --set-string "workerUnfurl.tag=${HELM_TEST_BUILD_VERSION}" --set-string "workerLifecycle.tag=${HELM_TEST_BUILD_VERSION}" --set-string "workerBatch.tag=${HELM_TEST_BUILD_VERSION}" --set-string "workerRealtime.build.version=${HELM_TEST_BUILD_VERSION}" --set-string "workerUnfurl.build.version=${HELM_TEST_BUILD_VERSION}" --set-string "workerLifecycle.build.version=${HELM_TEST_BUILD_VERSION}" --set-string "workerBatch.build.version=${HELM_TEST_BUILD_VERSION}")
|
EXTRA_SETS+=(--set-string "workerRealtime.tag=${HELM_TEST_BUILD_VERSION}" --set-string "workerUnfurl.tag=${HELM_TEST_BUILD_VERSION}" --set-string "workerLifecycle.tag=${HELM_TEST_BUILD_VERSION}" --set-string "workerBatch.tag=${HELM_TEST_BUILD_VERSION}" --set-string "workerRealtime.build.version=${HELM_TEST_BUILD_VERSION}" --set-string "workerUnfurl.build.version=${HELM_TEST_BUILD_VERSION}" --set-string "workerLifecycle.build.version=${HELM_TEST_BUILD_VERSION}" --set-string "workerBatch.build.version=${HELM_TEST_BUILD_VERSION}")
|
||||||
;;
|
;;
|
||||||
*member-lists*|*messages*|*presence*|*snowflakes*|*unfurl*|*users*|*voice-states*)
|
*gifs*|*member-lists*|*messages*|*presence*|*snowflakes*|*unfurl*|*users*|*voice-states*)
|
||||||
EXTRA_SETS+=(--set-string "svc.tag=${HELM_TEST_BUILD_VERSION}" --set-string "svc.build.version=${HELM_TEST_BUILD_VERSION}" --set-string svc.build.channel=stable)
|
EXTRA_SETS+=(--set-string "svc.tag=${HELM_TEST_BUILD_VERSION}" --set-string "svc.build.version=${HELM_TEST_BUILD_VERSION}" --set-string svc.build.channel=stable)
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|||||||
Generated
+19
-2
@@ -1953,6 +1953,25 @@ dependencies = [
|
|||||||
"urlencoding",
|
"urlencoding",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fluxer-gifs"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"base64",
|
||||||
|
"fluxer-svc",
|
||||||
|
"hmac 0.13.0",
|
||||||
|
"moka",
|
||||||
|
"reqwest",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"sha2 0.11.0",
|
||||||
|
"tokio",
|
||||||
|
"tracing",
|
||||||
|
"url",
|
||||||
|
"urlencoding",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fluxer-i18n-auto"
|
name = "fluxer-i18n-auto"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
@@ -2097,11 +2116,9 @@ dependencies = [
|
|||||||
"fluxer-svc",
|
"fluxer-svc",
|
||||||
"hmac 0.13.0",
|
"hmac 0.13.0",
|
||||||
"infer",
|
"infer",
|
||||||
"mime_guess",
|
|
||||||
"moka",
|
"moka",
|
||||||
"regex",
|
"regex",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"rmp-serde",
|
|
||||||
"scraper",
|
"scraper",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ members = [
|
|||||||
"fluxer_common",
|
"fluxer_common",
|
||||||
"fluxer_marketing",
|
"fluxer_marketing",
|
||||||
"fluxer_media_proxy",
|
"fluxer_media_proxy",
|
||||||
|
"fluxer_gifs",
|
||||||
"fluxer_svc",
|
"fluxer_svc",
|
||||||
"fluxer_messages",
|
"fluxer_messages",
|
||||||
"fluxer_snowflakes",
|
"fluxer_snowflakes",
|
||||||
|
|||||||
Vendored
-1
@@ -132,7 +132,6 @@ FLUXER_SEARCH_TLS_REJECT_UNAUTHORIZED=false
|
|||||||
FLUXER_STRIPE_ENABLED=false
|
FLUXER_STRIPE_ENABLED=false
|
||||||
FLUXER_NCMEC_ENABLED=false
|
FLUXER_NCMEC_ENABLED=false
|
||||||
FLUXER_CLAMAV_ENABLED=false
|
FLUXER_CLAMAV_ENABLED=false
|
||||||
FLUXER_GIF_PROVIDER=tenor
|
|
||||||
FLUXER_DISCOVERY_ENABLED=true
|
FLUXER_DISCOVERY_ENABLED=true
|
||||||
FLUXER_SELF_HOSTED=true
|
FLUXER_SELF_HOSTED=true
|
||||||
FLUXER_DISABLE_RATE_LIMITS=true
|
FLUXER_DISABLE_RATE_LIMITS=true
|
||||||
|
|||||||
@@ -0,0 +1,9 @@
|
|||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
apiVersion: v2
|
||||||
|
name: gifs
|
||||||
|
version: 0.1.0
|
||||||
|
dependencies:
|
||||||
|
- name: svc-common
|
||||||
|
version: 0.1.0
|
||||||
|
repository: file://../svc-common
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
{{ include "svc-common.statefulset" . }}
|
||||||
|
---
|
||||||
|
{{ include "svc-common.deployment" . }}
|
||||||
|
---
|
||||||
|
{{ include "svc-common.headless-service" . }}
|
||||||
|
---
|
||||||
|
{{ include "svc-common.service" . }}
|
||||||
|
---
|
||||||
|
{{ include "svc-common.pdb" . }}
|
||||||
|
---
|
||||||
|
{{ include "svc-common.router-pdb" . }}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
svc:
|
||||||
|
shard:
|
||||||
|
replicas: 4
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 512Mi
|
||||||
|
limits:
|
||||||
|
memory: 1Gi
|
||||||
|
router:
|
||||||
|
replicas: 3
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 256Mi
|
||||||
|
limits:
|
||||||
|
memory: 512Mi
|
||||||
|
cache:
|
||||||
|
maxEntries: 500000
|
||||||
|
ttlMs: '30000'
|
||||||
|
extraEnv:
|
||||||
|
- name: FLUXER_MEDIA_PROXY_ENDPOINT
|
||||||
|
value: http://media-proxy:8080
|
||||||
|
- name: FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT
|
||||||
|
value: https://fluxerusercontent.com
|
||||||
|
- name: FLUXER_MEDIA_PROXY_SECRET_KEY
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: fluxer-media-proxy-v2-env
|
||||||
|
key: FLUXER_MEDIA_PROXY_SECRET_KEY
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
global:
|
||||||
|
namespace: fluxer
|
||||||
|
imagePullSecret: ghcr-pull-secret
|
||||||
|
registry: ""
|
||||||
|
|
||||||
|
svc:
|
||||||
|
name: gifs
|
||||||
|
image: fluxer-gifs
|
||||||
|
tag: ''
|
||||||
|
shard:
|
||||||
|
replicas: 2
|
||||||
|
port: 8090
|
||||||
|
minReadySeconds: 10
|
||||||
|
terminationGracePeriodSeconds: 60
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 256Mi
|
||||||
|
limits:
|
||||||
|
memory: 512Mi
|
||||||
|
router:
|
||||||
|
replicas: 2
|
||||||
|
port: 8090
|
||||||
|
minReadySeconds: 10
|
||||||
|
maxSurge: 1
|
||||||
|
maxUnavailable: 0
|
||||||
|
terminationGracePeriodSeconds: 60
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 128Mi
|
||||||
|
limits:
|
||||||
|
memory: 256Mi
|
||||||
|
nats:
|
||||||
|
url: nats://nats-core:4222
|
||||||
|
cache:
|
||||||
|
maxEntries: 250000
|
||||||
|
ttlMs: '30000'
|
||||||
|
extraEnv:
|
||||||
|
- name: FLUXER_MEDIA_PROXY_ENDPOINT
|
||||||
|
value: http://media-proxy:8080
|
||||||
|
- name: FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT
|
||||||
|
value: https://fluxerusercontent.com
|
||||||
|
- name: FLUXER_MEDIA_PROXY_SECRET_KEY
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: fluxer-media-proxy-v2-env
|
||||||
|
key: FLUXER_MEDIA_PROXY_SECRET_KEY
|
||||||
|
nodeSelector: {}
|
||||||
|
tolerations: []
|
||||||
|
|
||||||
|
pdb:
|
||||||
|
minAvailable: '50%'
|
||||||
@@ -27,6 +27,8 @@ [email protected]
|
|||||||
LIVEKIT_API_KEY=fluxer
|
LIVEKIT_API_KEY=fluxer
|
||||||
LIVEKIT_API_SECRET=CHANGE_ME
|
LIVEKIT_API_SECRET=CHANGE_ME
|
||||||
|
|
||||||
|
FLUXER_KLIPY_API_KEY=
|
||||||
|
|
||||||
FLUXER_EMAIL_ENABLED=false
|
FLUXER_EMAIL_ENABLED=false
|
||||||
FLUXER_EMAIL_PROVIDER=none
|
FLUXER_EMAIL_PROVIDER=none
|
||||||
FLUXER_EMAIL_FROM_EMAIL=[email protected]
|
FLUXER_EMAIL_FROM_EMAIL=[email protected]
|
||||||
|
|||||||
@@ -49,6 +49,8 @@ x-fluxer-env: &fluxer-env
|
|||||||
FLUXER_LIVEKIT_API_SECRET: ${LIVEKIT_API_SECRET:?set LIVEKIT_API_SECRET in .env}
|
FLUXER_LIVEKIT_API_SECRET: ${LIVEKIT_API_SECRET:?set LIVEKIT_API_SECRET in .env}
|
||||||
FLUXER_LIVEKIT_WEBHOOK_URL: http://api:8080/webhooks/livekit
|
FLUXER_LIVEKIT_WEBHOOK_URL: http://api:8080/webhooks/livekit
|
||||||
|
|
||||||
|
FLUXER_KLIPY_API_KEY: ${FLUXER_KLIPY_API_KEY:-}
|
||||||
|
|
||||||
FLUXER_EMAIL_ENABLED: ${FLUXER_EMAIL_ENABLED:-false}
|
FLUXER_EMAIL_ENABLED: ${FLUXER_EMAIL_ENABLED:-false}
|
||||||
FLUXER_EMAIL_PROVIDER: ${FLUXER_EMAIL_PROVIDER:-none}
|
FLUXER_EMAIL_PROVIDER: ${FLUXER_EMAIL_PROVIDER:-none}
|
||||||
FLUXER_EMAIL_FROM_EMAIL: ${FLUXER_EMAIL_FROM_EMAIL:-noreply@localhost}
|
FLUXER_EMAIL_FROM_EMAIL: ${FLUXER_EMAIL_FROM_EMAIL:-noreply@localhost}
|
||||||
@@ -210,6 +212,8 @@ services:
|
|||||||
nats: {condition: service_started}
|
nats: {condition: service_started}
|
||||||
meilisearch: {condition: service_started}
|
meilisearch: {condition: service_started}
|
||||||
seaweedfs-init: {condition: service_completed_successfully}
|
seaweedfs-init: {condition: service_completed_successfully}
|
||||||
|
gifs: {condition: service_started}
|
||||||
|
gifs-shard: {condition: service_started}
|
||||||
snowflakes: {condition: service_started}
|
snowflakes: {condition: service_started}
|
||||||
snowflakes-shard: {condition: service_started}
|
snowflakes-shard: {condition: service_started}
|
||||||
messages: {condition: service_started}
|
messages: {condition: service_started}
|
||||||
@@ -319,6 +323,29 @@ services:
|
|||||||
depends_on:
|
depends_on:
|
||||||
nats: {condition: service_started}
|
nats: {condition: service_started}
|
||||||
|
|
||||||
|
gifs:
|
||||||
|
<<: *fluxer-service
|
||||||
|
image: ${FLUXER_REGISTRY:-ghcr.io/${FLUXER_REGISTRY_OWNER:-fluxerapp}}/fluxer-gifs:${FLUXER_IMAGE_TAG:-v1}
|
||||||
|
environment:
|
||||||
|
<<: *fluxer-env
|
||||||
|
FLUXER_SVC_NAME: gifs
|
||||||
|
FLUXER_SVC_MODE: router
|
||||||
|
FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}/media
|
||||||
|
depends_on:
|
||||||
|
nats: {condition: service_started}
|
||||||
|
|
||||||
|
gifs-shard:
|
||||||
|
<<: *fluxer-service
|
||||||
|
image: ${FLUXER_REGISTRY:-ghcr.io/${FLUXER_REGISTRY_OWNER:-fluxerapp}}/fluxer-gifs:${FLUXER_IMAGE_TAG:-v1}
|
||||||
|
environment:
|
||||||
|
<<: *fluxer-env
|
||||||
|
FLUXER_SVC_NAME: gifs
|
||||||
|
FLUXER_SVC_MODE: shard
|
||||||
|
FLUXER_SVC_SHARD_ID: "0"
|
||||||
|
FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}/media
|
||||||
|
depends_on:
|
||||||
|
nats: {condition: service_started}
|
||||||
|
|
||||||
messages:
|
messages:
|
||||||
<<: *fluxer-service
|
<<: *fluxer-service
|
||||||
image: ${FLUXER_REGISTRY:-ghcr.io/${FLUXER_REGISTRY_OWNER:-fluxerapp}}/fluxer-messages:${FLUXER_IMAGE_TAG:-v1}
|
image: ${FLUXER_REGISTRY:-ghcr.io/${FLUXER_REGISTRY_OWNER:-fluxerapp}}/fluxer-messages:${FLUXER_IMAGE_TAG:-v1}
|
||||||
|
|||||||
@@ -12594,20 +12594,8 @@
|
|||||||
"properties": {
|
"properties": {
|
||||||
"gif": {
|
"gif": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {"klipy_api_key_set": {"type": "boolean"}, "effective_available": {"type": "boolean"}},
|
||||||
"provider": {"nullable": true, "enum": ["tenor", "klipy"], "type": "string"},
|
"required": ["klipy_api_key_set", "effective_available"]
|
||||||
"effective_provider": {"enum": ["tenor", "klipy"], "type": "string"},
|
|
||||||
"tenor_api_key_set": {"type": "boolean"},
|
|
||||||
"klipy_api_key_set": {"type": "boolean"},
|
|
||||||
"effective_available": {"type": "boolean"}
|
|
||||||
},
|
|
||||||
"required": [
|
|
||||||
"provider",
|
|
||||||
"effective_provider",
|
|
||||||
"tenor_api_key_set",
|
|
||||||
"klipy_api_key_set",
|
|
||||||
"effective_available"
|
|
||||||
]
|
|
||||||
},
|
},
|
||||||
"youtube": {
|
"youtube": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
@@ -12990,11 +12978,7 @@
|
|||||||
"gif": {
|
"gif": {
|
||||||
"nullable": true,
|
"nullable": true,
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {"klipy_api_key": {"nullable": true, "type": "string", "maxLength": 4096}}
|
||||||
"provider": {"nullable": true, "enum": ["tenor", "klipy"], "type": "string"},
|
|
||||||
"tenor_api_key": {"nullable": true, "type": "string", "maxLength": 4096},
|
|
||||||
"klipy_api_key": {"nullable": true, "type": "string", "maxLength": 4096}
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
"youtube": {
|
"youtube": {
|
||||||
"nullable": true,
|
"nullable": true,
|
||||||
|
|||||||
@@ -100,12 +100,6 @@ pub struct InstanceIntegrationsResponse {
|
|||||||
|
|
||||||
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
|
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
|
||||||
pub struct InstanceGifIntegrationResponse {
|
pub struct InstanceGifIntegrationResponse {
|
||||||
pub provider: Option<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
pub effective_provider: String,
|
|
||||||
#[serde(default)]
|
|
||||||
pub tenor_api_key_set: bool,
|
|
||||||
#[serde(default)]
|
|
||||||
pub klipy_api_key_set: bool,
|
pub klipy_api_key_set: bool,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub effective_available: bool,
|
pub effective_available: bool,
|
||||||
@@ -549,10 +543,6 @@ pub struct InstanceIntegrationsUpdateRequest {
|
|||||||
|
|
||||||
#[derive(Clone, Debug, Default, Serialize)]
|
#[derive(Clone, Debug, Default, Serialize)]
|
||||||
pub struct InstanceGifIntegrationUpdateRequest {
|
pub struct InstanceGifIntegrationUpdateRequest {
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
|
||||||
pub provider: Option<String>,
|
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
|
||||||
pub tenor_api_key: Option<String>,
|
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub klipy_api_key: Option<String>,
|
pub klipy_api_key: Option<String>,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -603,8 +603,6 @@ fn build_integrations_update(form: &MultiValueForm) -> InstanceConfigUpdateReque
|
|||||||
policy: None,
|
policy: None,
|
||||||
integrations: Some(InstanceIntegrationsUpdateRequest {
|
integrations: Some(InstanceIntegrationsUpdateRequest {
|
||||||
gif: Some(InstanceGifIntegrationUpdateRequest {
|
gif: Some(InstanceGifIntegrationUpdateRequest {
|
||||||
provider: clean("integration_gif_provider"),
|
|
||||||
tenor_api_key: clean("integration_tenor_api_key"),
|
|
||||||
klipy_api_key: clean("integration_klipy_api_key"),
|
klipy_api_key: clean("integration_klipy_api_key"),
|
||||||
}),
|
}),
|
||||||
youtube: Some(InstanceYoutubeIntegrationUpdateRequest {
|
youtube: Some(InstanceYoutubeIntegrationUpdateRequest {
|
||||||
|
|||||||
@@ -408,11 +408,6 @@ fn integrations_config_section(
|
|||||||
csrf_token: &str,
|
csrf_token: &str,
|
||||||
integrations: &InstanceIntegrationsResponse,
|
integrations: &InstanceIntegrationsResponse,
|
||||||
) -> Markup {
|
) -> Markup {
|
||||||
let gif_provider = integrations
|
|
||||||
.gif
|
|
||||||
.provider
|
|
||||||
.as_deref()
|
|
||||||
.unwrap_or(integrations.gif.effective_provider.as_str());
|
|
||||||
let captcha_provider = integrations
|
let captcha_provider = integrations
|
||||||
.captcha
|
.captcha
|
||||||
.provider
|
.provider
|
||||||
@@ -435,15 +430,9 @@ fn integrations_config_section(
|
|||||||
div class="flex flex-wrap items-center gap-2" {
|
div class="flex flex-wrap items-center gap-2" {
|
||||||
h3 class="text-sm font-semibold text-neutral-900" { "GIF provider" }
|
h3 class="text-sm font-semibold text-neutral-900" { "GIF provider" }
|
||||||
(secret_badge("KLIPY key", integrations.gif.klipy_api_key_set))
|
(secret_badge("KLIPY key", integrations.gif.klipy_api_key_set))
|
||||||
(secret_badge("Tenor key", integrations.gif.tenor_api_key_set))
|
|
||||||
}
|
}
|
||||||
div class="grid grid-cols-1 gap-4 sm:grid-cols-3" {
|
div class="grid grid-cols-1 gap-4" {
|
||||||
(select_input("integration_gif_provider", "Provider", &[
|
|
||||||
("klipy", "KLIPY"),
|
|
||||||
("tenor", "Tenor"),
|
|
||||||
], gif_provider))
|
|
||||||
(password_input("integration_klipy_api_key", "KLIPY API key", Some("Leave blank to keep the current key.")))
|
(password_input("integration_klipy_api_key", "KLIPY API key", Some("Leave blank to keep the current key.")))
|
||||||
(password_input("integration_tenor_api_key", "Tenor API key", Some("Leave blank to keep the current key.")))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -360,15 +360,9 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
|
|||||||
bluesky: master.auth.bluesky as BlueskyOAuthConfig,
|
bluesky: master.auth.bluesky as BlueskyOAuthConfig,
|
||||||
},
|
},
|
||||||
cookie: master.cookie,
|
cookie: master.cookie,
|
||||||
gif: {
|
|
||||||
provider: master.integrations.gif.provider,
|
|
||||||
},
|
|
||||||
klipy: {
|
klipy: {
|
||||||
apiKey: master.integrations.klipy.api_key,
|
apiKey: master.integrations.klipy.api_key,
|
||||||
},
|
},
|
||||||
tenor: {
|
|
||||||
apiKey: master.integrations.tenor.api_key,
|
|
||||||
},
|
|
||||||
youtube: {
|
youtube: {
|
||||||
apiKey: master.integrations.youtube.api_key,
|
apiKey: master.integrations.youtube.api_key,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -290,8 +290,6 @@ export function InstanceConfigAdminController(app: HonoApp) {
|
|||||||
await instanceConfigRepository.setInstanceIntegrationsConfig({
|
await instanceConfigRepository.setInstanceIntegrationsConfig({
|
||||||
gif: data.integrations.gif
|
gif: data.integrations.gif
|
||||||
? omitUndefinedFields({
|
? omitUndefinedFields({
|
||||||
provider: readOptionalField(data.integrations.gif, 'provider'),
|
|
||||||
tenor_api_key: readOptionalField(data.integrations.gif, 'tenor_api_key'),
|
|
||||||
klipy_api_key: readOptionalField(data.integrations.gif, 'klipy_api_key'),
|
klipy_api_key: readOptionalField(data.integrations.gif, 'klipy_api_key'),
|
||||||
})
|
})
|
||||||
: undefined,
|
: undefined,
|
||||||
|
|||||||
@@ -68,7 +68,7 @@ export function configureMiddleware(routes: HonoApp, options: MiddlewarePipeline
|
|||||||
enabled: true,
|
enabled: true,
|
||||||
logger,
|
logger,
|
||||||
trustClientIpHeader,
|
trustClientIpHeader,
|
||||||
clientIpHeaderName: clientIpHeaderName ?? 'x-forwarded-for',
|
clientIpHeaderName: resolvedHeader,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -271,15 +271,9 @@ export interface APIConfig {
|
|||||||
domain: string;
|
domain: string;
|
||||||
secure: boolean;
|
secure: boolean;
|
||||||
};
|
};
|
||||||
gif: {
|
|
||||||
provider: 'klipy' | 'tenor';
|
|
||||||
};
|
|
||||||
klipy: {
|
klipy: {
|
||||||
apiKey?: string;
|
apiKey?: string;
|
||||||
};
|
};
|
||||||
tenor: {
|
|
||||||
apiKey?: string;
|
|
||||||
};
|
|
||||||
youtube: {
|
youtube: {
|
||||||
apiKey?: string;
|
apiKey?: string;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
import {ResolveGifUrlsBodySchema, ResolveGifUrlsResponse} from '@fluxer/schema/src/domains/gif/FavoriteGifSchemas';
|
import {ResolveGifUrlsBodySchema, ResolveGifUrlsResponse} from '@fluxer/schema/src/domains/gif/FavoriteGifSchemas';
|
||||||
import type {Context} from 'hono';
|
import type {Context} from 'hono';
|
||||||
|
import {resolveGifRequestCountry} from '../gif/GifRequestCountry';
|
||||||
import {DefaultUserOnly, LoginRequired} from '../middleware/AuthMiddleware';
|
import {DefaultUserOnly, LoginRequired} from '../middleware/AuthMiddleware';
|
||||||
import {RateLimitMiddleware} from '../middleware/RateLimitMiddleware';
|
import {RateLimitMiddleware} from '../middleware/RateLimitMiddleware';
|
||||||
import {OpenAPI} from '../middleware/ResponseTypeMiddleware';
|
import {OpenAPI} from '../middleware/ResponseTypeMiddleware';
|
||||||
@@ -11,8 +12,8 @@ import type {HonoApp, HonoEnv} from '../types/HonoEnv';
|
|||||||
import {Validator} from '../Validator';
|
import {Validator} from '../Validator';
|
||||||
import {resolveFavoriteGifEntry} from './FavoriteGifResolver';
|
import {resolveFavoriteGifEntry} from './FavoriteGifResolver';
|
||||||
|
|
||||||
function getCountry(ctx: Context<HonoEnv>): string {
|
async function getCountry(ctx: Context<HonoEnv>): Promise<string> {
|
||||||
return ctx.req.header('CF-IPCountry') || 'US';
|
return resolveGifRequestCountry(ctx.req.raw);
|
||||||
}
|
}
|
||||||
|
|
||||||
function getLocale(ctx: Context<HonoEnv>): string {
|
function getLocale(ctx: Context<HonoEnv>): string {
|
||||||
@@ -42,7 +43,7 @@ export function FavoriteGifController(app: HonoApp) {
|
|||||||
const gifService = ctx.get('gifService');
|
const gifService = ctx.get('gifService');
|
||||||
const unfurlerService = getUnfurlerService();
|
const unfurlerService = getUnfurlerService();
|
||||||
const locale = getLocale(ctx);
|
const locale = getLocale(ctx);
|
||||||
const country = getCountry(ctx);
|
const country = await getCountry(ctx);
|
||||||
const entries = await Promise.all(
|
const entries = await Promise.all(
|
||||||
urls.map((url) => resolveFavoriteGifEntry({url, locale, country, gifService, mediaService, unfurlerService})),
|
urls.map((url) => resolveFavoriteGifEntry({url, locale, country, gifService, mediaService, unfurlerService})),
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -0,0 +1,87 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
import type {GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
|
||||||
|
import {describe, expect, it, vi} from 'vitest';
|
||||||
|
import {GifService} from '../gif/GifService';
|
||||||
|
import type {IGifProvider} from '../gif/IGifProvider';
|
||||||
|
import type {IMediaService} from '../infrastructure/IMediaService';
|
||||||
|
import type {IUnfurlerService} from '../infrastructure/IUnfurlerService';
|
||||||
|
import {resolveFavoriteGifEntry} from './FavoriteGifResolver';
|
||||||
|
|
||||||
|
function createProvider(gif: GifResponse): IGifProvider {
|
||||||
|
return {
|
||||||
|
meta: {
|
||||||
|
name: 'klipy',
|
||||||
|
displayName: 'KLIPY',
|
||||||
|
attributionRequired: true,
|
||||||
|
},
|
||||||
|
isAvailable: async () => true,
|
||||||
|
search: async () => [],
|
||||||
|
registerShare: async () => undefined,
|
||||||
|
getFeatured: async () => ({gifs: [], categories: []}),
|
||||||
|
getTrendingGifs: async () => [],
|
||||||
|
suggest: async () => [],
|
||||||
|
resolveByUrl: async () => gif,
|
||||||
|
buildShareUrl: (slug) => `https://klipy.com/gifs/${slug}`,
|
||||||
|
extractSlugFromUrl: () => gif.slug,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('resolveFavoriteGifEntry', () => {
|
||||||
|
it('returns provider GIF media without probing generic media fallbacks', async () => {
|
||||||
|
const gif: GifResponse = {
|
||||||
|
id: 'goatplaybanjo-chat-4',
|
||||||
|
slug: 'goatplaybanjo-chat-4',
|
||||||
|
provider: 'klipy',
|
||||||
|
title: 'Goatplaybanjo Chat',
|
||||||
|
url: 'https://klipy.com/gifs/goatplaybanjo-chat-4',
|
||||||
|
src: 'https://static.klipy.example/fallback.gif',
|
||||||
|
proxy_src: 'https://media.example/fallback.gif',
|
||||||
|
width: 120,
|
||||||
|
height: 100,
|
||||||
|
media: {
|
||||||
|
tinygif: {
|
||||||
|
src: 'https://static.klipy.example/tiny.gif',
|
||||||
|
proxy_src: 'https://media.example/tiny.gif',
|
||||||
|
width: 80,
|
||||||
|
height: 60,
|
||||||
|
},
|
||||||
|
webm: {
|
||||||
|
src: 'https://static.klipy.example/full.webm',
|
||||||
|
proxy_src: 'https://media.example/full.webm',
|
||||||
|
width: 220,
|
||||||
|
height: 229,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
placeholder: 'thumbhash',
|
||||||
|
};
|
||||||
|
const mediaService = {
|
||||||
|
getMetadata: vi.fn(),
|
||||||
|
getExternalMediaProxyURL: vi.fn(),
|
||||||
|
} as unknown as IMediaService;
|
||||||
|
const unfurlerService = {
|
||||||
|
unfurl: vi.fn(),
|
||||||
|
} as unknown as IUnfurlerService;
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
resolveFavoriteGifEntry({
|
||||||
|
url: gif.url,
|
||||||
|
locale: 'en-US',
|
||||||
|
country: 'US',
|
||||||
|
gifService: new GifService(createProvider(gif)),
|
||||||
|
mediaService,
|
||||||
|
unfurlerService,
|
||||||
|
}),
|
||||||
|
).resolves.toEqual({
|
||||||
|
url: gif.url,
|
||||||
|
proxy_url: 'https://media.example/full.webm',
|
||||||
|
width: 220,
|
||||||
|
height: 229,
|
||||||
|
media: gif.media,
|
||||||
|
content_type: 'video/webm',
|
||||||
|
placeholder: 'thumbhash',
|
||||||
|
});
|
||||||
|
expect(mediaService.getMetadata).not.toHaveBeenCalled();
|
||||||
|
expect(unfurlerService.unfurl).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -4,6 +4,7 @@ import {Logger} from '@fluxer/logger/src/Logger';
|
|||||||
import type {ResolvedGifEntrySchema} from '@fluxer/schema/src/domains/gif/FavoriteGifSchemas';
|
import type {ResolvedGifEntrySchema} from '@fluxer/schema/src/domains/gif/FavoriteGifSchemas';
|
||||||
import type {GifMediaFormat, GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
|
import type {GifMediaFormat, GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
|
||||||
import type {EmbedMediaResponse} from '@fluxer/schema/src/domains/message/EmbedSchemas';
|
import type {EmbedMediaResponse} from '@fluxer/schema/src/domains/message/EmbedSchemas';
|
||||||
|
import {tryExtractGifProviderSlug} from '../gif/GifProviderUtils';
|
||||||
import type {GifService} from '../gif/GifService';
|
import type {GifService} from '../gif/GifService';
|
||||||
import type {IGifProvider} from '../gif/IGifProvider';
|
import type {IGifProvider} from '../gif/IGifProvider';
|
||||||
import type {IMediaService, MediaProxyMetadataResponse} from '../infrastructure/IMediaService';
|
import type {IMediaService, MediaProxyMetadataResponse} from '../infrastructure/IMediaService';
|
||||||
@@ -91,12 +92,9 @@ async function resolveProviderGifUrl({
|
|||||||
country: string;
|
country: string;
|
||||||
gifService: GifService;
|
gifService: GifService;
|
||||||
}): Promise<GifResponse | null> {
|
}): Promise<GifResponse | null> {
|
||||||
for (const provider of gifService.listProviders()) {
|
const provider = gifService.getProvider();
|
||||||
if (!(await provider.isAvailable()) || !provider.extractSlugFromUrl(url)) continue;
|
if (!(await tryExtractGifProviderSlug(provider, url))) return null;
|
||||||
const gif = await resolveProviderUrl(provider, {url, locale, country});
|
return resolveProviderUrl(provider, {url, locale, country});
|
||||||
if (gif) return gif;
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function resolveProviderUrl(
|
async function resolveProviderUrl(
|
||||||
|
|||||||
@@ -18,8 +18,12 @@ import {createAttachmentID, createMemeID, userIdToChannelId} from '../BrandedTyp
|
|||||||
import {Config} from '../Config';
|
import {Config} from '../Config';
|
||||||
import type {ChannelService} from '../channel/services/ChannelService';
|
import type {ChannelService} from '../channel/services/ChannelService';
|
||||||
import {makeAttachmentCdnKey, makeAttachmentCdnUrl} from '../channel/services/message/MessageHelpers';
|
import {makeAttachmentCdnKey, makeAttachmentCdnUrl} from '../channel/services/message/MessageHelpers';
|
||||||
|
import {
|
||||||
|
isOptionalGifProviderError,
|
||||||
|
type ResolvedGifProviderSlug,
|
||||||
|
tryExtractGifProviderSlug,
|
||||||
|
} from '../gif/GifProviderUtils';
|
||||||
import type {GifService} from '../gif/GifService';
|
import type {GifService} from '../gif/GifService';
|
||||||
import type {IGifProvider} from '../gif/IGifProvider';
|
|
||||||
import type {MediaProxyMetadataResponse} from '../infrastructure/IMediaService';
|
import type {MediaProxyMetadataResponse} from '../infrastructure/IMediaService';
|
||||||
import type {IStorageService} from '../infrastructure/IStorageService';
|
import type {IStorageService} from '../infrastructure/IStorageService';
|
||||||
import type {IUnfurlerService} from '../infrastructure/IUnfurlerService';
|
import type {IUnfurlerService} from '../infrastructure/IUnfurlerService';
|
||||||
@@ -122,7 +126,7 @@ export class FavoriteMemeService {
|
|||||||
if (!message) {
|
if (!message) {
|
||||||
throw new UnknownMessageError();
|
throw new UnknownMessageError();
|
||||||
}
|
}
|
||||||
const media = this.findMediaInMessage(message, attachmentId, embedIndex);
|
const media = await this.findMediaInMessage(message, attachmentId, embedIndex);
|
||||||
if (!media) {
|
if (!media) {
|
||||||
throw InputValidationError.fromCode('media', ValidationErrorCodes.NO_VALID_MEDIA_IN_MESSAGE);
|
throw InputValidationError.fromCode('media', ValidationErrorCodes.NO_VALID_MEDIA_IN_MESSAGE);
|
||||||
}
|
}
|
||||||
@@ -319,18 +323,9 @@ export class FavoriteMemeService {
|
|||||||
throw new MediaMetadataError('URL');
|
throw new MediaMetadataError('URL');
|
||||||
}
|
}
|
||||||
let contentHash = metadata.content_hash;
|
let contentHash = metadata.content_hash;
|
||||||
const fileData = Buffer.from(metadata.base64 ?? '', 'base64');
|
const resolvedGif = await this.resolveGifFromInputs({slug: gifSlug, providerName: gifProvider, url});
|
||||||
const resolvedGif = this.resolveGifFromInputs({slug: gifSlug, providerName: gifProvider, url});
|
|
||||||
if (resolvedGif) {
|
if (resolvedGif) {
|
||||||
const canonicalUrl = resolvedGif.provider.buildShareUrl(resolvedGif.slug);
|
contentHash = await this.resolveProviderGifContentHash(resolvedGif, contentHash);
|
||||||
const unfurled = await this.unfurlerService.unfurl(canonicalUrl, 'allow');
|
|
||||||
if (unfurled.length > 0 && unfurled[0].video?.content_hash) {
|
|
||||||
contentHash = unfurled[0].video.content_hash;
|
|
||||||
Logger.debug(
|
|
||||||
{provider: resolvedGif.provider.meta.name, gifSlug: resolvedGif.slug, contentHash},
|
|
||||||
'Using unfurled video content_hash for provider GIF',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
const existingMemes = await this.favoriteMemeRepository.findByUserId(user.id);
|
const existingMemes = await this.favoriteMemeRepository.findByUserId(user.id);
|
||||||
const duplicate = existingMemes.find((meme) => meme.contentHash === contentHash);
|
const duplicate = existingMemes.find((meme) => meme.contentHash === contentHash);
|
||||||
@@ -343,6 +338,7 @@ export class FavoriteMemeService {
|
|||||||
const userChannelId = userIdToChannelId(user.id);
|
const userChannelId = userIdToChannelId(user.id);
|
||||||
const newAttachmentId = createAttachmentID(await this.apiContext.services.snowflake.generate());
|
const newAttachmentId = createAttachmentID(await this.apiContext.services.snowflake.generate());
|
||||||
const storageKey = makeAttachmentCdnKey(userChannelId, newAttachmentId, filename);
|
const storageKey = makeAttachmentCdnKey(userChannelId, newAttachmentId, filename);
|
||||||
|
const fileData = Buffer.from(metadata.base64 ?? '', 'base64');
|
||||||
await this.storageService.uploadObject({
|
await this.storageService.uploadObject({
|
||||||
bucket: Config.s3.buckets.cdn,
|
bucket: Config.s3.buckets.cdn,
|
||||||
key: storageKey,
|
key: storageKey,
|
||||||
@@ -470,7 +466,7 @@ export class FavoriteMemeService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private resolveGifFromInputs({
|
private async resolveGifFromInputs({
|
||||||
slug,
|
slug,
|
||||||
providerName,
|
providerName,
|
||||||
url,
|
url,
|
||||||
@@ -478,34 +474,51 @@ export class FavoriteMemeService {
|
|||||||
slug?: string | null;
|
slug?: string | null;
|
||||||
providerName?: string | null;
|
providerName?: string | null;
|
||||||
url: string;
|
url: string;
|
||||||
}): {
|
}): Promise<ResolvedGifProviderSlug | null> {
|
||||||
provider: IGifProvider;
|
|
||||||
slug: string;
|
|
||||||
} | null {
|
|
||||||
if (slug && providerName) {
|
if (slug && providerName) {
|
||||||
const provider = this.gifService.getByName(providerName);
|
const provider = this.gifService.getByName(providerName);
|
||||||
const trimmed = slug.trim();
|
const trimmed = slug.trim();
|
||||||
if (provider && trimmed) {
|
if (provider && trimmed) {
|
||||||
const normalized = provider.extractSlugFromUrl(trimmed) ?? trimmed;
|
const normalized = (await tryExtractGifProviderSlug(provider, trimmed)) ?? trimmed;
|
||||||
return {provider, slug: normalized};
|
return {provider, slug: normalized};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for (const provider of this.gifService.listProviders()) {
|
const provider = this.gifService.getProvider();
|
||||||
const extracted = provider.extractSlugFromUrl(url);
|
const extracted = await tryExtractGifProviderSlug(provider, url);
|
||||||
if (extracted) {
|
return extracted ? {provider, slug: extracted} : null;
|
||||||
return {provider, slug: extracted};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private detectGifFromUrl(url: string): {
|
private async detectGifFromUrl(url: string): Promise<ResolvedGifProviderSlug | null> {
|
||||||
provider: IGifProvider;
|
|
||||||
slug: string;
|
|
||||||
} | null {
|
|
||||||
return this.resolveGifFromInputs({url});
|
return this.resolveGifFromInputs({url});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async resolveProviderGifContentHash(
|
||||||
|
resolvedGif: ResolvedGifProviderSlug,
|
||||||
|
fallbackContentHash: string,
|
||||||
|
): Promise<string> {
|
||||||
|
try {
|
||||||
|
const canonicalUrl = resolvedGif.provider.buildShareUrl(resolvedGif.slug);
|
||||||
|
const unfurled = await this.unfurlerService.unfurl(canonicalUrl, 'allow');
|
||||||
|
if (unfurled.length > 0 && unfurled[0].video?.content_hash) {
|
||||||
|
Logger.debug(
|
||||||
|
{
|
||||||
|
provider: resolvedGif.provider.meta.name,
|
||||||
|
gifSlug: resolvedGif.slug,
|
||||||
|
contentHash: unfurled[0].video.content_hash,
|
||||||
|
},
|
||||||
|
'Using unfurled video content_hash for provider GIF',
|
||||||
|
);
|
||||||
|
return unfurled[0].video.content_hash;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (!isOptionalGifProviderError(error)) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
Logger.debug({error, provider: resolvedGif.provider.meta.name}, 'Skipping unavailable GIF provider enrichment');
|
||||||
|
}
|
||||||
|
return fallbackContentHash;
|
||||||
|
}
|
||||||
|
|
||||||
private resolveFavoriteMemeName(name: string | undefined | null, fallbackFilename: string): string {
|
private resolveFavoriteMemeName(name: string | undefined | null, fallbackFilename: string): string {
|
||||||
const normalizedInput = typeof name === 'string' ? name.trim() : '';
|
const normalizedInput = typeof name === 'string' ? name.trim() : '';
|
||||||
const fallbackName = fallbackFilename.trim() || 'favorite meme';
|
const fallbackName = fallbackFilename.trim() || 'favorite meme';
|
||||||
@@ -517,11 +530,11 @@ export class FavoriteMemeService {
|
|||||||
return finalName;
|
return finalName;
|
||||||
}
|
}
|
||||||
|
|
||||||
private findMediaInMessage(
|
private async findMediaInMessage(
|
||||||
message: Message,
|
message: Message,
|
||||||
preferredAttachmentId?: string,
|
preferredAttachmentId?: string,
|
||||||
preferredEmbedIndex?: number,
|
preferredEmbedIndex?: number,
|
||||||
): FavoriteMemeMedia | null {
|
): Promise<FavoriteMemeMedia | null> {
|
||||||
const attachments = this.getMessageAttachmentCandidates(message);
|
const attachments = this.getMessageAttachmentCandidates(message);
|
||||||
const embeds = this.getMessageEmbedCandidates(message);
|
const embeds = this.getMessageEmbedCandidates(message);
|
||||||
if (preferredEmbedIndex !== undefined) {
|
if (preferredEmbedIndex !== undefined) {
|
||||||
@@ -553,7 +566,7 @@ export class FavoriteMemeService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
for (const embed of embeds) {
|
for (const embed of embeds) {
|
||||||
const media = this.mediaFromEmbed(embed, 'media');
|
const media = await this.mediaFromEmbed(embed, 'media');
|
||||||
if (media) return media;
|
if (media) return media;
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
@@ -591,7 +604,10 @@ export class FavoriteMemeService {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private mediaFromEmbed(embed: MessageEmbedCandidate, fallbackFilename: string): FavoriteMemeMedia | null {
|
private async mediaFromEmbed(
|
||||||
|
embed: MessageEmbedCandidate,
|
||||||
|
fallbackFilename: string,
|
||||||
|
): Promise<FavoriteMemeMedia | null> {
|
||||||
const media = embed.image || embed.video || embed.thumbnail;
|
const media = embed.image || embed.video || embed.thumbnail;
|
||||||
if (!media?.url) {
|
if (!media?.url) {
|
||||||
return null;
|
return null;
|
||||||
@@ -603,7 +619,7 @@ export class FavoriteMemeService {
|
|||||||
}
|
}
|
||||||
const isExternal = !this.isInternalCDNUrl(media.url);
|
const isExternal = !this.isInternalCDNUrl(media.url);
|
||||||
const isGifv = embed.type === 'gifv' || isAnimatedEmbedMedia(media.contentType, media.flags);
|
const isGifv = embed.type === 'gifv' || isAnimatedEmbedMedia(media.contentType, media.flags);
|
||||||
const detectedGif = embed.type === 'gifv' ? this.detectGifFromUrl(media.url) : null;
|
const detectedGif = embed.type === 'gifv' ? await this.detectGifFromUrl(media.url) : null;
|
||||||
return {
|
return {
|
||||||
isExternal,
|
isExternal,
|
||||||
url: media.url,
|
url: media.url,
|
||||||
|
|||||||
@@ -0,0 +1,71 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
import {GIF_PROVIDER_HEADER, type GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
|
||||||
|
import {Hono} from 'hono';
|
||||||
|
import {describe, expect, it, vi} from 'vitest';
|
||||||
|
import type {HonoEnv} from '../types/HonoEnv';
|
||||||
|
import {GifController} from './GifController';
|
||||||
|
import {GifService} from './GifService';
|
||||||
|
import type {IGifProvider} from './IGifProvider';
|
||||||
|
|
||||||
|
function createProvider(gifs: Array<GifResponse>): IGifProvider {
|
||||||
|
return {
|
||||||
|
meta: {
|
||||||
|
name: 'klipy',
|
||||||
|
displayName: 'KLIPY',
|
||||||
|
attributionRequired: true,
|
||||||
|
},
|
||||||
|
isAvailable: async () => true,
|
||||||
|
search: vi.fn(async () => gifs),
|
||||||
|
registerShare: async () => undefined,
|
||||||
|
getFeatured: async () => ({gifs: [], categories: []}),
|
||||||
|
getTrendingGifs: async () => [],
|
||||||
|
suggest: async () => [],
|
||||||
|
resolveByUrl: async () => null,
|
||||||
|
buildShareUrl: (slug) => `https://klipy.com/gifs/${slug}`,
|
||||||
|
extractSlugFromUrl: () => null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createApp(gifService: GifService): Hono<HonoEnv> {
|
||||||
|
const app = new Hono<HonoEnv>({strict: true});
|
||||||
|
app.use('*', async (ctx, next) => {
|
||||||
|
ctx.set('gifService', gifService);
|
||||||
|
ctx.set('user', {
|
||||||
|
isBot: false,
|
||||||
|
suspiciousActivityFlags: 0,
|
||||||
|
} as HonoEnv['Variables']['user']);
|
||||||
|
ctx.set('authTokenType', 'session');
|
||||||
|
await next();
|
||||||
|
});
|
||||||
|
GifController(app);
|
||||||
|
return app;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('GifController', () => {
|
||||||
|
it('serves deprecated Tenor routes from the configured KLIPY provider', async () => {
|
||||||
|
const gifs: Array<GifResponse> = [
|
||||||
|
{
|
||||||
|
id: 'goatplaybanjo-chat-4',
|
||||||
|
slug: 'goatplaybanjo-chat-4',
|
||||||
|
provider: 'klipy',
|
||||||
|
title: 'Goatplaybanjo Chat',
|
||||||
|
url: 'https://klipy.com/gifs/goatplaybanjo-chat-4',
|
||||||
|
src: 'https://static.klipy.example/full.webm',
|
||||||
|
proxy_src: 'https://media.example/full.webm',
|
||||||
|
width: 220,
|
||||||
|
height: 229,
|
||||||
|
media: {},
|
||||||
|
placeholder: null,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const provider = createProvider(gifs);
|
||||||
|
const response = await createApp(new GifService(provider)).request('/tenor/search?q=cat&locale=en-US');
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(response.headers.get('Deprecation')).toBe('true');
|
||||||
|
expect(response.headers.get(GIF_PROVIDER_HEADER)).toBe('klipy');
|
||||||
|
await expect(response.json()).resolves.toEqual(gifs);
|
||||||
|
expect(provider.search).toHaveBeenCalledWith({q: 'cat', locale: 'en_US', country: 'US'});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -17,6 +17,7 @@ import {RateLimitConfigs} from '../RateLimitConfig';
|
|||||||
import type {HonoApp, HonoEnv} from '../types/HonoEnv';
|
import type {HonoApp, HonoEnv} from '../types/HonoEnv';
|
||||||
import {Validator} from '../Validator';
|
import {Validator} from '../Validator';
|
||||||
import {GifProviderHeaderMiddleware} from './GifProviderHeaderMiddleware';
|
import {GifProviderHeaderMiddleware} from './GifProviderHeaderMiddleware';
|
||||||
|
import {resolveGifRequestCountry} from './GifRequestCountry';
|
||||||
|
|
||||||
const TAGS = ['GIFs'];
|
const TAGS = ['GIFs'];
|
||||||
|
|
||||||
@@ -34,8 +35,8 @@ const PREFIXES: ReadonlyArray<PrefixConfig> = [
|
|||||||
];
|
];
|
||||||
const DEPRECATION_NOTICE = 'Use /gifs/* instead - these vendor-specific paths are deprecated and will be removed.';
|
const DEPRECATION_NOTICE = 'Use /gifs/* instead - these vendor-specific paths are deprecated and will be removed.';
|
||||||
|
|
||||||
function getCountry(ctx: Context<HonoEnv>): string {
|
async function getCountry(ctx: Context<HonoEnv>): Promise<string> {
|
||||||
return ctx.req.header('CF-IPCountry') || 'US';
|
return resolveGifRequestCountry(ctx.req.raw);
|
||||||
}
|
}
|
||||||
|
|
||||||
function deprecationMiddleware(deprecated: boolean): MiddlewareHandler<HonoEnv> {
|
function deprecationMiddleware(deprecated: boolean): MiddlewareHandler<HonoEnv> {
|
||||||
@@ -72,7 +73,8 @@ function registerRoutes(app: HonoApp, cfg: PrefixConfig) {
|
|||||||
async (ctx) => {
|
async (ctx) => {
|
||||||
const {q, locale} = ctx.req.valid('query');
|
const {q, locale} = ctx.req.valid('query');
|
||||||
const provider = await ctx.get('gifService').getActive();
|
const provider = await ctx.get('gifService').getActive();
|
||||||
return ctx.json(await provider.search({q, locale, country: getCountry(ctx)}));
|
const country = await getCountry(ctx);
|
||||||
|
return ctx.json(await provider.search({q, locale, country}));
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
app.get(
|
app.get(
|
||||||
@@ -94,7 +96,8 @@ function registerRoutes(app: HonoApp, cfg: PrefixConfig) {
|
|||||||
Validator('query', GifLocaleQuery),
|
Validator('query', GifLocaleQuery),
|
||||||
async (ctx) => {
|
async (ctx) => {
|
||||||
const provider = await ctx.get('gifService').getActive();
|
const provider = await ctx.get('gifService').getActive();
|
||||||
return ctx.json(await provider.getFeatured({locale: ctx.req.valid('query').locale, country: getCountry(ctx)}));
|
const country = await getCountry(ctx);
|
||||||
|
return ctx.json(await provider.getFeatured({locale: ctx.req.valid('query').locale, country}));
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
const trendingPath = prefix === '/gifs' ? `${prefix}/trending` : `${prefix}/trending-gifs`;
|
const trendingPath = prefix === '/gifs' ? `${prefix}/trending` : `${prefix}/trending-gifs`;
|
||||||
@@ -117,10 +120,11 @@ function registerRoutes(app: HonoApp, cfg: PrefixConfig) {
|
|||||||
Validator('query', GifLocaleQuery),
|
Validator('query', GifLocaleQuery),
|
||||||
async (ctx) => {
|
async (ctx) => {
|
||||||
const provider = await ctx.get('gifService').getActive();
|
const provider = await ctx.get('gifService').getActive();
|
||||||
|
const country = await getCountry(ctx);
|
||||||
return ctx.json(
|
return ctx.json(
|
||||||
await provider.getTrendingGifs({
|
await provider.getTrendingGifs({
|
||||||
locale: ctx.req.valid('query').locale,
|
locale: ctx.req.valid('query').locale,
|
||||||
country: getCountry(ctx),
|
country,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
@@ -145,7 +149,8 @@ function registerRoutes(app: HonoApp, cfg: PrefixConfig) {
|
|||||||
async (ctx) => {
|
async (ctx) => {
|
||||||
const {id, q, locale} = ctx.req.valid('json');
|
const {id, q, locale} = ctx.req.valid('json');
|
||||||
const provider = await ctx.get('gifService').getActive();
|
const provider = await ctx.get('gifService').getActive();
|
||||||
await provider.registerShare({id, q: q ?? '', locale, country: getCountry(ctx)});
|
const country = await getCountry(ctx);
|
||||||
|
await provider.registerShare({id, q: q ?? '', locale, country});
|
||||||
return ctx.body(null, 204);
|
return ctx.body(null, 204);
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1,99 +0,0 @@
|
|||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
import type {GifCategoryTagResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
|
|
||||||
import type {ICacheService} from '@pkgs/cache/src/ICacheService';
|
|
||||||
import {ms} from 'itty-time';
|
|
||||||
|
|
||||||
const ENRICHED_CATEGORIES_FRESH_MS = ms('30 minutes');
|
|
||||||
const ENRICHED_CATEGORIES_TTL_SECONDS = Math.floor(ms('6 hours') / 1000);
|
|
||||||
const SEEN_LOCALES_TTL_SECONDS = Math.floor(ms('7 days') / 1000);
|
|
||||||
export const REFRESH_LOCK_TTL_SECONDS = Math.floor(ms('5 minutes') / 1000);
|
|
||||||
|
|
||||||
interface EnrichedCategoriesEntry {
|
|
||||||
data: Array<GifCategoryTagResponse>;
|
|
||||||
timestamp: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
function enrichedCategoriesCacheKey(provider: string, locale: string, country: string): string {
|
|
||||||
return `gif:featured_categories_enriched:${provider}:${locale}:${country}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function seenLocalesSetKey(provider: string): string {
|
|
||||||
return `gif:featured_categories_seen_locales:${provider}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function encodeLocaleMember(locale: string, country: string): string {
|
|
||||||
return `${locale}|${country}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function decodeLocaleMember(member: string): {
|
|
||||||
locale: string;
|
|
||||||
country: string;
|
|
||||||
} | null {
|
|
||||||
const idx = member.indexOf('|');
|
|
||||||
if (idx <= 0) return null;
|
|
||||||
const locale = member.slice(0, idx);
|
|
||||||
const country = member.slice(idx + 1);
|
|
||||||
if (!locale || !country) return null;
|
|
||||||
return {locale, country};
|
|
||||||
}
|
|
||||||
|
|
||||||
export function refreshLockKey(provider: string, locale: string, country: string): string {
|
|
||||||
return `gif:featured_categories_refresh_lock:${provider}:${locale}:${country}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function readEnrichedCategoriesCache(
|
|
||||||
cache: ICacheService,
|
|
||||||
provider: string,
|
|
||||||
locale: string,
|
|
||||||
country: string,
|
|
||||||
): Promise<{
|
|
||||||
data: Array<GifCategoryTagResponse>;
|
|
||||||
isStale: boolean;
|
|
||||||
} | null> {
|
|
||||||
const entry = await cache.get<EnrichedCategoriesEntry>(enrichedCategoriesCacheKey(provider, locale, country));
|
|
||||||
if (!entry) return null;
|
|
||||||
const isStale = Date.now() - entry.timestamp > ENRICHED_CATEGORIES_FRESH_MS;
|
|
||||||
return {data: entry.data, isStale};
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function writeEnrichedCategoriesCache(
|
|
||||||
cache: ICacheService,
|
|
||||||
provider: string,
|
|
||||||
locale: string,
|
|
||||||
country: string,
|
|
||||||
data: Array<GifCategoryTagResponse>,
|
|
||||||
): Promise<void> {
|
|
||||||
const entry: EnrichedCategoriesEntry = {data, timestamp: Date.now()};
|
|
||||||
await cache.set(enrichedCategoriesCacheKey(provider, locale, country), entry, ENRICHED_CATEGORIES_TTL_SECONDS);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function trackSeenLocale(
|
|
||||||
cache: ICacheService,
|
|
||||||
provider: string,
|
|
||||||
locale: string,
|
|
||||||
country: string,
|
|
||||||
): Promise<void> {
|
|
||||||
await cache.sadd(seenLocalesSetKey(provider), encodeLocaleMember(locale, country), SEEN_LOCALES_TTL_SECONDS);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function listSeenLocales(
|
|
||||||
cache: ICacheService,
|
|
||||||
provider: string,
|
|
||||||
): Promise<
|
|
||||||
Array<{
|
|
||||||
locale: string;
|
|
||||||
country: string;
|
|
||||||
}>
|
|
||||||
> {
|
|
||||||
const members = await cache.smembers(seenLocalesSetKey(provider));
|
|
||||||
const out: Array<{
|
|
||||||
locale: string;
|
|
||||||
country: string;
|
|
||||||
}> = [];
|
|
||||||
for (const member of members) {
|
|
||||||
const decoded = decodeLocaleMember(member);
|
|
||||||
if (decoded) out.push(decoded);
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
@@ -25,7 +25,6 @@ function createProvider(): IGifProvider {
|
|||||||
getFeatured: async () => ({gifs: [], categories: []}),
|
getFeatured: async () => ({gifs: [], categories: []}),
|
||||||
getTrendingGifs: async () => [],
|
getTrendingGifs: async () => [],
|
||||||
suggest: async () => [],
|
suggest: async () => [],
|
||||||
refreshFeaturedCategories: async () => undefined,
|
|
||||||
resolveByUrl: async () => null,
|
resolveByUrl: async () => null,
|
||||||
buildShareUrl: (slug) => `https://klipy.example/${slug}`,
|
buildShareUrl: (slug) => `https://klipy.example/${slug}`,
|
||||||
extractSlugFromUrl: () => null,
|
extractSlugFromUrl: () => null,
|
||||||
@@ -47,7 +46,7 @@ function createApp(gifService?: GifService): Hono<HonoEnv> {
|
|||||||
|
|
||||||
describe('GifProviderHeaderMiddleware', () => {
|
describe('GifProviderHeaderMiddleware', () => {
|
||||||
it('emits active GIF provider metadata headers', async () => {
|
it('emits active GIF provider metadata headers', async () => {
|
||||||
const gifService = new GifService({providers: [createProvider()], activeName: 'klipy'});
|
const gifService = new GifService(createProvider());
|
||||||
const response = await createApp(gifService).request('/probe');
|
const response = await createApp(gifService).request('/probe');
|
||||||
|
|
||||||
expect(response.headers.get(GIF_PROVIDER_HEADER)).toBe('klipy');
|
expect(response.headers.get(GIF_PROVIDER_HEADER)).toBe('klipy');
|
||||||
|
|||||||
@@ -13,8 +13,8 @@ export const GifProviderHeaderMiddleware = createMiddleware<HonoEnv>(async (ctx,
|
|||||||
await next();
|
await next();
|
||||||
const gifService = ctx.get('gifService') as GifService | undefined;
|
const gifService = ctx.get('gifService') as GifService | undefined;
|
||||||
if (!gifService) return;
|
if (!gifService) return;
|
||||||
const provider = gifService.getByName(await gifService.getActiveName());
|
const provider = await gifService.getActive().catch(() => null);
|
||||||
if (!provider || !(await provider.isAvailable())) return;
|
if (!provider) return;
|
||||||
ctx.header(GIF_PROVIDER_HEADER, provider.meta.name);
|
ctx.header(GIF_PROVIDER_HEADER, provider.meta.name);
|
||||||
ctx.header(GIF_PROVIDER_DISPLAY_NAME_HEADER, provider.meta.displayName);
|
ctx.header(GIF_PROVIDER_DISPLAY_NAME_HEADER, provider.meta.displayName);
|
||||||
ctx.header(GIF_PROVIDER_ATTRIBUTION_HEADER, provider.meta.attributionRequired ? 'true' : 'false');
|
ctx.header(GIF_PROVIDER_ATTRIBUTION_HEADER, provider.meta.attributionRequired ? 'true' : 'false');
|
||||||
|
|||||||
@@ -0,0 +1,82 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
import {ServiceUnavailableError} from '@fluxer/errors/src/domains/core/ServiceUnavailableError';
|
||||||
|
import {describe, expect, it} from 'vitest';
|
||||||
|
import {tryExtractGifProviderSlug} from './GifProviderUtils';
|
||||||
|
import type {IGifProvider} from './IGifProvider';
|
||||||
|
|
||||||
|
function createProvider(overrides: Partial<IGifProvider> = {}): IGifProvider {
|
||||||
|
return {
|
||||||
|
meta: {
|
||||||
|
name: 'klipy',
|
||||||
|
displayName: 'KLIPY',
|
||||||
|
attributionRequired: true,
|
||||||
|
},
|
||||||
|
isAvailable: async () => true,
|
||||||
|
search: async () => [],
|
||||||
|
registerShare: async () => undefined,
|
||||||
|
getFeatured: async () => ({gifs: [], categories: []}),
|
||||||
|
getTrendingGifs: async () => [],
|
||||||
|
suggest: async () => [],
|
||||||
|
resolveByUrl: async () => null,
|
||||||
|
buildShareUrl: (slug) => `https://klipy.example/${slug}`,
|
||||||
|
extractSlugFromUrl: () => 'slug',
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('GifProviderUtils', () => {
|
||||||
|
it('returns a trimmed slug from an available provider', async () => {
|
||||||
|
await expect(
|
||||||
|
tryExtractGifProviderSlug(
|
||||||
|
createProvider({
|
||||||
|
extractSlugFromUrl: () => ' goatplaybanjo-chat-4 ',
|
||||||
|
}),
|
||||||
|
'https://klipy.com/gifs/goatplaybanjo-chat-4',
|
||||||
|
),
|
||||||
|
).resolves.toBe('goatplaybanjo-chat-4');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('treats unavailable optional provider failures as no match', async () => {
|
||||||
|
await expect(
|
||||||
|
tryExtractGifProviderSlug(
|
||||||
|
createProvider({
|
||||||
|
extractSlugFromUrl: () => {
|
||||||
|
throw new ServiceUnavailableError();
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
'https://klipy.com/gifs/goatplaybanjo-chat-4',
|
||||||
|
),
|
||||||
|
).resolves.toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not check availability when a provider cannot extract a slug', async () => {
|
||||||
|
let availabilityChecks = 0;
|
||||||
|
await expect(
|
||||||
|
tryExtractGifProviderSlug(
|
||||||
|
createProvider({
|
||||||
|
isAvailable: async () => {
|
||||||
|
availabilityChecks += 1;
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
extractSlugFromUrl: () => null,
|
||||||
|
}),
|
||||||
|
'https://example.com/media.gif',
|
||||||
|
),
|
||||||
|
).resolves.toBeNull();
|
||||||
|
|
||||||
|
expect(availabilityChecks).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns null when the configured provider is unavailable', async () => {
|
||||||
|
await expect(
|
||||||
|
tryExtractGifProviderSlug(
|
||||||
|
createProvider({
|
||||||
|
isAvailable: async () => false,
|
||||||
|
extractSlugFromUrl: () => 'matched',
|
||||||
|
}),
|
||||||
|
'https://klipy.com/gifs/matched',
|
||||||
|
),
|
||||||
|
).resolves.toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';
|
||||||
|
import {ServiceUnavailableError} from '@fluxer/errors/src/domains/core/ServiceUnavailableError';
|
||||||
|
import type {IGifProvider} from './IGifProvider';
|
||||||
|
|
||||||
|
export type ResolvedGifProviderSlug = {
|
||||||
|
provider: IGifProvider;
|
||||||
|
slug: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function isOptionalGifProviderError(error: unknown): boolean {
|
||||||
|
return error instanceof FeatureTemporarilyDisabledError || error instanceof ServiceUnavailableError;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function tryExtractGifProviderSlug(provider: IGifProvider, value: string): Promise<string | null> {
|
||||||
|
try {
|
||||||
|
const slug = provider.extractSlugFromUrl(value);
|
||||||
|
const trimmed = slug?.trim() ?? '';
|
||||||
|
if (trimmed.length === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return (await provider.isAvailable()) ? trimmed : null;
|
||||||
|
} catch (error) {
|
||||||
|
if (isOptionalGifProviderError(error)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
import type {GeoipResult} from '@pkgs/geoip/src/GeoipLookup';
|
||||||
|
import {beforeEach, describe, expect, it, vi} from 'vitest';
|
||||||
|
import {resolveGifRequestCountry} from './GifRequestCountry';
|
||||||
|
|
||||||
|
const {lookupGeoipMock} = vi.hoisted(() => ({
|
||||||
|
lookupGeoipMock: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('../utils/IpUtils', () => ({
|
||||||
|
lookupGeoip: lookupGeoipMock,
|
||||||
|
}));
|
||||||
|
|
||||||
|
function geoip(countryCode: string | null): GeoipResult {
|
||||||
|
return {
|
||||||
|
countryCode,
|
||||||
|
normalizedIp: '203.0.113.10',
|
||||||
|
city: null,
|
||||||
|
region: null,
|
||||||
|
countryName: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('resolveGifRequestCountry', () => {
|
||||||
|
const req = new Request('https://fluxer.test/gifs/search');
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
lookupGeoipMock.mockReset();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses the GeoIP country code for GIF provider requests', async () => {
|
||||||
|
lookupGeoipMock.mockResolvedValue(geoip('se'));
|
||||||
|
|
||||||
|
await expect(resolveGifRequestCountry(req)).resolves.toBe('SE');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back to US when GeoIP has no country', async () => {
|
||||||
|
lookupGeoipMock.mockResolvedValue(geoip(null));
|
||||||
|
|
||||||
|
await expect(resolveGifRequestCountry(req)).resolves.toBe('US');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back to US when GeoIP lookup fails', async () => {
|
||||||
|
lookupGeoipMock.mockRejectedValue(new Error('geoip unavailable'));
|
||||||
|
|
||||||
|
await expect(resolveGifRequestCountry(req)).resolves.toBe('US');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
import {lookupGeoip} from '../utils/IpUtils';
|
||||||
|
|
||||||
|
const KLIPY_DEFAULT_COUNTRY = 'US';
|
||||||
|
|
||||||
|
export async function resolveGifRequestCountry(req: Request): Promise<string> {
|
||||||
|
try {
|
||||||
|
const geoip = await lookupGeoip(req);
|
||||||
|
return geoip.countryCode?.trim().toUpperCase() || KLIPY_DEFAULT_COUNTRY;
|
||||||
|
} catch {
|
||||||
|
return KLIPY_DEFAULT_COUNTRY;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,46 +1,27 @@
|
|||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';
|
import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';
|
||||||
import {Logger} from '../Logger';
|
|
||||||
import type {IGifProvider} from './IGifProvider';
|
import type {IGifProvider} from './IGifProvider';
|
||||||
|
|
||||||
export class GifService {
|
export class GifService {
|
||||||
private readonly providersByName: Map<string, IGifProvider>;
|
private readonly provider: IGifProvider;
|
||||||
private readonly activeName: string | (() => Promise<string>);
|
|
||||||
|
|
||||||
constructor(params: {
|
constructor(provider: IGifProvider) {
|
||||||
providers: ReadonlyArray<IGifProvider>;
|
this.provider = provider;
|
||||||
activeName: string | (() => Promise<string>);
|
}
|
||||||
}) {
|
|
||||||
this.providersByName = new Map(params.providers.map((p) => [p.meta.name, p]));
|
getProvider(): IGifProvider {
|
||||||
this.activeName = params.activeName;
|
return this.provider;
|
||||||
if (typeof params.activeName === 'string' && !this.providersByName.has(params.activeName)) {
|
|
||||||
Logger.warn(
|
|
||||||
{activeName: params.activeName, registered: Array.from(this.providersByName.keys())},
|
|
||||||
'Active GIF provider is not registered; /gifs requests will fail until configuration is fixed',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async getActive(): Promise<IGifProvider> {
|
async getActive(): Promise<IGifProvider> {
|
||||||
const activeName = await this.getActiveName();
|
if (!(await this.provider.isAvailable())) {
|
||||||
const provider = this.providersByName.get(activeName);
|
|
||||||
if (!provider || !(await provider.isAvailable())) {
|
|
||||||
Logger.debug({activeName}, 'Active GIF provider unavailable');
|
|
||||||
throw new FeatureTemporarilyDisabledError();
|
throw new FeatureTemporarilyDisabledError();
|
||||||
}
|
}
|
||||||
return provider;
|
return this.provider;
|
||||||
}
|
|
||||||
|
|
||||||
async getActiveName(): Promise<string> {
|
|
||||||
return typeof this.activeName === 'string' ? this.activeName : this.activeName();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
getByName(name: string): IGifProvider | null {
|
getByName(name: string): IGifProvider | null {
|
||||||
return this.providersByName.get(name) ?? null;
|
return name === this.provider.meta.name ? this.provider : null;
|
||||||
}
|
|
||||||
|
|
||||||
listProviders(): ReadonlyArray<IGifProvider> {
|
|
||||||
return Array.from(this.providersByName.values());
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ export interface IGifProvider {
|
|||||||
}>;
|
}>;
|
||||||
getTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>>;
|
getTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>>;
|
||||||
suggest(params: {q: string; locale: string}): Promise<Array<string>>;
|
suggest(params: {q: string; locale: string}): Promise<Array<string>>;
|
||||||
refreshFeaturedCategories(params: {locale: string; country: string}): Promise<void>;
|
|
||||||
resolveByUrl(params: {url: string; locale: string; country: string}): Promise<GifResponse | null>;
|
resolveByUrl(params: {url: string; locale: string; country: string}): Promise<GifResponse | null>;
|
||||||
buildShareUrl(slug: string): string;
|
buildShareUrl(slug: string): string;
|
||||||
extractSlugFromUrl(url: string): string | null;
|
extractSlugFromUrl(url: string): string | null;
|
||||||
|
|||||||
@@ -1,563 +0,0 @@
|
|||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
import {FLUXER_USER_AGENT} from '@fluxer/constants/src/Core';
|
|
||||||
import type {GifCategoryTagResponse, GifMediaFormat, GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
|
|
||||||
import type {ICacheService} from '@pkgs/cache/src/ICacheService';
|
|
||||||
import {ms} from 'itty-time';
|
|
||||||
import {Config} from '../Config';
|
|
||||||
import type {IMediaService} from '../infrastructure/IMediaService';
|
|
||||||
import {Logger} from '../Logger';
|
|
||||||
import {getWorkerService} from '../middleware/ServiceRegistry';
|
|
||||||
import {EXTERNAL_RESPONSE_LIMITS} from '../utils/ExternalResponseLimits';
|
|
||||||
import * as FetchUtils from '../utils/FetchUtils';
|
|
||||||
import {isJsonRecord, parseJsonUnknown} from '../utils/JsonBoundaryUtils';
|
|
||||||
import {
|
|
||||||
REFRESH_LOCK_TTL_SECONDS,
|
|
||||||
readEnrichedCategoriesCache,
|
|
||||||
refreshLockKey,
|
|
||||||
trackSeenLocale,
|
|
||||||
writeEnrichedCategoriesCache,
|
|
||||||
} from './GifFeaturedCategoriesCache';
|
|
||||||
import type {GifProviderMeta, IGifProvider} from './IGifProvider';
|
|
||||||
|
|
||||||
const KLIPY_BASE_URL = 'https://api.klipy.com/v2';
|
|
||||||
const DEFAULT_CONTENT_FILTER = 'low';
|
|
||||||
const CLIENT_KEY = 'fluxer';
|
|
||||||
const MAX_RETRIES = 3;
|
|
||||||
const BACKOFF_BASE_DELAY = ms('1 second');
|
|
||||||
const CACHE_EXPIRATION_TIME = ms('5 minutes');
|
|
||||||
const KLIPY_PROVIDER_META: GifProviderMeta = {
|
|
||||||
name: 'klipy',
|
|
||||||
displayName: 'KLIPY',
|
|
||||||
attributionRequired: true,
|
|
||||||
};
|
|
||||||
|
|
||||||
interface KlipyFileEntry {
|
|
||||||
url?: string;
|
|
||||||
width?: number;
|
|
||||||
height?: number;
|
|
||||||
size?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
type KlipyFormatKey = 'gif' | 'webp' | 'mp4' | 'webm';
|
|
||||||
type KlipySizeKey = 'hd' | 'md' | 'sm' | 'xs';
|
|
||||||
type KlipyFileBucket = Partial<Record<KlipyFormatKey, KlipyFileEntry>>;
|
|
||||||
|
|
||||||
interface KlipyGif {
|
|
||||||
id: string;
|
|
||||||
slug?: string;
|
|
||||||
title: string;
|
|
||||||
itemurl: string;
|
|
||||||
file?: Partial<Record<KlipySizeKey, KlipyFileBucket>>;
|
|
||||||
media_formats?: {
|
|
||||||
webm?: {
|
|
||||||
url: string;
|
|
||||||
dims: [number, number];
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const KLIPY_SIZE_PREFERENCE: ReadonlyArray<KlipySizeKey> = ['hd', 'md', 'sm', 'xs'];
|
|
||||||
const KLIPY_FORMAT_KEYS: ReadonlyArray<KlipyFormatKey> = ['webm', 'mp4', 'webp', 'gif'];
|
|
||||||
const KLIPY_PUBLIC_FORMAT_KEYS: Record<KlipySizeKey, Record<KlipyFormatKey, string>> = {
|
|
||||||
hd: {webm: 'webm', mp4: 'mp4', webp: 'webp', gif: 'gif'},
|
|
||||||
md: {webm: 'mediumwebm', mp4: 'mediummp4', webp: 'mediumwebp', gif: 'mediumgif'},
|
|
||||||
sm: {webm: 'tinywebm', mp4: 'tinymp4', webp: 'tinywebp', gif: 'tinygif'},
|
|
||||||
xs: {webm: 'nanowebm', mp4: 'nanomp4', webp: 'nanowebp', gif: 'nanogif'},
|
|
||||||
};
|
|
||||||
|
|
||||||
interface KlipyCategoryTag {
|
|
||||||
searchterm: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
function isKlipyFileEntry(value: unknown): value is KlipyFileEntry {
|
|
||||||
return (
|
|
||||||
isJsonRecord(value) &&
|
|
||||||
(value.url === undefined || typeof value.url === 'string') &&
|
|
||||||
(value.width === undefined || typeof value.width === 'number') &&
|
|
||||||
(value.height === undefined || typeof value.height === 'number') &&
|
|
||||||
(value.size === undefined || typeof value.size === 'number')
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function isKlipyFileBucket(value: unknown): value is KlipyFileBucket {
|
|
||||||
return isJsonRecord(value) && Object.values(value).every(isKlipyFileEntry);
|
|
||||||
}
|
|
||||||
|
|
||||||
function isKlipyFallbackMediaFormat(value: unknown): value is NonNullable<KlipyGif['media_formats']>['webm'] {
|
|
||||||
if (!isJsonRecord(value) || typeof value.url !== 'string' || !Array.isArray(value.dims)) return false;
|
|
||||||
return value.dims.length === 2 && value.dims.every((dimension) => typeof dimension === 'number');
|
|
||||||
}
|
|
||||||
|
|
||||||
function isKlipyGif(value: unknown): value is KlipyGif {
|
|
||||||
if (
|
|
||||||
!isJsonRecord(value) ||
|
|
||||||
typeof value.id !== 'string' ||
|
|
||||||
typeof value.title !== 'string' ||
|
|
||||||
typeof value.itemurl !== 'string'
|
|
||||||
) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return (
|
|
||||||
(value.slug === undefined || typeof value.slug === 'string') &&
|
|
||||||
(value.file === undefined || (isJsonRecord(value.file) && Object.values(value.file).every(isKlipyFileBucket))) &&
|
|
||||||
(value.media_formats === undefined ||
|
|
||||||
(isJsonRecord(value.media_formats) &&
|
|
||||||
(value.media_formats.webm === undefined || isKlipyFallbackMediaFormat(value.media_formats.webm))))
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function isKlipyCategoryTag(value: unknown): value is KlipyCategoryTag {
|
|
||||||
return isJsonRecord(value) && typeof value.searchterm === 'string';
|
|
||||||
}
|
|
||||||
|
|
||||||
function readResultsArray(value: unknown): Array<unknown> {
|
|
||||||
if (!isJsonRecord(value) || !Array.isArray(value.results)) {
|
|
||||||
throw new Error('KLIPY API response did not include a results array');
|
|
||||||
}
|
|
||||||
return value.results;
|
|
||||||
}
|
|
||||||
|
|
||||||
function readTagsArray(value: unknown): Array<unknown> {
|
|
||||||
if (!isJsonRecord(value) || !Array.isArray(value.tags)) {
|
|
||||||
throw new Error('KLIPY API response did not include a tags array');
|
|
||||||
}
|
|
||||||
return value.tags;
|
|
||||||
}
|
|
||||||
|
|
||||||
type CacheEntry<T> = {
|
|
||||||
data: T;
|
|
||||||
timestamp: number;
|
|
||||||
};
|
|
||||||
|
|
||||||
interface KlipyPath {
|
|
||||||
type: 'gif' | 'clip';
|
|
||||||
slug: string;
|
|
||||||
}
|
|
||||||
type GifApiKeyResolver = () => Promise<string | null>;
|
|
||||||
|
|
||||||
export class KlipyGifProvider implements IGifProvider {
|
|
||||||
readonly meta = KLIPY_PROVIDER_META;
|
|
||||||
private readonly FEATURED_CACHE_KEY = 'klipy:featured';
|
|
||||||
private readonly TRENDING_CACHE_KEY = 'klipy:trending';
|
|
||||||
private refreshingKeys: Map<string, boolean> = new Map();
|
|
||||||
|
|
||||||
constructor(
|
|
||||||
private cacheService: ICacheService,
|
|
||||||
private mediaService: IMediaService,
|
|
||||||
private apiKeyResolver: GifApiKeyResolver = async () => Config.klipy.apiKey || null,
|
|
||||||
) {}
|
|
||||||
|
|
||||||
async isAvailable(): Promise<boolean> {
|
|
||||||
return Boolean(await this.apiKeyResolver());
|
|
||||||
}
|
|
||||||
|
|
||||||
private async getApiKey(): Promise<string> {
|
|
||||||
const apiKey = await this.apiKeyResolver();
|
|
||||||
if (!apiKey) {
|
|
||||||
throw new Error('KLIPY API key is not configured');
|
|
||||||
}
|
|
||||||
return apiKey;
|
|
||||||
}
|
|
||||||
|
|
||||||
private createURL({endpoint, params}: {endpoint: string; params: Record<string, string | number | undefined>}): URL {
|
|
||||||
const url = new URL(`${KLIPY_BASE_URL}/${endpoint}`);
|
|
||||||
const defaultParams = {
|
|
||||||
client_key: CLIENT_KEY,
|
|
||||||
contentfilter: DEFAULT_CONTENT_FILTER,
|
|
||||||
...params,
|
|
||||||
};
|
|
||||||
for (const [key, value] of Object.entries(defaultParams)) {
|
|
||||||
if (value !== undefined) {
|
|
||||||
url.searchParams.append(key, value.toString());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return url;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async fetchKlipyData(url: URL): Promise<unknown> {
|
|
||||||
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
|
||||||
try {
|
|
||||||
const response = await fetch(url.toString(), {
|
|
||||||
headers: {'User-Agent': FLUXER_USER_AGENT},
|
|
||||||
signal: AbortSignal.timeout(ms('30 seconds')),
|
|
||||||
});
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(`Failed to fetch KLIPY data: ${response.statusText}`);
|
|
||||||
}
|
|
||||||
const responseText = await FetchUtils.streamToStringWithLimit(response.body, {
|
|
||||||
maxBytes: EXTERNAL_RESPONSE_LIMITS.klipyApiBytes,
|
|
||||||
headers: response.headers,
|
|
||||||
url: response.url,
|
|
||||||
description: 'KLIPY API response',
|
|
||||||
});
|
|
||||||
return parseJsonUnknown(responseText);
|
|
||||||
} catch (error) {
|
|
||||||
if (attempt < MAX_RETRIES - 1) {
|
|
||||||
const delay = BACKOFF_BASE_DELAY * 2 ** attempt;
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
||||||
} else {
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
throw new Error('Exceeded maximum retries');
|
|
||||||
}
|
|
||||||
|
|
||||||
private async fetchAndTransformGifs(url: URL): Promise<Array<GifResponse>> {
|
|
||||||
const results = readResultsArray(await this.fetchKlipyData(url)).filter(isKlipyGif);
|
|
||||||
return results.map((gif) => this.transformKlipyGif(gif)).filter((gif): gif is GifResponse => gif !== null);
|
|
||||||
}
|
|
||||||
|
|
||||||
private async getCache<T>(key: string): Promise<{
|
|
||||||
data: T;
|
|
||||||
isStale: boolean;
|
|
||||||
} | null> {
|
|
||||||
const cached = await this.cacheService.get<CacheEntry<T>>(key);
|
|
||||||
if (!cached) return null;
|
|
||||||
const age = Date.now() - cached.timestamp;
|
|
||||||
const isStale = age > CACHE_EXPIRATION_TIME;
|
|
||||||
return {data: cached.data, isStale};
|
|
||||||
}
|
|
||||||
|
|
||||||
private async setCache<T>(key: string, data: T): Promise<void> {
|
|
||||||
const cacheEntry: CacheEntry<T> = {
|
|
||||||
data,
|
|
||||||
timestamp: Date.now(),
|
|
||||||
};
|
|
||||||
await this.cacheService.set(key, cacheEntry);
|
|
||||||
}
|
|
||||||
|
|
||||||
private triggerBackgroundRefresh<T>(key: string, refreshFn: () => Promise<T>): void {
|
|
||||||
if (this.refreshingKeys.get(key)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
this.refreshingKeys.set(key, true);
|
|
||||||
setImmediate(async () => {
|
|
||||||
try {
|
|
||||||
const freshData = await refreshFn();
|
|
||||||
await this.setCache(key, freshData);
|
|
||||||
} catch (error) {
|
|
||||||
Logger.debug({key, error}, `Background refresh failed for key ${key}`);
|
|
||||||
} finally {
|
|
||||||
this.refreshingKeys.delete(key);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async search(params: {q: string; locale: string; country: string}): Promise<Array<GifResponse>> {
|
|
||||||
const apiKey = await this.getApiKey();
|
|
||||||
const url = this.createURL({
|
|
||||||
endpoint: 'search',
|
|
||||||
params: {
|
|
||||||
key: apiKey,
|
|
||||||
q: params.q,
|
|
||||||
country: params.country,
|
|
||||||
locale: params.locale,
|
|
||||||
limit: 50,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
return this.fetchAndTransformGifs(url);
|
|
||||||
}
|
|
||||||
|
|
||||||
async registerShare(params: {id: string; q: string; locale: string; country: string}): Promise<void> {
|
|
||||||
const apiKey = await this.getApiKey();
|
|
||||||
const url = this.createURL({
|
|
||||||
endpoint: 'registershare',
|
|
||||||
params: {
|
|
||||||
key: apiKey,
|
|
||||||
id: params.id,
|
|
||||||
country: params.country,
|
|
||||||
locale: params.locale,
|
|
||||||
q: params.q,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
await fetch(url.toString(), {
|
|
||||||
headers: {'User-Agent': FLUXER_USER_AGENT},
|
|
||||||
signal: AbortSignal.timeout(ms('30 seconds')),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async getFeatured(params: {locale: string; country: string}): Promise<{
|
|
||||||
gifs: Array<GifResponse>;
|
|
||||||
categories: Array<GifCategoryTagResponse>;
|
|
||||||
}> {
|
|
||||||
const cached = await this.getCache<{
|
|
||||||
gifs: Array<GifResponse>;
|
|
||||||
categories: Array<GifCategoryTagResponse>;
|
|
||||||
}>(this.FEATURED_CACHE_KEY);
|
|
||||||
if (cached) {
|
|
||||||
if (cached.isStale) {
|
|
||||||
this.triggerBackgroundRefresh(this.FEATURED_CACHE_KEY, () => this.fetchFeaturedData(params));
|
|
||||||
}
|
|
||||||
return cached.data;
|
|
||||||
}
|
|
||||||
const data = await this.fetchFeaturedData(params);
|
|
||||||
await this.setCache(this.FEATURED_CACHE_KEY, data);
|
|
||||||
return data;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async fetchFeaturedData(params: {locale: string; country: string}): Promise<{
|
|
||||||
gifs: Array<GifResponse>;
|
|
||||||
categories: Array<GifCategoryTagResponse>;
|
|
||||||
}> {
|
|
||||||
const [gifs, categories] = await Promise.all([this.getFeaturedGifs(params), this.getFeaturedCategories(params)]);
|
|
||||||
return {gifs, categories};
|
|
||||||
}
|
|
||||||
|
|
||||||
async getTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
|
|
||||||
const cached = await this.getCache<Array<GifResponse>>(this.TRENDING_CACHE_KEY);
|
|
||||||
if (cached) {
|
|
||||||
if (cached.isStale) {
|
|
||||||
this.triggerBackgroundRefresh(this.TRENDING_CACHE_KEY, () => this.fetchTrendingGifs(params));
|
|
||||||
}
|
|
||||||
return cached.data;
|
|
||||||
}
|
|
||||||
const gifs = await this.fetchTrendingGifs(params);
|
|
||||||
await this.setCache(this.TRENDING_CACHE_KEY, gifs);
|
|
||||||
return gifs;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async fetchTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
|
|
||||||
const apiKey = await this.getApiKey();
|
|
||||||
const url = this.createURL({
|
|
||||||
endpoint: 'featured',
|
|
||||||
params: {
|
|
||||||
key: apiKey,
|
|
||||||
country: params.country,
|
|
||||||
locale: params.locale,
|
|
||||||
limit: 50,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
return this.fetchAndTransformGifs(url);
|
|
||||||
}
|
|
||||||
|
|
||||||
async suggest(params: {q: string; locale: string}): Promise<Array<string>> {
|
|
||||||
const apiKey = await this.getApiKey();
|
|
||||||
const url = this.createURL({
|
|
||||||
endpoint: 'autocomplete',
|
|
||||||
params: {
|
|
||||||
key: apiKey,
|
|
||||||
q: params.q,
|
|
||||||
locale: params.locale,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
return readResultsArray(await this.fetchKlipyData(url)).filter(
|
|
||||||
(result): result is string => typeof result === 'string',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async resolveByUrl(params: {url: string; locale: string; country: string}): Promise<GifResponse | null> {
|
|
||||||
const slug = this.extractSlugFromUrl(params.url);
|
|
||||||
if (!slug) return null;
|
|
||||||
const results = await this.search({q: slug, locale: params.locale, country: params.country});
|
|
||||||
return results.find((gif) => gif.slug === slug || this.extractSlugFromUrl(gif.url) === slug) ?? results[0] ?? null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async getFeaturedGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
|
|
||||||
const apiKey = await this.getApiKey();
|
|
||||||
const url = this.createURL({
|
|
||||||
endpoint: 'featured',
|
|
||||||
params: {
|
|
||||||
key: apiKey,
|
|
||||||
country: params.country,
|
|
||||||
locale: params.locale,
|
|
||||||
limit: 1,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
return this.fetchAndTransformGifs(url);
|
|
||||||
}
|
|
||||||
|
|
||||||
private async getFeaturedCategories(params: {
|
|
||||||
locale: string;
|
|
||||||
country: string;
|
|
||||||
}): Promise<Array<GifCategoryTagResponse>> {
|
|
||||||
trackSeenLocale(this.cacheService, this.meta.name, params.locale, params.country).catch((error) => {
|
|
||||||
Logger.debug({err: error, ...params}, 'Failed to track seen GIF locale');
|
|
||||||
});
|
|
||||||
const rawTags = await this.fetchRawCategoryTags(params);
|
|
||||||
const cached = await readEnrichedCategoriesCache(this.cacheService, this.meta.name, params.locale, params.country);
|
|
||||||
if (cached) {
|
|
||||||
if (cached.isStale) {
|
|
||||||
this.scheduleEnrichmentRefresh(params);
|
|
||||||
}
|
|
||||||
const byName = new Map(cached.data.map((entry) => [entry.name, entry]));
|
|
||||||
return rawTags.map((tag) => byName.get(tag.name) ?? tag);
|
|
||||||
}
|
|
||||||
this.scheduleEnrichmentRefresh(params);
|
|
||||||
return rawTags;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async fetchRawCategoryTags(params: {
|
|
||||||
locale: string;
|
|
||||||
country: string;
|
|
||||||
}): Promise<Array<GifCategoryTagResponse>> {
|
|
||||||
const apiKey = await this.getApiKey();
|
|
||||||
const url = this.createURL({
|
|
||||||
endpoint: 'categories',
|
|
||||||
params: {
|
|
||||||
key: apiKey,
|
|
||||||
country: params.country,
|
|
||||||
locale: params.locale,
|
|
||||||
type: 'featured',
|
|
||||||
},
|
|
||||||
});
|
|
||||||
const tags = readTagsArray(await this.fetchKlipyData(url)).filter(isKlipyCategoryTag);
|
|
||||||
return tags
|
|
||||||
.filter((tag) => Boolean(tag.searchterm))
|
|
||||||
.map((tag) => ({
|
|
||||||
name: tag.searchterm,
|
|
||||||
src: '',
|
|
||||||
proxy_src: '',
|
|
||||||
gif: null,
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
private scheduleEnrichmentRefresh(params: {locale: string; country: string}): void {
|
|
||||||
(async () => {
|
|
||||||
try {
|
|
||||||
const workerService = getWorkerService();
|
|
||||||
await workerService.addJob('refreshGifFeaturedCategories', {
|
|
||||||
provider: this.meta.name,
|
|
||||||
locale: params.locale,
|
|
||||||
country: params.country,
|
|
||||||
});
|
|
||||||
} catch (error) {
|
|
||||||
Logger.debug({err: error, ...params}, 'Failed to enqueue GIF featured-categories refresh');
|
|
||||||
}
|
|
||||||
})();
|
|
||||||
}
|
|
||||||
|
|
||||||
async refreshFeaturedCategories(params: {locale: string; country: string}): Promise<void> {
|
|
||||||
const lockKey = refreshLockKey(this.meta.name, params.locale, params.country);
|
|
||||||
const token = await this.cacheService.acquireLock(lockKey, REFRESH_LOCK_TTL_SECONDS);
|
|
||||||
if (!token) {
|
|
||||||
Logger.debug(params, 'Skipping enriched GIF categories refresh; another worker holds the lock');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
const rawTags = await this.fetchRawCategoryTags(params);
|
|
||||||
const enriched = await Promise.all(
|
|
||||||
rawTags.map(async (tag) => {
|
|
||||||
try {
|
|
||||||
const [gif] = await this.search({q: tag.name, locale: params.locale, country: params.country});
|
|
||||||
if (!gif) return tag;
|
|
||||||
return {
|
|
||||||
...tag,
|
|
||||||
src: gif.src,
|
|
||||||
proxy_src: gif.proxy_src,
|
|
||||||
gif,
|
|
||||||
};
|
|
||||||
} catch (error) {
|
|
||||||
Logger.debug({err: error, tag: tag.name, ...params}, 'Failed to enrich GIF category');
|
|
||||||
return tag;
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
await writeEnrichedCategoriesCache(this.cacheService, this.meta.name, params.locale, params.country, enriched);
|
|
||||||
} catch (error) {
|
|
||||||
Logger.warn({err: error, ...params}, 'Failed to refresh enriched GIF categories');
|
|
||||||
throw error;
|
|
||||||
} finally {
|
|
||||||
await this.cacheService.releaseLock(lockKey, token).catch(() => undefined);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private parseKlipyPath(url: string): KlipyPath | null {
|
|
||||||
try {
|
|
||||||
const parsed = new URL(url);
|
|
||||||
const hostname = parsed.hostname.toLowerCase();
|
|
||||||
if (hostname !== 'klipy.com' && hostname !== 'www.klipy.com') return null;
|
|
||||||
const match = parsed.pathname.match(/^\/(gif|gifs|clip|clips)\/([^/]+)/i);
|
|
||||||
if (!match?.[1] || !match[2]) return null;
|
|
||||||
const type = match[1].toLowerCase().startsWith('clip') ? 'clip' : 'gif';
|
|
||||||
const slug = decodeURIComponent(match[2]).trim();
|
|
||||||
return slug ? {type, slug} : null;
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private buildKlipyPageUrl(path: KlipyPath): string {
|
|
||||||
const basePath = path.type === 'clip' ? 'clips' : 'gifs';
|
|
||||||
return `https://klipy.com/${basePath}/${encodeURIComponent(path.slug)}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
private toMediaFormat(entry: KlipyFileEntry | undefined): GifMediaFormat | null {
|
|
||||||
if (!entry?.url) return null;
|
|
||||||
const width = typeof entry.width === 'number' && entry.width > 0 ? entry.width : 0;
|
|
||||||
const height = typeof entry.height === 'number' && entry.height > 0 ? entry.height : 0;
|
|
||||||
if (!width || !height) return null;
|
|
||||||
return {
|
|
||||||
src: entry.url,
|
|
||||||
proxy_src: this.mediaService.getExternalMediaProxyURL(entry.url),
|
|
||||||
width,
|
|
||||||
height,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private collectKlipyMedia(input: KlipyGif): {
|
|
||||||
media: Record<string, GifMediaFormat>;
|
|
||||||
preferred: GifMediaFormat | null;
|
|
||||||
} {
|
|
||||||
const media: Record<string, GifMediaFormat> = {};
|
|
||||||
let preferred: GifMediaFormat | null = null;
|
|
||||||
for (const size of KLIPY_SIZE_PREFERENCE) {
|
|
||||||
const bucket = input.file?.[size];
|
|
||||||
if (!bucket) continue;
|
|
||||||
for (const format of KLIPY_FORMAT_KEYS) {
|
|
||||||
const entry = this.toMediaFormat(bucket[format]);
|
|
||||||
if (!entry) continue;
|
|
||||||
const publicKey = KLIPY_PUBLIC_FORMAT_KEYS[size][format];
|
|
||||||
media[publicKey] = entry;
|
|
||||||
if (!preferred) preferred = entry;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (Object.keys(media).length === 0 && input.media_formats?.webm) {
|
|
||||||
const webm = input.media_formats.webm;
|
|
||||||
const fallback: GifMediaFormat = {
|
|
||||||
src: webm.url,
|
|
||||||
proxy_src: this.mediaService.getExternalMediaProxyURL(webm.url),
|
|
||||||
width: webm.dims[0],
|
|
||||||
height: webm.dims[1],
|
|
||||||
};
|
|
||||||
media.webm = fallback;
|
|
||||||
preferred = fallback;
|
|
||||||
}
|
|
||||||
return {media, preferred};
|
|
||||||
}
|
|
||||||
|
|
||||||
private transformKlipyGif(input: KlipyGif): GifResponse | null {
|
|
||||||
const parsedPath = this.parseKlipyPath(input.itemurl);
|
|
||||||
const explicitSlug = input.slug?.trim();
|
|
||||||
const normalizedSlug = explicitSlug || parsedPath?.slug || input.id;
|
|
||||||
const normalizedType = parsedPath?.type ?? 'gif';
|
|
||||||
const normalizedUrl =
|
|
||||||
parsedPath || explicitSlug ? this.buildKlipyPageUrl({type: normalizedType, slug: normalizedSlug}) : input.itemurl;
|
|
||||||
const {media, preferred} = this.collectKlipyMedia(input);
|
|
||||||
const top = media.webm ?? preferred;
|
|
||||||
if (!top) return null;
|
|
||||||
return {
|
|
||||||
id: normalizedSlug,
|
|
||||||
slug: normalizedSlug,
|
|
||||||
provider: this.meta.name,
|
|
||||||
title: input.title,
|
|
||||||
url: normalizedUrl,
|
|
||||||
src: top.src,
|
|
||||||
proxy_src: top.proxy_src,
|
|
||||||
width: top.width,
|
|
||||||
height: top.height,
|
|
||||||
media,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
extractSlugFromUrl(url: string): string | null {
|
|
||||||
return this.parseKlipyPath(url)?.slug ?? null;
|
|
||||||
}
|
|
||||||
|
|
||||||
buildShareUrl(slug: string): string {
|
|
||||||
const trimmed = slug.trim();
|
|
||||||
if (!trimmed) return 'https://klipy.com/gifs';
|
|
||||||
return this.buildKlipyPageUrl({type: 'gif', slug: trimmed});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
import {describe, expect, it} from 'vitest';
|
||||||
|
import {buildKlipyShareUrl, extractKlipySlugFromUrl} from './NatsGifProvider';
|
||||||
|
|
||||||
|
describe('NatsGifProvider KLIPY URL helpers', () => {
|
||||||
|
it('extracts GIF and clip slugs from KLIPY share URLs', () => {
|
||||||
|
expect(extractKlipySlugFromUrl('https://klipy.com/gifs/goatplaybanjo-chat-4')).toBe('goatplaybanjo-chat-4');
|
||||||
|
expect(extractKlipySlugFromUrl('https://www.klipy.com/gif/funny-123')).toBe('funny-123');
|
||||||
|
expect(extractKlipySlugFromUrl('https://klipy.com/clips/clip-123')).toBe('clip-123');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects non-KLIPY and unsupported URLs', () => {
|
||||||
|
expect(extractKlipySlugFromUrl('https://notklipy.com/gifs/funny-123')).toBeNull();
|
||||||
|
expect(extractKlipySlugFromUrl('https://klipy.com/search/funny-123')).toBeNull();
|
||||||
|
expect(extractKlipySlugFromUrl('not a url')).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('builds canonical GIF share URLs locally', () => {
|
||||||
|
expect(buildKlipyShareUrl('goatplaybanjo-chat-4')).toBe('https://klipy.com/gifs/goatplaybanjo-chat-4');
|
||||||
|
expect(buildKlipyShareUrl(' ')).toBe('https://klipy.com/gifs');
|
||||||
|
expect(buildKlipyShareUrl('a slug')).toBe('https://klipy.com/gifs/a%20slug');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,290 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';
|
||||||
|
import {ServiceUnavailableError} from '@fluxer/errors/src/domains/core/ServiceUnavailableError';
|
||||||
|
import {
|
||||||
|
type GifCategoryTagResponse,
|
||||||
|
GifCategoryTagResponse as GifCategoryTagResponseSchema,
|
||||||
|
type GifResponse,
|
||||||
|
GifResponse as GifResponseSchema,
|
||||||
|
} from '@fluxer/schema/src/domains/gif/GifSchemas';
|
||||||
|
import type {INatsConnectionManager} from '@pkgs/nats/src/INatsConnectionManager';
|
||||||
|
import {NatsConnectionManager} from '@pkgs/nats/src/NatsConnectionManager';
|
||||||
|
import {StringCodec} from 'nats';
|
||||||
|
import {Config} from '../Config';
|
||||||
|
import {Logger} from '../Logger';
|
||||||
|
import {isJsonRecord, parseJsonUnknown} from '../utils/JsonBoundaryUtils';
|
||||||
|
import type {GifProviderMeta, IGifProvider} from './IGifProvider';
|
||||||
|
|
||||||
|
const GIF_SERVICE_SUBJECT = process.env.FLUXER_GIF_SERVICE_SUBJECT || 'svc.gifs';
|
||||||
|
const DEFAULT_GIF_SERVICE_TIMEOUT_MS = 12_000;
|
||||||
|
const DEFAULT_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS = 3_000;
|
||||||
|
const GIF_PROVIDER_META: GifProviderMeta = {
|
||||||
|
name: 'klipy',
|
||||||
|
displayName: 'KLIPY',
|
||||||
|
attributionRequired: true,
|
||||||
|
};
|
||||||
|
const KLIPY_SHARE_ORIGIN = 'https://klipy.com';
|
||||||
|
const KLIPY_SHARE_HOSTS = new Set(['klipy.com', 'www.klipy.com']);
|
||||||
|
|
||||||
|
type GifApiKeyResolver = () => Promise<string | null>;
|
||||||
|
|
||||||
|
type NatsGifRequest =
|
||||||
|
| {op: 'IsAvailable'; api_key: string | null}
|
||||||
|
| {op: 'Search'; api_key: string; q: string; locale: string; country: string}
|
||||||
|
| {op: 'GetFeatured'; api_key: string; locale: string; country: string}
|
||||||
|
| {op: 'GetTrendingGifs'; api_key: string; locale: string; country: string}
|
||||||
|
| {op: 'Suggest'; api_key: string; q: string; locale: string}
|
||||||
|
| {op: 'RegisterShare'; api_key: string; id: string; q: string; locale: string; country: string}
|
||||||
|
| {op: 'ResolveByUrl'; api_key: string; url: string; locale: string; country: string};
|
||||||
|
|
||||||
|
export function extractKlipySlugFromUrl(rawUrl: string): string | null {
|
||||||
|
let parsed: URL;
|
||||||
|
try {
|
||||||
|
parsed = new URL(rawUrl);
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (!KLIPY_SHARE_HOSTS.has(parsed.hostname.toLowerCase())) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const segments = parsed.pathname.split('/').filter(Boolean);
|
||||||
|
const kind = segments[0]?.toLowerCase();
|
||||||
|
const slug = segments[1]?.trim();
|
||||||
|
if (!slug) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
switch (kind) {
|
||||||
|
case 'gif':
|
||||||
|
case 'gifs':
|
||||||
|
case 'clip':
|
||||||
|
case 'clips':
|
||||||
|
return slug;
|
||||||
|
default:
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildKlipyShareUrl(slug: string): string {
|
||||||
|
const trimmed = slug.trim();
|
||||||
|
if (!trimmed) {
|
||||||
|
return `${KLIPY_SHARE_ORIGIN}/gifs`;
|
||||||
|
}
|
||||||
|
return `${KLIPY_SHARE_ORIGIN}/gifs/${encodeURIComponent(trimmed)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function readPositiveIntegerEnv(name: string, fallback: number): number {
|
||||||
|
const value = process.env[name];
|
||||||
|
if (!value) return fallback;
|
||||||
|
const parsed = Number(value);
|
||||||
|
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
function readFailedMessage(value: unknown): string | null {
|
||||||
|
if (!isJsonRecord(value)) return null;
|
||||||
|
if (typeof value.error === 'string') return value.error;
|
||||||
|
if (!('Failed' in value)) return null;
|
||||||
|
const failed = value.Failed;
|
||||||
|
return isJsonRecord(failed) && typeof failed.message === 'string' ? failed.message : 'GIF service failed';
|
||||||
|
}
|
||||||
|
|
||||||
|
function readVariant(value: unknown, variant: string): unknown {
|
||||||
|
if (!isJsonRecord(value) || !(variant in value)) {
|
||||||
|
throw new ServiceUnavailableError({message: `GIF service returned an unexpected ${variant} response`});
|
||||||
|
}
|
||||||
|
return value[variant];
|
||||||
|
}
|
||||||
|
|
||||||
|
function readGifList(value: unknown, variant: string): Array<GifResponse> {
|
||||||
|
const parsed = GifResponseSchema.array().safeParse(readVariant(value, variant));
|
||||||
|
if (!parsed.success) {
|
||||||
|
throw new ServiceUnavailableError({message: `GIF service returned invalid ${variant} data`});
|
||||||
|
}
|
||||||
|
return parsed.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
function readFeatured(value: unknown): {
|
||||||
|
gifs: Array<GifResponse>;
|
||||||
|
categories: Array<GifCategoryTagResponse>;
|
||||||
|
} {
|
||||||
|
const featured = readVariant(value, 'Featured');
|
||||||
|
if (!isJsonRecord(featured)) {
|
||||||
|
throw new ServiceUnavailableError({message: 'GIF service returned invalid featured data'});
|
||||||
|
}
|
||||||
|
const gifs = GifResponseSchema.array().safeParse(featured.gifs);
|
||||||
|
const categories = GifCategoryTagResponseSchema.array().safeParse(featured.categories);
|
||||||
|
if (!gifs.success || !categories.success) {
|
||||||
|
throw new ServiceUnavailableError({message: 'GIF service returned invalid featured data'});
|
||||||
|
}
|
||||||
|
return {gifs: gifs.data, categories: categories.data};
|
||||||
|
}
|
||||||
|
|
||||||
|
function readSuggestions(value: unknown): Array<string> {
|
||||||
|
const suggestions = readVariant(value, 'Suggestions');
|
||||||
|
if (!Array.isArray(suggestions) || !suggestions.every((suggestion) => typeof suggestion === 'string')) {
|
||||||
|
throw new ServiceUnavailableError({message: 'GIF service returned invalid suggestions'});
|
||||||
|
}
|
||||||
|
return suggestions;
|
||||||
|
}
|
||||||
|
|
||||||
|
function readResolved(value: unknown): GifResponse | null {
|
||||||
|
const resolved = readVariant(value, 'Resolved');
|
||||||
|
if (!isJsonRecord(resolved)) {
|
||||||
|
throw new ServiceUnavailableError({message: 'GIF service returned invalid resolved data'});
|
||||||
|
}
|
||||||
|
if (resolved.gif === null || resolved.gif === undefined) return null;
|
||||||
|
const parsed = GifResponseSchema.safeParse(resolved.gif);
|
||||||
|
if (!parsed.success) {
|
||||||
|
throw new ServiceUnavailableError({message: 'GIF service returned invalid resolved GIF'});
|
||||||
|
}
|
||||||
|
return parsed.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
class NatsGifProvider implements IGifProvider {
|
||||||
|
readonly meta = GIF_PROVIDER_META;
|
||||||
|
private readonly codec = StringCodec();
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly connectionManager: INatsConnectionManager,
|
||||||
|
private readonly apiKeyResolver: GifApiKeyResolver,
|
||||||
|
private readonly requestTimeoutMs = readPositiveIntegerEnv(
|
||||||
|
'FLUXER_GIF_SERVICE_TIMEOUT_MS',
|
||||||
|
DEFAULT_GIF_SERVICE_TIMEOUT_MS,
|
||||||
|
),
|
||||||
|
private readonly registerShareTimeoutMs = readPositiveIntegerEnv(
|
||||||
|
'FLUXER_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS',
|
||||||
|
DEFAULT_GIF_SERVICE_REGISTER_SHARE_TIMEOUT_MS,
|
||||||
|
),
|
||||||
|
private readonly subject = GIF_SERVICE_SUBJECT,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
async isAvailable(): Promise<boolean> {
|
||||||
|
return Boolean((await this.apiKeyResolver())?.trim());
|
||||||
|
}
|
||||||
|
|
||||||
|
async search(params: {q: string; locale: string; country: string}): Promise<Array<GifResponse>> {
|
||||||
|
const response = await this.request({
|
||||||
|
op: 'Search',
|
||||||
|
api_key: await this.getApiKey(),
|
||||||
|
q: params.q,
|
||||||
|
locale: params.locale,
|
||||||
|
country: params.country,
|
||||||
|
});
|
||||||
|
return readGifList(response, 'SearchResults');
|
||||||
|
}
|
||||||
|
|
||||||
|
async registerShare(params: {id: string; q: string; locale: string; country: string}): Promise<void> {
|
||||||
|
const response = await this.request(
|
||||||
|
{
|
||||||
|
op: 'RegisterShare',
|
||||||
|
api_key: await this.getApiKey(),
|
||||||
|
id: params.id,
|
||||||
|
q: params.q,
|
||||||
|
locale: params.locale,
|
||||||
|
country: params.country,
|
||||||
|
},
|
||||||
|
this.registerShareTimeoutMs,
|
||||||
|
);
|
||||||
|
if (response !== 'Registered') {
|
||||||
|
throw new ServiceUnavailableError({message: 'GIF service returned an unexpected register-share response'});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async getFeatured(params: {locale: string; country: string}): Promise<{
|
||||||
|
gifs: Array<GifResponse>;
|
||||||
|
categories: Array<GifCategoryTagResponse>;
|
||||||
|
}> {
|
||||||
|
return readFeatured(
|
||||||
|
await this.request({
|
||||||
|
op: 'GetFeatured',
|
||||||
|
api_key: await this.getApiKey(),
|
||||||
|
locale: params.locale,
|
||||||
|
country: params.country,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
|
||||||
|
const response = await this.request({
|
||||||
|
op: 'GetTrendingGifs',
|
||||||
|
api_key: await this.getApiKey(),
|
||||||
|
locale: params.locale,
|
||||||
|
country: params.country,
|
||||||
|
});
|
||||||
|
return readGifList(response, 'TrendingResults');
|
||||||
|
}
|
||||||
|
|
||||||
|
async suggest(params: {q: string; locale: string}): Promise<Array<string>> {
|
||||||
|
return readSuggestions(
|
||||||
|
await this.request({
|
||||||
|
op: 'Suggest',
|
||||||
|
api_key: await this.getApiKey(),
|
||||||
|
q: params.q,
|
||||||
|
locale: params.locale,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async resolveByUrl(params: {url: string; locale: string; country: string}): Promise<GifResponse | null> {
|
||||||
|
return readResolved(
|
||||||
|
await this.request({
|
||||||
|
op: 'ResolveByUrl',
|
||||||
|
api_key: await this.getApiKey(),
|
||||||
|
url: params.url,
|
||||||
|
locale: params.locale,
|
||||||
|
country: params.country,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
buildShareUrl(slug: string): string {
|
||||||
|
return buildKlipyShareUrl(slug);
|
||||||
|
}
|
||||||
|
|
||||||
|
extractSlugFromUrl(url: string): string | null {
|
||||||
|
return extractKlipySlugFromUrl(url);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async getApiKey(): Promise<string> {
|
||||||
|
const apiKey = (await this.apiKeyResolver())?.trim();
|
||||||
|
if (!apiKey) {
|
||||||
|
throw new FeatureTemporarilyDisabledError();
|
||||||
|
}
|
||||||
|
return apiKey;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async request(payload: NatsGifRequest, timeout = this.requestTimeoutMs): Promise<unknown> {
|
||||||
|
try {
|
||||||
|
if (this.connectionManager.isClosed()) {
|
||||||
|
await this.connectionManager.connect();
|
||||||
|
}
|
||||||
|
const connection = this.connectionManager.getConnection();
|
||||||
|
const response = await connection.request(this.subject, this.codec.encode(JSON.stringify(payload)), {timeout});
|
||||||
|
const decoded = this.codec.decode(response.data);
|
||||||
|
const parsed = parseJsonUnknown(decoded);
|
||||||
|
const failedMessage = readFailedMessage(parsed);
|
||||||
|
if (failedMessage) {
|
||||||
|
throw new ServiceUnavailableError({message: failedMessage});
|
||||||
|
}
|
||||||
|
return parsed;
|
||||||
|
} catch (error) {
|
||||||
|
Logger.warn({error, op: payload.op}, '[gif-service] request failed');
|
||||||
|
if (error instanceof FeatureTemporarilyDisabledError || error instanceof ServiceUnavailableError) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
throw new ServiceUnavailableError({message: 'GIF service is temporarily unavailable'});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createNatsGifProvider(apiKeyResolver: GifApiKeyResolver): NatsGifProvider {
|
||||||
|
const manager = new NatsConnectionManager({
|
||||||
|
url: Config.nats.coreUrl,
|
||||||
|
token: Config.nats.authToken || undefined,
|
||||||
|
name: process.env.FLUXER_GIF_SERVICE_NATS_CLIENT_NAME || 'fluxer-api-gifs',
|
||||||
|
});
|
||||||
|
void manager.connect().catch((error) => {
|
||||||
|
Logger.warn({error}, '[gif-service] Failed to establish NATS connection');
|
||||||
|
});
|
||||||
|
return new NatsGifProvider(manager, apiKeyResolver);
|
||||||
|
}
|
||||||
@@ -1,523 +0,0 @@
|
|||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
import {FLUXER_USER_AGENT} from '@fluxer/constants/src/Core';
|
|
||||||
import type {GifCategoryTagResponse, GifMediaFormat, GifResponse} from '@fluxer/schema/src/domains/gif/GifSchemas';
|
|
||||||
import type {ICacheService} from '@pkgs/cache/src/ICacheService';
|
|
||||||
import {ms} from 'itty-time';
|
|
||||||
import {Config} from '../Config';
|
|
||||||
import type {IMediaService} from '../infrastructure/IMediaService';
|
|
||||||
import {Logger} from '../Logger';
|
|
||||||
import {getWorkerService} from '../middleware/ServiceRegistry';
|
|
||||||
import {EXTERNAL_RESPONSE_LIMITS} from '../utils/ExternalResponseLimits';
|
|
||||||
import * as FetchUtils from '../utils/FetchUtils';
|
|
||||||
import {isJsonRecord, parseJsonUnknown} from '../utils/JsonBoundaryUtils';
|
|
||||||
import {
|
|
||||||
REFRESH_LOCK_TTL_SECONDS,
|
|
||||||
readEnrichedCategoriesCache,
|
|
||||||
refreshLockKey,
|
|
||||||
trackSeenLocale,
|
|
||||||
writeEnrichedCategoriesCache,
|
|
||||||
} from './GifFeaturedCategoriesCache';
|
|
||||||
import type {GifProviderMeta, IGifProvider} from './IGifProvider';
|
|
||||||
|
|
||||||
const TENOR_BASE_URL = 'https://tenor.googleapis.com/v2';
|
|
||||||
const DEFAULT_MEDIA_FILTER = 'webm,mp4,webp,gif,tinywebm,tinymp4,tinygif,nanogif';
|
|
||||||
const PUBLIC_MEDIA_FORMATS = ['webm', 'mp4', 'webp', 'gif', 'tinywebm', 'tinymp4', 'tinygif', 'nanogif'] as const;
|
|
||||||
const DEFAULT_CONTENT_FILTER = 'low';
|
|
||||||
const CLIENT_KEY = 'fluxer';
|
|
||||||
const MAX_RETRIES = 3;
|
|
||||||
const BACKOFF_BASE_DELAY = ms('1 second');
|
|
||||||
const CACHE_EXPIRATION_TIME = ms('5 minutes');
|
|
||||||
const TENOR_PROVIDER_META: GifProviderMeta = {
|
|
||||||
name: 'tenor',
|
|
||||||
displayName: 'Tenor',
|
|
||||||
attributionRequired: false,
|
|
||||||
};
|
|
||||||
type GifApiKeyResolver = () => Promise<string | null>;
|
|
||||||
|
|
||||||
interface TenorMediaFormat {
|
|
||||||
url: string;
|
|
||||||
dims: [number, number];
|
|
||||||
}
|
|
||||||
|
|
||||||
interface TenorGif {
|
|
||||||
id: string;
|
|
||||||
title?: string;
|
|
||||||
content_description?: string;
|
|
||||||
media_formats?: Record<string, TenorMediaFormat>;
|
|
||||||
itemurl?: string;
|
|
||||||
url?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface TenorCategoryTag {
|
|
||||||
searchterm: string;
|
|
||||||
image: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
function isTenorMediaFormat(value: unknown): value is TenorMediaFormat {
|
|
||||||
if (!isJsonRecord(value) || typeof value.url !== 'string' || !Array.isArray(value.dims)) return false;
|
|
||||||
return value.dims.length === 2 && value.dims.every((dimension) => typeof dimension === 'number');
|
|
||||||
}
|
|
||||||
|
|
||||||
function isTenorGif(value: unknown): value is TenorGif {
|
|
||||||
if (!isJsonRecord(value) || typeof value.id !== 'string') return false;
|
|
||||||
const mediaFormats = value.media_formats;
|
|
||||||
return (
|
|
||||||
(value.title === undefined || typeof value.title === 'string') &&
|
|
||||||
(value.content_description === undefined || typeof value.content_description === 'string') &&
|
|
||||||
(mediaFormats === undefined ||
|
|
||||||
(isJsonRecord(mediaFormats) && Object.values(mediaFormats).every(isTenorMediaFormat))) &&
|
|
||||||
(value.itemurl === undefined || typeof value.itemurl === 'string') &&
|
|
||||||
(value.url === undefined || typeof value.url === 'string')
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function isTenorCategoryTag(value: unknown): value is TenorCategoryTag {
|
|
||||||
return isJsonRecord(value) && typeof value.searchterm === 'string' && typeof value.image === 'string';
|
|
||||||
}
|
|
||||||
|
|
||||||
function readResultsArray(value: unknown): Array<unknown> {
|
|
||||||
if (!isJsonRecord(value) || !Array.isArray(value.results)) {
|
|
||||||
throw new Error('Tenor API response did not include a results array');
|
|
||||||
}
|
|
||||||
return value.results;
|
|
||||||
}
|
|
||||||
|
|
||||||
function readTagsArray(value: unknown): Array<unknown> {
|
|
||||||
if (!isJsonRecord(value) || !Array.isArray(value.tags)) {
|
|
||||||
throw new Error('Tenor API response did not include a tags array');
|
|
||||||
}
|
|
||||||
return value.tags;
|
|
||||||
}
|
|
||||||
|
|
||||||
type CacheEntry<T> = {
|
|
||||||
data: T;
|
|
||||||
timestamp: number;
|
|
||||||
};
|
|
||||||
|
|
||||||
export class TenorGifProvider implements IGifProvider {
|
|
||||||
readonly meta = TENOR_PROVIDER_META;
|
|
||||||
private refreshingKeys: Map<string, boolean> = new Map();
|
|
||||||
|
|
||||||
private featuredCacheKey(locale: string, country: string): string {
|
|
||||||
return `tenor:featured:${locale}:${country}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
private trendingCacheKey(locale: string, country: string): string {
|
|
||||||
return `tenor:trending:${locale}:${country}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
constructor(
|
|
||||||
private cacheService: ICacheService,
|
|
||||||
private mediaService: IMediaService,
|
|
||||||
private apiKeyResolver: GifApiKeyResolver = async () => Config.tenor.apiKey || null,
|
|
||||||
) {}
|
|
||||||
|
|
||||||
async isAvailable(): Promise<boolean> {
|
|
||||||
return Boolean(await this.apiKeyResolver());
|
|
||||||
}
|
|
||||||
|
|
||||||
private async getApiKey(): Promise<string> {
|
|
||||||
const apiKey = await this.apiKeyResolver();
|
|
||||||
if (!apiKey) {
|
|
||||||
throw new Error('Tenor API key is not configured');
|
|
||||||
}
|
|
||||||
return apiKey;
|
|
||||||
}
|
|
||||||
|
|
||||||
private createURL({endpoint, params}: {endpoint: string; params: Record<string, string | number | undefined>}): URL {
|
|
||||||
const url = new URL(`${TENOR_BASE_URL}/${endpoint}`);
|
|
||||||
for (const [key, value] of Object.entries(params)) {
|
|
||||||
if (value !== undefined && value !== '') {
|
|
||||||
url.searchParams.append(key, value.toString());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return url;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async fetchTenorData(url: URL): Promise<unknown> {
|
|
||||||
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
|
||||||
try {
|
|
||||||
const response = await fetch(url.toString(), {
|
|
||||||
headers: {'User-Agent': FLUXER_USER_AGENT},
|
|
||||||
signal: AbortSignal.timeout(ms('30 seconds')),
|
|
||||||
});
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(`Failed to fetch Tenor data: ${response.statusText}`);
|
|
||||||
}
|
|
||||||
const responseText = await FetchUtils.streamToStringWithLimit(response.body, {
|
|
||||||
maxBytes: EXTERNAL_RESPONSE_LIMITS.tenorApiBytes,
|
|
||||||
headers: response.headers,
|
|
||||||
url: response.url,
|
|
||||||
description: 'Tenor API response',
|
|
||||||
});
|
|
||||||
return parseJsonUnknown(responseText);
|
|
||||||
} catch (error) {
|
|
||||||
if (attempt < MAX_RETRIES - 1) {
|
|
||||||
const delay = BACKOFF_BASE_DELAY * 2 ** attempt;
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
||||||
} else {
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
throw new Error('Exceeded maximum retries');
|
|
||||||
}
|
|
||||||
|
|
||||||
private async fetchAndTransformGifs(url: URL): Promise<Array<GifResponse>> {
|
|
||||||
const results = readResultsArray(await this.fetchTenorData(url)).filter(isTenorGif);
|
|
||||||
return results.map((gif) => this.transformTenorGif(gif)).filter((gif): gif is GifResponse => gif !== null);
|
|
||||||
}
|
|
||||||
|
|
||||||
private async getCache<T>(key: string): Promise<{
|
|
||||||
data: T;
|
|
||||||
isStale: boolean;
|
|
||||||
} | null> {
|
|
||||||
const cached = await this.cacheService.get<CacheEntry<T>>(key);
|
|
||||||
if (!cached) return null;
|
|
||||||
const age = Date.now() - cached.timestamp;
|
|
||||||
const isStale = age > CACHE_EXPIRATION_TIME;
|
|
||||||
return {data: cached.data, isStale};
|
|
||||||
}
|
|
||||||
|
|
||||||
private async setCache<T>(key: string, data: T): Promise<void> {
|
|
||||||
const cacheEntry: CacheEntry<T> = {
|
|
||||||
data,
|
|
||||||
timestamp: Date.now(),
|
|
||||||
};
|
|
||||||
await this.cacheService.set(key, cacheEntry);
|
|
||||||
}
|
|
||||||
|
|
||||||
private triggerBackgroundRefresh<T>(key: string, refreshFn: () => Promise<T>): void {
|
|
||||||
if (this.refreshingKeys.get(key)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
this.refreshingKeys.set(key, true);
|
|
||||||
setImmediate(async () => {
|
|
||||||
try {
|
|
||||||
const freshData = await refreshFn();
|
|
||||||
await this.setCache(key, freshData);
|
|
||||||
} catch (error) {
|
|
||||||
Logger.debug({key, error}, `Background refresh failed for key ${key}`);
|
|
||||||
} finally {
|
|
||||||
this.refreshingKeys.delete(key);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async search(params: {q: string; locale: string; country: string}): Promise<Array<GifResponse>> {
|
|
||||||
const apiKey = await this.getApiKey();
|
|
||||||
const url = this.createURL({
|
|
||||||
endpoint: 'search',
|
|
||||||
params: {
|
|
||||||
key: apiKey,
|
|
||||||
client_key: CLIENT_KEY,
|
|
||||||
q: params.q,
|
|
||||||
country: params.country,
|
|
||||||
locale: params.locale,
|
|
||||||
contentfilter: DEFAULT_CONTENT_FILTER,
|
|
||||||
media_filter: DEFAULT_MEDIA_FILTER,
|
|
||||||
limit: 50,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
return this.fetchAndTransformGifs(url);
|
|
||||||
}
|
|
||||||
|
|
||||||
async registerShare(params: {id: string; q: string; locale: string; country: string}): Promise<void> {
|
|
||||||
const apiKey = await this.getApiKey();
|
|
||||||
const url = this.createURL({
|
|
||||||
endpoint: 'registershare',
|
|
||||||
params: {
|
|
||||||
key: apiKey,
|
|
||||||
client_key: CLIENT_KEY,
|
|
||||||
id: params.id,
|
|
||||||
country: params.country,
|
|
||||||
locale: params.locale,
|
|
||||||
q: params.q,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
await fetch(url.toString(), {
|
|
||||||
headers: {'User-Agent': FLUXER_USER_AGENT},
|
|
||||||
signal: AbortSignal.timeout(ms('30 seconds')),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async getFeatured(params: {locale: string; country: string}): Promise<{
|
|
||||||
gifs: Array<GifResponse>;
|
|
||||||
categories: Array<GifCategoryTagResponse>;
|
|
||||||
}> {
|
|
||||||
const cacheKey = this.featuredCacheKey(params.locale, params.country);
|
|
||||||
const cached = await this.getCache<{
|
|
||||||
gifs: Array<GifResponse>;
|
|
||||||
categories: Array<GifCategoryTagResponse>;
|
|
||||||
}>(cacheKey);
|
|
||||||
if (cached) {
|
|
||||||
if (cached.isStale) {
|
|
||||||
this.triggerBackgroundRefresh(cacheKey, () => this.fetchFeaturedData(params));
|
|
||||||
}
|
|
||||||
return cached.data;
|
|
||||||
}
|
|
||||||
const data = await this.fetchFeaturedData(params);
|
|
||||||
await this.setCache(cacheKey, data);
|
|
||||||
return data;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async fetchFeaturedData(params: {locale: string; country: string}): Promise<{
|
|
||||||
gifs: Array<GifResponse>;
|
|
||||||
categories: Array<GifCategoryTagResponse>;
|
|
||||||
}> {
|
|
||||||
const [gifs, categories] = await Promise.all([this.getFeaturedGifs(params), this.getFeaturedCategories(params)]);
|
|
||||||
return {gifs, categories};
|
|
||||||
}
|
|
||||||
|
|
||||||
async getTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
|
|
||||||
const cacheKey = this.trendingCacheKey(params.locale, params.country);
|
|
||||||
const cached = await this.getCache<Array<GifResponse>>(cacheKey);
|
|
||||||
if (cached) {
|
|
||||||
if (cached.isStale) {
|
|
||||||
this.triggerBackgroundRefresh(cacheKey, () => this.fetchTrendingGifs(params));
|
|
||||||
}
|
|
||||||
return cached.data;
|
|
||||||
}
|
|
||||||
const gifs = await this.fetchTrendingGifs(params);
|
|
||||||
await this.setCache(cacheKey, gifs);
|
|
||||||
return gifs;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async fetchTrendingGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
|
|
||||||
const apiKey = await this.getApiKey();
|
|
||||||
const url = this.createURL({
|
|
||||||
endpoint: 'featured',
|
|
||||||
params: {
|
|
||||||
key: apiKey,
|
|
||||||
client_key: CLIENT_KEY,
|
|
||||||
country: params.country,
|
|
||||||
locale: params.locale,
|
|
||||||
contentfilter: DEFAULT_CONTENT_FILTER,
|
|
||||||
media_filter: DEFAULT_MEDIA_FILTER,
|
|
||||||
limit: 50,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
return this.fetchAndTransformGifs(url);
|
|
||||||
}
|
|
||||||
|
|
||||||
async suggest(params: {q: string; locale: string}): Promise<Array<string>> {
|
|
||||||
const apiKey = await this.getApiKey();
|
|
||||||
const url = this.createURL({
|
|
||||||
endpoint: 'search_suggestions',
|
|
||||||
params: {
|
|
||||||
key: apiKey,
|
|
||||||
client_key: CLIENT_KEY,
|
|
||||||
q: params.q,
|
|
||||||
locale: params.locale,
|
|
||||||
limit: 20,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
return readResultsArray(await this.fetchTenorData(url)).filter(
|
|
||||||
(result): result is string => typeof result === 'string',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async resolveByUrl(params: {url: string; locale: string; country: string}): Promise<GifResponse | null> {
|
|
||||||
const slug = this.extractSlugFromUrl(params.url);
|
|
||||||
if (!slug) return null;
|
|
||||||
const id = this.extractIdFromSlug(slug);
|
|
||||||
if (!id) return null;
|
|
||||||
const apiKey = await this.getApiKey();
|
|
||||||
const url = this.createURL({
|
|
||||||
endpoint: 'posts',
|
|
||||||
params: {
|
|
||||||
key: apiKey,
|
|
||||||
client_key: CLIENT_KEY,
|
|
||||||
ids: id,
|
|
||||||
country: params.country,
|
|
||||||
locale: params.locale,
|
|
||||||
media_filter: DEFAULT_MEDIA_FILTER,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
const [gif] = await this.fetchAndTransformGifs(url);
|
|
||||||
return gif ?? null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async getFeaturedGifs(params: {locale: string; country: string}): Promise<Array<GifResponse>> {
|
|
||||||
const apiKey = await this.getApiKey();
|
|
||||||
const url = this.createURL({
|
|
||||||
endpoint: 'featured',
|
|
||||||
params: {
|
|
||||||
key: apiKey,
|
|
||||||
client_key: CLIENT_KEY,
|
|
||||||
country: params.country,
|
|
||||||
locale: params.locale,
|
|
||||||
contentfilter: DEFAULT_CONTENT_FILTER,
|
|
||||||
media_filter: DEFAULT_MEDIA_FILTER,
|
|
||||||
limit: 1,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
return this.fetchAndTransformGifs(url);
|
|
||||||
}
|
|
||||||
|
|
||||||
private async getFeaturedCategories(params: {
|
|
||||||
locale: string;
|
|
||||||
country: string;
|
|
||||||
}): Promise<Array<GifCategoryTagResponse>> {
|
|
||||||
trackSeenLocale(this.cacheService, this.meta.name, params.locale, params.country).catch((error) => {
|
|
||||||
Logger.debug({err: error, ...params}, 'Failed to track seen GIF locale');
|
|
||||||
});
|
|
||||||
const rawTags = await this.fetchRawCategoryTags(params);
|
|
||||||
const cached = await readEnrichedCategoriesCache(this.cacheService, this.meta.name, params.locale, params.country);
|
|
||||||
if (cached) {
|
|
||||||
if (cached.isStale) {
|
|
||||||
this.scheduleEnrichmentRefresh(params);
|
|
||||||
}
|
|
||||||
const byName = new Map(cached.data.map((entry) => [entry.name, entry]));
|
|
||||||
return rawTags.map((tag) => byName.get(tag.name) ?? tag);
|
|
||||||
}
|
|
||||||
this.scheduleEnrichmentRefresh(params);
|
|
||||||
return rawTags;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async fetchRawCategoryTags(params: {
|
|
||||||
locale: string;
|
|
||||||
country: string;
|
|
||||||
}): Promise<Array<GifCategoryTagResponse>> {
|
|
||||||
const apiKey = await this.getApiKey();
|
|
||||||
const url = this.createURL({
|
|
||||||
endpoint: 'categories',
|
|
||||||
params: {
|
|
||||||
key: apiKey,
|
|
||||||
client_key: CLIENT_KEY,
|
|
||||||
country: params.country,
|
|
||||||
locale: params.locale,
|
|
||||||
contentfilter: DEFAULT_CONTENT_FILTER,
|
|
||||||
type: 'featured',
|
|
||||||
},
|
|
||||||
});
|
|
||||||
const tags = readTagsArray(await this.fetchTenorData(url)).filter(isTenorCategoryTag);
|
|
||||||
return tags
|
|
||||||
.filter((tag) => Boolean(tag.searchterm) && Boolean(tag.image))
|
|
||||||
.map((tag) => ({
|
|
||||||
name: tag.searchterm,
|
|
||||||
src: tag.image,
|
|
||||||
proxy_src: this.mediaService.getExternalMediaProxyURL(tag.image),
|
|
||||||
gif: null,
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
private scheduleEnrichmentRefresh(params: {locale: string; country: string}): void {
|
|
||||||
(async () => {
|
|
||||||
try {
|
|
||||||
const workerService = getWorkerService();
|
|
||||||
await workerService.addJob('refreshGifFeaturedCategories', {
|
|
||||||
provider: this.meta.name,
|
|
||||||
locale: params.locale,
|
|
||||||
country: params.country,
|
|
||||||
});
|
|
||||||
} catch (error) {
|
|
||||||
Logger.debug({err: error, ...params}, 'Failed to enqueue GIF featured-categories refresh');
|
|
||||||
}
|
|
||||||
})();
|
|
||||||
}
|
|
||||||
|
|
||||||
async refreshFeaturedCategories(params: {locale: string; country: string}): Promise<void> {
|
|
||||||
const lockKey = refreshLockKey(this.meta.name, params.locale, params.country);
|
|
||||||
const token = await this.cacheService.acquireLock(lockKey, REFRESH_LOCK_TTL_SECONDS);
|
|
||||||
if (!token) {
|
|
||||||
Logger.debug(params, 'Skipping enriched GIF categories refresh; another worker holds the lock');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
const rawTags = await this.fetchRawCategoryTags(params);
|
|
||||||
const enriched = await Promise.all(
|
|
||||||
rawTags.map(async (tag) => {
|
|
||||||
try {
|
|
||||||
const [gif] = await this.search({q: tag.name, locale: params.locale, country: params.country});
|
|
||||||
return {...tag, gif: gif ?? null};
|
|
||||||
} catch (error) {
|
|
||||||
Logger.debug({err: error, tag: tag.name, ...params}, 'Failed to enrich GIF category');
|
|
||||||
return tag;
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
await writeEnrichedCategoriesCache(this.cacheService, this.meta.name, params.locale, params.country, enriched);
|
|
||||||
} catch (error) {
|
|
||||||
Logger.warn({err: error, ...params}, 'Failed to refresh enriched GIF categories');
|
|
||||||
throw error;
|
|
||||||
} finally {
|
|
||||||
await this.cacheService.releaseLock(lockKey, token).catch(() => undefined);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private selectMediaFormat(mediaFormats: Record<string, TenorMediaFormat> | undefined): TenorMediaFormat | null {
|
|
||||||
if (!mediaFormats) return null;
|
|
||||||
const preferredKeys = ['webm', 'mp4', 'tinywebm', 'tinymp4', 'webp', 'gif', 'tinygif', 'nanogif'];
|
|
||||||
for (const key of preferredKeys) {
|
|
||||||
const candidate = mediaFormats[key];
|
|
||||||
if (candidate?.url && candidate.dims?.length === 2) return candidate;
|
|
||||||
}
|
|
||||||
for (const candidate of Object.values(mediaFormats)) {
|
|
||||||
if (candidate?.url && candidate.dims?.length === 2) return candidate;
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private transformTenorGif(input: TenorGif): GifResponse | null {
|
|
||||||
const best = this.selectMediaFormat(input.media_formats);
|
|
||||||
if (!best) return null;
|
|
||||||
const title = input.title?.trim() || input.content_description?.trim() || '';
|
|
||||||
const fallbackUrl = `https://tenor.com/view/${encodeURIComponent(input.id)}`;
|
|
||||||
const url = input.itemurl?.trim() || input.url?.trim() || fallbackUrl;
|
|
||||||
const slug = this.extractSlugFromUrl(url) ?? `view/${input.id}`;
|
|
||||||
const media: Record<string, GifMediaFormat> = {};
|
|
||||||
for (const key of PUBLIC_MEDIA_FORMATS) {
|
|
||||||
const candidate = input.media_formats?.[key];
|
|
||||||
if (candidate?.url && candidate.dims?.length === 2) {
|
|
||||||
media[key] = {
|
|
||||||
src: candidate.url,
|
|
||||||
proxy_src: this.mediaService.getExternalMediaProxyURL(candidate.url),
|
|
||||||
width: candidate.dims[0],
|
|
||||||
height: candidate.dims[1],
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
id: input.id,
|
|
||||||
slug,
|
|
||||||
provider: this.meta.name,
|
|
||||||
title,
|
|
||||||
url,
|
|
||||||
src: best.url,
|
|
||||||
proxy_src: this.mediaService.getExternalMediaProxyURL(best.url),
|
|
||||||
width: best.dims[0],
|
|
||||||
height: best.dims[1],
|
|
||||||
media,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
extractSlugFromUrl(url: string): string | null {
|
|
||||||
try {
|
|
||||||
const parsed = new URL(url);
|
|
||||||
const hostname = parsed.hostname.toLowerCase();
|
|
||||||
if (hostname !== 'tenor.com' && hostname !== 'www.tenor.com') return null;
|
|
||||||
const match = parsed.pathname.match(/^\/(?:[a-z]{2}\/)?view\/([^/]+)/i);
|
|
||||||
if (!match?.[1]) return null;
|
|
||||||
const slug = decodeURIComponent(match[1]).trim();
|
|
||||||
return slug ? `view/${slug}` : null;
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private extractIdFromSlug(slug: string): string | null {
|
|
||||||
const normalized = slug.trim().replace(/^view\//i, '');
|
|
||||||
if (!normalized) return null;
|
|
||||||
const lastDashIndex = normalized.lastIndexOf('-');
|
|
||||||
const candidate = lastDashIndex === -1 ? normalized : normalized.slice(lastDashIndex + 1);
|
|
||||||
return candidate.trim() || null;
|
|
||||||
}
|
|
||||||
|
|
||||||
buildShareUrl(slug: string): string {
|
|
||||||
const trimmed = slug.trim().replace(/^\/+|\/+$/g, '');
|
|
||||||
const normalized = trimmed.toLowerCase().startsWith('view/') ? trimmed : `view/${trimmed}`;
|
|
||||||
return `https://tenor.com/${normalized}`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -70,6 +70,7 @@ describe('NatsUnfurlerService', () => {
|
|||||||
bypass_cache: true,
|
bypass_cache: true,
|
||||||
cache_only: false,
|
cache_only: false,
|
||||||
youtube_api_key: null,
|
youtube_api_key: null,
|
||||||
|
klipy_api_key: null,
|
||||||
},
|
},
|
||||||
timeout: 12000,
|
timeout: 12000,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ interface NatsUnfurlRequest {
|
|||||||
bypass_cache: boolean;
|
bypass_cache: boolean;
|
||||||
cache_only: boolean;
|
cache_only: boolean;
|
||||||
youtube_api_key: string | null;
|
youtube_api_key: string | null;
|
||||||
|
klipy_api_key: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface NatsUnfurlInnerResult {
|
interface NatsUnfurlInnerResult {
|
||||||
@@ -59,6 +60,7 @@ export class NatsUnfurlerService extends IUnfurlerService {
|
|||||||
constructor(
|
constructor(
|
||||||
connectionManager: INatsConnectionManager,
|
connectionManager: INatsConnectionManager,
|
||||||
private readonly resolveYoutubeApiKey: (() => Promise<string | null>) | null = null,
|
private readonly resolveYoutubeApiKey: (() => Promise<string | null>) | null = null,
|
||||||
|
private readonly resolveKlipyApiKey: (() => Promise<string | null>) | null = null,
|
||||||
) {
|
) {
|
||||||
super();
|
super();
|
||||||
this.connectionManager = connectionManager;
|
this.connectionManager = connectionManager;
|
||||||
@@ -77,6 +79,7 @@ export class NatsUnfurlerService extends IUnfurlerService {
|
|||||||
bypass_cache: options.bypassCache === true,
|
bypass_cache: options.bypassCache === true,
|
||||||
cache_only: options.cacheOnly === true,
|
cache_only: options.cacheOnly === true,
|
||||||
youtube_api_key: this.resolveYoutubeApiKey ? await this.resolveYoutubeApiKey() : null,
|
youtube_api_key: this.resolveYoutubeApiKey ? await this.resolveYoutubeApiKey() : null,
|
||||||
|
klipy_api_key: this.resolveKlipyApiKey ? await this.resolveKlipyApiKey() : null,
|
||||||
};
|
};
|
||||||
if (this.connectionManager.isClosed()) {
|
if (this.connectionManager.isClosed()) {
|
||||||
await this.connectionManager.connect();
|
await this.connectionManager.connect();
|
||||||
|
|||||||
@@ -99,13 +99,10 @@ interface InstanceServicesPublicConfig {
|
|||||||
bluesky_enabled: boolean;
|
bluesky_enabled: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type InstanceGifProvider = 'tenor' | 'klipy';
|
|
||||||
export type InstanceCaptchaProvider = 'hcaptcha' | 'turnstile' | 'none';
|
export type InstanceCaptchaProvider = 'hcaptcha' | 'turnstile' | 'none';
|
||||||
type InstanceEmailProvider = 'smtp' | 'none';
|
type InstanceEmailProvider = 'smtp' | 'none';
|
||||||
|
|
||||||
interface InstanceGifIntegrationConfig {
|
interface InstanceGifIntegrationConfig {
|
||||||
provider: InstanceGifProvider | null;
|
|
||||||
tenor_api_key: string | null;
|
|
||||||
klipy_api_key: string | null;
|
klipy_api_key: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -160,9 +157,7 @@ interface InstanceIntegrationsConfig {
|
|||||||
bluesky: InstanceBlueskyIntegrationConfig;
|
bluesky: InstanceBlueskyIntegrationConfig;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface InstanceGifEffectiveConfig {
|
interface InstanceGifEffectiveConfig {
|
||||||
provider: InstanceGifProvider;
|
|
||||||
tenor_api_key: string | null;
|
|
||||||
klipy_api_key: string | null;
|
klipy_api_key: string | null;
|
||||||
active_api_key: string | null;
|
active_api_key: string | null;
|
||||||
available: boolean;
|
available: boolean;
|
||||||
@@ -179,9 +174,6 @@ export interface InstanceCaptchaEffectiveConfig {
|
|||||||
|
|
||||||
interface InstanceIntegrationsAdminConfig {
|
interface InstanceIntegrationsAdminConfig {
|
||||||
gif: {
|
gif: {
|
||||||
provider: InstanceGifProvider | null;
|
|
||||||
effective_provider: InstanceGifProvider;
|
|
||||||
tenor_api_key_set: boolean;
|
|
||||||
klipy_api_key_set: boolean;
|
klipy_api_key_set: boolean;
|
||||||
effective_available: boolean;
|
effective_available: boolean;
|
||||||
};
|
};
|
||||||
@@ -438,8 +430,6 @@ function normalizeInstancePolicyConfig(value: unknown): InstancePolicyConfig {
|
|||||||
|
|
||||||
const DEFAULT_INSTANCE_INTEGRATIONS_CONFIG: InstanceIntegrationsConfig = {
|
const DEFAULT_INSTANCE_INTEGRATIONS_CONFIG: InstanceIntegrationsConfig = {
|
||||||
gif: {
|
gif: {
|
||||||
provider: null,
|
|
||||||
tenor_api_key: null,
|
|
||||||
klipy_api_key: null,
|
klipy_api_key: null,
|
||||||
},
|
},
|
||||||
youtube: {
|
youtube: {
|
||||||
@@ -492,10 +482,6 @@ const DEFAULT_INSTANCE_MEDIA_CONFIG: InstanceMediaConfig = {
|
|||||||
attachment_decay: DEFAULT_INSTANCE_ATTACHMENT_DECAY_CONFIG,
|
attachment_decay: DEFAULT_INSTANCE_ATTACHMENT_DECAY_CONFIG,
|
||||||
};
|
};
|
||||||
|
|
||||||
function isGifProvider(value: unknown): value is InstanceGifProvider {
|
|
||||||
return value === 'tenor' || value === 'klipy';
|
|
||||||
}
|
|
||||||
|
|
||||||
function isCaptchaProvider(value: unknown): value is InstanceCaptchaProvider {
|
function isCaptchaProvider(value: unknown): value is InstanceCaptchaProvider {
|
||||||
return value === 'hcaptcha' || value === 'turnstile' || value === 'none';
|
return value === 'hcaptcha' || value === 'turnstile' || value === 'none';
|
||||||
}
|
}
|
||||||
@@ -561,8 +547,6 @@ function normalizeInstanceIntegrationsConfig(value: unknown): InstanceIntegratio
|
|||||||
: defaults.bluesky.keys;
|
: defaults.bluesky.keys;
|
||||||
return {
|
return {
|
||||||
gif: {
|
gif: {
|
||||||
provider: isGifProvider(gif.provider) ? gif.provider : defaults.gif.provider,
|
|
||||||
tenor_api_key: normalizeSecretString(gif.tenor_api_key),
|
|
||||||
klipy_api_key: normalizeSecretString(gif.klipy_api_key),
|
klipy_api_key: normalizeSecretString(gif.klipy_api_key),
|
||||||
},
|
},
|
||||||
youtube: {
|
youtube: {
|
||||||
@@ -1214,16 +1198,11 @@ export class InstanceConfigRepository {
|
|||||||
|
|
||||||
async getEffectiveGifConfig(): Promise<InstanceGifEffectiveConfig> {
|
async getEffectiveGifConfig(): Promise<InstanceGifEffectiveConfig> {
|
||||||
const integrations = await this.getInstanceIntegrationsConfig();
|
const integrations = await this.getInstanceIntegrationsConfig();
|
||||||
const provider = integrations.gif.provider ?? Config.gif.provider;
|
|
||||||
const tenorApiKey = integrations.gif.tenor_api_key ?? normalizeSecretString(Config.tenor.apiKey);
|
|
||||||
const klipyApiKey = integrations.gif.klipy_api_key ?? normalizeSecretString(Config.klipy.apiKey);
|
const klipyApiKey = integrations.gif.klipy_api_key ?? normalizeSecretString(Config.klipy.apiKey);
|
||||||
const activeApiKey = provider === 'tenor' ? tenorApiKey : klipyApiKey;
|
|
||||||
return {
|
return {
|
||||||
provider,
|
|
||||||
tenor_api_key: tenorApiKey,
|
|
||||||
klipy_api_key: klipyApiKey,
|
klipy_api_key: klipyApiKey,
|
||||||
active_api_key: activeApiKey,
|
active_api_key: klipyApiKey,
|
||||||
available: Boolean(activeApiKey),
|
available: Boolean(klipyApiKey),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1323,9 +1302,6 @@ export class InstanceConfigRepository {
|
|||||||
]);
|
]);
|
||||||
return {
|
return {
|
||||||
gif: {
|
gif: {
|
||||||
provider: integrations.gif.provider,
|
|
||||||
effective_provider: gif.provider,
|
|
||||||
tenor_api_key_set: secretIsSet(integrations.gif.tenor_api_key) || secretIsSet(Config.tenor.apiKey),
|
|
||||||
klipy_api_key_set: secretIsSet(integrations.gif.klipy_api_key) || secretIsSet(Config.klipy.apiKey),
|
klipy_api_key_set: secretIsSet(integrations.gif.klipy_api_key) || secretIsSet(Config.klipy.apiKey),
|
||||||
effective_available: gif.available,
|
effective_available: gif.available,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import {RateLimitMiddleware} from '../middleware/RateLimitMiddleware';
|
|||||||
import {OpenAPI} from '../middleware/ResponseTypeMiddleware';
|
import {OpenAPI} from '../middleware/ResponseTypeMiddleware';
|
||||||
import {RateLimitConfigs} from '../RateLimitConfig';
|
import {RateLimitConfigs} from '../RateLimitConfig';
|
||||||
import type {HonoEnv} from '../types/HonoEnv';
|
import type {HonoEnv} from '../types/HonoEnv';
|
||||||
import type {InstanceCaptchaEffectiveConfig, InstanceGifEffectiveConfig} from './InstanceConfigRepository';
|
import type {InstanceCaptchaEffectiveConfig} from './InstanceConfigRepository';
|
||||||
|
|
||||||
function buildDiscoveryStaticInput(
|
function buildDiscoveryStaticInput(
|
||||||
gifService: GifService | undefined,
|
gifService: GifService | undefined,
|
||||||
@@ -20,12 +20,11 @@ function buildDiscoveryStaticInput(
|
|||||||
runtime: {
|
runtime: {
|
||||||
captcha: InstanceCaptchaEffectiveConfig;
|
captcha: InstanceCaptchaEffectiveConfig;
|
||||||
emailEnabled: boolean;
|
emailEnabled: boolean;
|
||||||
gif: InstanceGifEffectiveConfig;
|
|
||||||
},
|
},
|
||||||
): DiscoveryStaticInput {
|
): DiscoveryStaticInput {
|
||||||
const apiClientEndpoint = Config.endpoints.apiClient;
|
const apiClientEndpoint = Config.endpoints.apiClient;
|
||||||
const apiPublicEndpoint = Config.endpoints.apiPublic;
|
const apiPublicEndpoint = Config.endpoints.apiPublic;
|
||||||
const activeGif = gifService?.getByName(runtime.gif.provider);
|
const gifProvider = gifService?.getProvider();
|
||||||
return {
|
return {
|
||||||
apiCodeVersion: API_CODE_VERSION,
|
apiCodeVersion: API_CODE_VERSION,
|
||||||
endpoints: {
|
endpoints: {
|
||||||
@@ -54,9 +53,9 @@ function buildDiscoveryStaticInput(
|
|||||||
emails_enabled: runtime.emailEnabled,
|
emails_enabled: runtime.emailEnabled,
|
||||||
},
|
},
|
||||||
gif: {
|
gif: {
|
||||||
provider: runtime.gif.provider,
|
provider: gifProvider?.meta.name ?? 'klipy',
|
||||||
display_name: activeGif?.meta.displayName ?? runtime.gif.provider,
|
display_name: gifProvider?.meta.displayName ?? 'KLIPY',
|
||||||
attribution_required: activeGif?.meta.attributionRequired ?? false,
|
attribution_required: gifProvider?.meta.attributionRequired ?? true,
|
||||||
},
|
},
|
||||||
push: {
|
push: {
|
||||||
public_vapid_key: Config.push.publicVapidKey ?? null,
|
public_vapid_key: Config.push.publicVapidKey ?? null,
|
||||||
@@ -86,14 +85,13 @@ export function InstanceController(app: Hono<HonoEnv>) {
|
|||||||
const limits = limitConfigService?.getConfigWireFormat();
|
const limits = limitConfigService?.getConfigWireFormat();
|
||||||
const sso = await ctx.get('ssoService').getPublicStatus();
|
const sso = await ctx.get('ssoService').getPublicStatus();
|
||||||
const instanceConfigRepository = ctx.get('instanceConfigRepository');
|
const instanceConfigRepository = ctx.get('instanceConfigRepository');
|
||||||
const [registration, community, services, appPublicConfig, captcha, email, gif] = await Promise.all([
|
const [registration, community, services, appPublicConfig, captcha, email] = await Promise.all([
|
||||||
instanceConfigRepository.getRegistrationPublicConfig(),
|
instanceConfigRepository.getRegistrationPublicConfig(),
|
||||||
instanceConfigRepository.getInstanceCommunityPublicConfig(),
|
instanceConfigRepository.getInstanceCommunityPublicConfig(),
|
||||||
instanceConfigRepository.getResolvedServicesConfig(),
|
instanceConfigRepository.getResolvedServicesConfig(),
|
||||||
instanceConfigRepository.getAppPublicConfig(),
|
instanceConfigRepository.getAppPublicConfig(),
|
||||||
instanceConfigRepository.getEffectiveCaptchaConfig(),
|
instanceConfigRepository.getEffectiveCaptchaConfig(),
|
||||||
instanceConfigRepository.getEffectiveEmailConfig(),
|
instanceConfigRepository.getEffectiveEmailConfig(),
|
||||||
instanceConfigRepository.getEffectiveGifConfig(),
|
|
||||||
]);
|
]);
|
||||||
if (!limits) {
|
if (!limits) {
|
||||||
throw new Error('limit_config_service is not bound');
|
throw new Error('limit_config_service is not bound');
|
||||||
@@ -111,7 +109,6 @@ export function InstanceController(app: Hono<HonoEnv>) {
|
|||||||
{
|
{
|
||||||
captcha,
|
captcha,
|
||||||
emailEnabled: email.enabled,
|
emailEnabled: email.enabled,
|
||||||
gif,
|
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
|
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
|
||||||
import {ForbiddenError} from '@fluxer/errors/src/domains/core/ForbiddenError';
|
import {ForbiddenError} from '@fluxer/errors/src/domains/core/ForbiddenError';
|
||||||
|
import {resolveClientIpHeaderName} from '@fluxer/ip_utils/src/ClientIp';
|
||||||
import {createMiddleware} from 'hono/factory';
|
import {createMiddleware} from 'hono/factory';
|
||||||
import {Config} from '../Config';
|
import {Config} from '../Config';
|
||||||
import {Logger} from '../Logger';
|
import {Logger} from '../Logger';
|
||||||
@@ -23,7 +24,7 @@ const defaultExemptPaths: Array<string> = [
|
|||||||
|
|
||||||
export function RequireClientIpMiddleware({
|
export function RequireClientIpMiddleware({
|
||||||
exemptPaths = defaultExemptPaths,
|
exemptPaths = defaultExemptPaths,
|
||||||
requiredHeaders = ['x-forwarded-for'],
|
requiredHeaders = [resolveClientIpHeaderName(Config.proxy.client_ip_header)],
|
||||||
}: RequireClientIpOptions = {}) {
|
}: RequireClientIpOptions = {}) {
|
||||||
return createMiddleware<HonoEnv>(async (ctx, next) => {
|
return createMiddleware<HonoEnv>(async (ctx, next) => {
|
||||||
if (Config.dev.testModeEnabled) {
|
if (Config.dev.testModeEnabled) {
|
||||||
|
|||||||
@@ -82,6 +82,7 @@ import {UserChannelRequestService} from '../user/services/UserChannelRequestServ
|
|||||||
import {UserContentRequestService} from '../user/services/UserContentRequestService';
|
import {UserContentRequestService} from '../user/services/UserContentRequestService';
|
||||||
import {UserRelationshipRequestService} from '../user/services/UserRelationshipRequestService';
|
import {UserRelationshipRequestService} from '../user/services/UserRelationshipRequestService';
|
||||||
import {UserService} from '../user/services/UserService';
|
import {UserService} from '../user/services/UserService';
|
||||||
|
import {resolveRequestClientIp} from '../utils/IpUtils';
|
||||||
import {VoicePresenceHeartbeatStore} from '../voice/VoicePresenceHeartbeatStore';
|
import {VoicePresenceHeartbeatStore} from '../voice/VoicePresenceHeartbeatStore';
|
||||||
import {VoiceService} from '../voice/VoiceService';
|
import {VoiceService} from '../voice/VoiceService';
|
||||||
import {WebhookRequestService} from '../webhook/WebhookRequestService';
|
import {WebhookRequestService} from '../webhook/WebhookRequestService';
|
||||||
@@ -364,7 +365,7 @@ function getLiveKitWebhookService(): LiveKitWebhookService | null {
|
|||||||
export const ServiceMiddleware = createMiddleware<HonoEnv>(async (ctx, next) => {
|
export const ServiceMiddleware = createMiddleware<HonoEnv>(async (ctx, next) => {
|
||||||
const apiContext = createApiContext({
|
const apiContext = createApiContext({
|
||||||
requestId: ctx.get('requestId') ?? crypto.randomUUID(),
|
requestId: ctx.get('requestId') ?? crypto.randomUUID(),
|
||||||
clientIp: ctx.req.header('x-forwarded-for') ?? null,
|
clientIp: resolveRequestClientIp(ctx.req.raw),
|
||||||
userAgent: ctx.req.header('user-agent') ?? null,
|
userAgent: ctx.req.header('user-agent') ?? null,
|
||||||
});
|
});
|
||||||
ctx.set('apiContext', apiContext);
|
ctx.set('apiContext', apiContext);
|
||||||
|
|||||||
@@ -38,8 +38,7 @@ import {DownloadService} from '../download/DownloadService';
|
|||||||
import {createEmailProvider} from '../email/EmailProviderFactory';
|
import {createEmailProvider} from '../email/EmailProviderFactory';
|
||||||
import {FavoriteMemeRepository} from '../favorite_meme/FavoriteMemeRepository';
|
import {FavoriteMemeRepository} from '../favorite_meme/FavoriteMemeRepository';
|
||||||
import {GifService} from '../gif/GifService';
|
import {GifService} from '../gif/GifService';
|
||||||
import {KlipyGifProvider} from '../gif/KlipyGifProvider';
|
import {createNatsGifProvider} from '../gif/NatsGifProvider';
|
||||||
import {TenorGifProvider} from '../gif/TenorGifProvider';
|
|
||||||
import {GuildAuditLogService} from '../guild/GuildAuditLogService';
|
import {GuildAuditLogService} from '../guild/GuildAuditLogService';
|
||||||
import {GuildDiscoveryRepository} from '../guild/repositories/GuildDiscoveryRepository';
|
import {GuildDiscoveryRepository} from '../guild/repositories/GuildDiscoveryRepository';
|
||||||
import {GuildRepository} from '../guild/repositories/GuildRepository';
|
import {GuildRepository} from '../guild/repositories/GuildRepository';
|
||||||
@@ -349,6 +348,7 @@ export function setInjectedUnfurlerService(service: IUnfurlerService | undefined
|
|||||||
}
|
}
|
||||||
|
|
||||||
const getDefaultUnfurlerService = singleton(() => {
|
const getDefaultUnfurlerService = singleton(() => {
|
||||||
|
const instanceConfigRepository = getInstanceConfigRepository();
|
||||||
const manager = new NatsConnectionManager({
|
const manager = new NatsConnectionManager({
|
||||||
url: Config.nats.coreUrl,
|
url: Config.nats.coreUrl,
|
||||||
token: Config.nats.authToken || undefined,
|
token: Config.nats.authToken || undefined,
|
||||||
@@ -357,7 +357,11 @@ const getDefaultUnfurlerService = singleton(() => {
|
|||||||
void manager.connect().catch((error) => {
|
void manager.connect().catch((error) => {
|
||||||
Logger.error({error}, '[nats-unfurl] Failed to establish NATS connection');
|
Logger.error({error}, '[nats-unfurl] Failed to establish NATS connection');
|
||||||
});
|
});
|
||||||
return new NatsUnfurlerService(manager, async () => getInstanceConfigRepository().getEffectiveYoutubeApiKey());
|
return new NatsUnfurlerService(
|
||||||
|
manager,
|
||||||
|
async () => instanceConfigRepository.getEffectiveYoutubeApiKey(),
|
||||||
|
async () => (await instanceConfigRepository.getEffectiveGifConfig()).klipy_api_key,
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
export function getUnfurlerService(): IUnfurlerService {
|
export function getUnfurlerService(): IUnfurlerService {
|
||||||
@@ -376,24 +380,10 @@ export const getBotMfaMirrorService = singleton(
|
|||||||
() => new BotMfaMirrorService(getApplicationRepository(), getUserRepository(), getGatewayService()),
|
() => new BotMfaMirrorService(getApplicationRepository(), getUserRepository(), getGatewayService()),
|
||||||
);
|
);
|
||||||
export const getGifService = singleton(() => {
|
export const getGifService = singleton(() => {
|
||||||
const cache = getCacheService();
|
|
||||||
const media = getMediaService();
|
|
||||||
const instanceConfigRepository = getInstanceConfigRepository();
|
const instanceConfigRepository = getInstanceConfigRepository();
|
||||||
return new GifService({
|
return new GifService(
|
||||||
providers: [
|
createNatsGifProvider(async () => (await instanceConfigRepository.getEffectiveGifConfig()).klipy_api_key),
|
||||||
new TenorGifProvider(
|
);
|
||||||
cache,
|
|
||||||
media,
|
|
||||||
async () => (await instanceConfigRepository.getEffectiveGifConfig()).tenor_api_key,
|
|
||||||
),
|
|
||||||
new KlipyGifProvider(
|
|
||||||
cache,
|
|
||||||
media,
|
|
||||||
async () => (await instanceConfigRepository.getEffectiveGifConfig()).klipy_api_key,
|
|
||||||
),
|
|
||||||
],
|
|
||||||
activeName: async () => (await instanceConfigRepository.getEffectiveGifConfig()).provider,
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
export const getExpressionAssetPurger = singleton(() => new ExpressionAssetPurger(getAssetDeletionQueue()));
|
export const getExpressionAssetPurger = singleton(() => new ExpressionAssetPurger(getAssetDeletionQueue()));
|
||||||
export const getGuildAuditLogService = singleton(
|
export const getGuildAuditLogService = singleton(
|
||||||
|
|||||||
@@ -24808,10 +24808,7 @@
|
|||||||
"gif": {
|
"gif": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
"provider": {
|
"provider": {"type": "string", "description": "Stable machine name of the active GIF provider."},
|
||||||
"type": "string",
|
|
||||||
"description": "Stable machine name of the active GIF provider (e.g. \"klipy\", \"tenor\")"
|
|
||||||
},
|
|
||||||
"display_name": {"type": "string", "description": "Human-readable provider name shown in the UI"},
|
"display_name": {"type": "string", "description": "Human-readable provider name shown in the UI"},
|
||||||
"attribution_required": {
|
"attribution_required": {
|
||||||
"type": "boolean",
|
"type": "boolean",
|
||||||
@@ -29258,7 +29255,7 @@
|
|||||||
},
|
},
|
||||||
"gif_provider": {
|
"gif_provider": {
|
||||||
"anyOf": [{"type": "string"}, {"type": "null"}],
|
"anyOf": [{"type": "string"}, {"type": "null"}],
|
||||||
"description": "Stable name of the GIF provider that issued gif_slug (e.g. \"klipy\", \"tenor\"), if any"
|
"description": "Stable name of the GIF provider that issued gif_slug, if any. Legacy records may contain older provider names."
|
||||||
},
|
},
|
||||||
"media": {
|
"media": {
|
||||||
"anyOf": [
|
"anyOf": [
|
||||||
@@ -35767,7 +35764,7 @@
|
|||||||
},
|
},
|
||||||
"gif_provider": {
|
"gif_provider": {
|
||||||
"anyOf": [{"type": "string"}, {"type": "null"}],
|
"anyOf": [{"type": "string"}, {"type": "null"}],
|
||||||
"description": "Stable name of the GIF provider that issued gif_slug (e.g. \"klipy\", \"tenor\")"
|
"description": "Stable name of the GIF provider that issued gif_slug. New provider GIFs are sourced from KLIPY."
|
||||||
},
|
},
|
||||||
"media": {
|
"media": {
|
||||||
"anyOf": [
|
"anyOf": [
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import {Logger} from '../Logger';
|
|||||||
import type {HonoApp} from '../types/HonoEnv';
|
import type {HonoApp} from '../types/HonoEnv';
|
||||||
import type {IUserRepository} from '../user/IUserRepository';
|
import type {IUserRepository} from '../user/IUserRepository';
|
||||||
import {mapUserToPrivateResponse} from '../user/UserMappers';
|
import {mapUserToPrivateResponse} from '../user/UserMappers';
|
||||||
|
import {resolveRequestClientIp} from '../utils/IpUtils';
|
||||||
|
|
||||||
interface TwilioInboundSmsWebhookContext {
|
interface TwilioInboundSmsWebhookContext {
|
||||||
authToken: string;
|
authToken: string;
|
||||||
@@ -29,7 +30,7 @@ export function installTwilioInboundSmsWebhook(app: HonoApp, ctx: TwilioInboundS
|
|||||||
const params = parseFormUrlEncoded(rawBody);
|
const params = parseFormUrlEncoded(rawBody);
|
||||||
const signature = c.req.header('x-twilio-signature') ?? '';
|
const signature = c.req.header('x-twilio-signature') ?? '';
|
||||||
if (!verifyTwilioSignature(ctx.authToken, ctx.publicWebhookUrl, params, signature)) {
|
if (!verifyTwilioSignature(ctx.authToken, ctx.publicWebhookUrl, params, signature)) {
|
||||||
Logger.warn({ip: c.req.header('cf-connecting-ip') ?? '?'}, 'Twilio webhook signature failed; rejecting');
|
Logger.warn({ip: resolveRequestClientIp(c.req.raw) ?? '?'}, 'Twilio webhook signature failed; rejecting');
|
||||||
return c.text('forbidden', 403);
|
return c.text('forbidden', 403);
|
||||||
}
|
}
|
||||||
const fromPhone = params.get('From') ?? '';
|
const fromPhone = params.get('From') ?? '';
|
||||||
|
|||||||
@@ -18,8 +18,6 @@ export const EXTERNAL_RESPONSE_LIMITS = {
|
|||||||
blueskyProfileBytes: 256 * 1024,
|
blueskyProfileBytes: 256 * 1024,
|
||||||
wikipediaSummaryBytes: 256 * 1024,
|
wikipediaSummaryBytes: 256 * 1024,
|
||||||
oEmbedBytes: 256 * 1024,
|
oEmbedBytes: 256 * 1024,
|
||||||
klipyApiBytes: 512 * 1024,
|
|
||||||
tenorApiBytes: 512 * 1024,
|
|
||||||
pwnedPasswordsBytes: 1024 * 1024,
|
pwnedPasswordsBytes: 1024 * 1024,
|
||||||
rdapBytes: 512 * 1024,
|
rdapBytes: 512 * 1024,
|
||||||
externalTemplateBytes: 512 * 1024,
|
externalTemplateBytes: 512 * 1024,
|
||||||
|
|||||||
@@ -16,16 +16,17 @@ interface GetIpAddressReverseOptions {
|
|||||||
cacheTtlSeconds?: number;
|
cacheTtlSeconds?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function resolveRequestClientIp(req: Request): string | null {
|
||||||
|
return extractClientIp(req, {
|
||||||
|
trustClientIpHeader: Config.proxy.trust_client_ip_header,
|
||||||
|
clientIpHeaderName: Config.proxy.client_ip_header,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export async function lookupGeoip(req: Request): Promise<GeoipResult>;
|
export async function lookupGeoip(req: Request): Promise<GeoipResult>;
|
||||||
export async function lookupGeoip(ip: string): Promise<GeoipResult>;
|
export async function lookupGeoip(ip: string): Promise<GeoipResult>;
|
||||||
export async function lookupGeoip(input: string | Request): Promise<GeoipResult> {
|
export async function lookupGeoip(input: string | Request): Promise<GeoipResult> {
|
||||||
const ip =
|
const ip = typeof input === 'string' ? input : resolveRequestClientIp(input);
|
||||||
typeof input === 'string'
|
|
||||||
? input
|
|
||||||
: extractClientIp(input, {
|
|
||||||
trustClientIpHeader: Config.proxy.trust_client_ip_header,
|
|
||||||
clientIpHeaderName: Config.proxy.client_ip_header,
|
|
||||||
});
|
|
||||||
if (!ip) {
|
if (!ip) {
|
||||||
return {countryCode: null, normalizedIp: null, city: null, region: null, countryName: null};
|
return {countryCode: null, normalizedIp: null, city: null, region: null, countryName: null};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -63,7 +63,6 @@ const LANE_CONFIG = {
|
|||||||
batch: {
|
batch: {
|
||||||
consumerName: 'workers_batch',
|
consumerName: 'workers_batch',
|
||||||
tasks: [
|
tasks: [
|
||||||
'enqueueGifFeaturedCategoriesRefresh',
|
|
||||||
'expireAttachments',
|
'expireAttachments',
|
||||||
'indexChannelMessages',
|
'indexChannelMessages',
|
||||||
'indexGuildMembers',
|
'indexGuildMembers',
|
||||||
@@ -74,7 +73,6 @@ const LANE_CONFIG = {
|
|||||||
'processPendingBulkMessageDeletions',
|
'processPendingBulkMessageDeletions',
|
||||||
'processPremiumStateReconciliationQueue',
|
'processPremiumStateReconciliationQueue',
|
||||||
'prunePostgresKvTtl',
|
'prunePostgresKvTtl',
|
||||||
'refreshGifFeaturedCategories',
|
|
||||||
'refreshSearchIndex',
|
'refreshSearchIndex',
|
||||||
'syncDiscoveryIndex',
|
'syncDiscoveryIndex',
|
||||||
'syncDisposableEmailDomains',
|
'syncDisposableEmailDomains',
|
||||||
|
|||||||
@@ -52,7 +52,6 @@ function registerCronJobs(cron: CronScheduler): void {
|
|||||||
cron.upsert('prunePostgresKvTtl', 'prunePostgresKvTtl', {}, '0 */5 * * * *');
|
cron.upsert('prunePostgresKvTtl', 'prunePostgresKvTtl', {}, '0 */5 * * * *');
|
||||||
cron.upsert('syncDiscoveryIndex', 'syncDiscoveryIndex', {}, '0 */15 * * * *');
|
cron.upsert('syncDiscoveryIndex', 'syncDiscoveryIndex', {}, '0 */15 * * * *');
|
||||||
cron.upsert('syncDisposableEmailDomains', 'syncDisposableEmailDomains', {}, '0 */30 * * * *');
|
cron.upsert('syncDisposableEmailDomains', 'syncDisposableEmailDomains', {}, '0 */30 * * * *');
|
||||||
cron.upsert('enqueueGifFeaturedCategoriesRefresh', 'enqueueGifFeaturedCategoriesRefresh', {}, '0 */30 * * * *');
|
|
||||||
cron.upsert('syncUrlBlocklists', 'syncUrlBlocklists', {}, '0 0 */6 * * *');
|
cron.upsert('syncUrlBlocklists', 'syncUrlBlocklists', {}, '0 0 */6 * * *');
|
||||||
cron.upsert('syncFileShaBlocklists', 'syncFileShaBlocklists', {}, '0 0 */12 * * *');
|
cron.upsert('syncFileShaBlocklists', 'syncFileShaBlocklists', {}, '0 0 */12 * * *');
|
||||||
cron.upsert('flushUserActivityBuffer', 'flushUserActivityBuffer', {}, '*/10 * * * * *');
|
cron.upsert('flushUserActivityBuffer', 'flushUserActivityBuffer', {}, '*/10 * * * * *');
|
||||||
|
|||||||
@@ -13,7 +13,6 @@ import bulkDeleteSelfMessagesImmediate from './tasks/BulkDeleteSelfMessagesImmed
|
|||||||
import bulkDeleteUserMessages from './tasks/BulkDeleteUserMessages';
|
import bulkDeleteUserMessages from './tasks/BulkDeleteUserMessages';
|
||||||
import bulkDeleteUserMessagesScoped from './tasks/BulkDeleteUserMessagesScoped';
|
import bulkDeleteUserMessagesScoped from './tasks/BulkDeleteUserMessagesScoped';
|
||||||
import deleteUserMessagesInGuildByTime from './tasks/DeleteUserMessagesInGuildByTime';
|
import deleteUserMessagesInGuildByTime from './tasks/DeleteUserMessagesInGuildByTime';
|
||||||
import enqueueGifFeaturedCategoriesRefresh from './tasks/EnqueueGifFeaturedCategoriesRefresh';
|
|
||||||
import expireAttachments from './tasks/ExpireAttachments';
|
import expireAttachments from './tasks/ExpireAttachments';
|
||||||
import extractEmbeds from './tasks/ExtractEmbeds';
|
import extractEmbeds from './tasks/ExtractEmbeds';
|
||||||
import finalizeNcmecAttachmentReport from './tasks/FinalizeNcmecAttachmentReport';
|
import finalizeNcmecAttachmentReport from './tasks/FinalizeNcmecAttachmentReport';
|
||||||
@@ -34,7 +33,6 @@ import processPremiumStateReconciliationQueue from './tasks/ProcessPremiumStateR
|
|||||||
import processStripeWebhook from './tasks/ProcessStripeWebhook';
|
import processStripeWebhook from './tasks/ProcessStripeWebhook';
|
||||||
import prunePostgresKvTtl from './tasks/PrunePostgresKvTtl';
|
import prunePostgresKvTtl from './tasks/PrunePostgresKvTtl';
|
||||||
import reconcileUserPayments from './tasks/ReconcileUserPayments';
|
import reconcileUserPayments from './tasks/ReconcileUserPayments';
|
||||||
import refreshGifFeaturedCategories from './tasks/RefreshGifFeaturedCategories';
|
|
||||||
import refreshSearchIndex from './tasks/RefreshSearchIndex';
|
import refreshSearchIndex from './tasks/RefreshSearchIndex';
|
||||||
import revalidateUserConnections from './tasks/RevalidateUserConnections';
|
import revalidateUserConnections from './tasks/RevalidateUserConnections';
|
||||||
import {sendScheduledMessage} from './tasks/SendScheduledMessage';
|
import {sendScheduledMessage} from './tasks/SendScheduledMessage';
|
||||||
@@ -60,7 +58,6 @@ export const workerTasks: Record<WorkerTaskName, WorkerTaskHandler> = {
|
|||||||
bulkUpdateSuspiciousActivityFlags: bulkUpdateSuspiciousActivityFlags,
|
bulkUpdateSuspiciousActivityFlags: bulkUpdateSuspiciousActivityFlags,
|
||||||
bulkUpdateUserFlags: bulkUpdateUserFlags,
|
bulkUpdateUserFlags: bulkUpdateUserFlags,
|
||||||
deleteUserMessagesInGuildByTime,
|
deleteUserMessagesInGuildByTime,
|
||||||
enqueueGifFeaturedCategoriesRefresh,
|
|
||||||
expireAttachments,
|
expireAttachments,
|
||||||
extractEmbeds,
|
extractEmbeds,
|
||||||
finalizeNcmecAttachmentReport,
|
finalizeNcmecAttachmentReport,
|
||||||
@@ -80,7 +77,6 @@ export const workerTasks: Record<WorkerTaskName, WorkerTaskHandler> = {
|
|||||||
processPremiumStateReconciliationQueue,
|
processPremiumStateReconciliationQueue,
|
||||||
reconcileUserPayments,
|
reconcileUserPayments,
|
||||||
prunePostgresKvTtl,
|
prunePostgresKvTtl,
|
||||||
refreshGifFeaturedCategories,
|
|
||||||
refreshSearchIndex,
|
refreshSearchIndex,
|
||||||
revalidateUserConnections,
|
revalidateUserConnections,
|
||||||
sendScheduledMessage,
|
sendScheduledMessage,
|
||||||
|
|||||||
@@ -1,39 +0,0 @@
|
|||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
|
|
||||||
import {listSeenLocales} from '../../gif/GifFeaturedCategoriesCache';
|
|
||||||
import {getGifService} from '../../middleware/ServiceSingletons';
|
|
||||||
import {getWorkerDependencies} from '../WorkerContext';
|
|
||||||
|
|
||||||
const enqueueGifFeaturedCategoriesRefresh: WorkerTaskHandler = async (_payload, helpers) => {
|
|
||||||
const {cacheService, workerService} = getWorkerDependencies();
|
|
||||||
const gifService = getGifService();
|
|
||||||
for (const provider of gifService.listProviders()) {
|
|
||||||
if (!(await provider.isAvailable())) continue;
|
|
||||||
const locales = await listSeenLocales(cacheService, provider.meta.name);
|
|
||||||
if (locales.length === 0) {
|
|
||||||
helpers.logger.debug({provider: provider.meta.name}, 'No seen locales for GIF provider; nothing to refresh');
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
helpers.logger.debug(
|
|
||||||
{provider: provider.meta.name, count: locales.length},
|
|
||||||
'Fanning out enriched GIF categories refresh',
|
|
||||||
);
|
|
||||||
for (const {locale, country} of locales) {
|
|
||||||
try {
|
|
||||||
await workerService.addJob('refreshGifFeaturedCategories', {
|
|
||||||
provider: provider.meta.name,
|
|
||||||
locale,
|
|
||||||
country,
|
|
||||||
});
|
|
||||||
} catch (error) {
|
|
||||||
helpers.logger.warn(
|
|
||||||
{err: error, provider: provider.meta.name, locale, country},
|
|
||||||
'Failed to enqueue per-locale GIF categories refresh',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export default enqueueGifFeaturedCategoriesRefresh;
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
|
|
||||||
import {getGifService} from '../../middleware/ServiceSingletons';
|
|
||||||
|
|
||||||
interface RefreshPayload {
|
|
||||||
provider?: string;
|
|
||||||
locale?: string;
|
|
||||||
country?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
const refreshGifFeaturedCategories: WorkerTaskHandler = async (rawPayload, helpers) => {
|
|
||||||
const payload = (rawPayload ?? {}) as RefreshPayload;
|
|
||||||
const {provider, locale, country} = payload;
|
|
||||||
if (!provider || !locale || !country) {
|
|
||||||
helpers.logger.warn({payload}, 'refreshGifFeaturedCategories called without required fields');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const gifService = getGifService();
|
|
||||||
const target = gifService.getByName(provider);
|
|
||||||
if (!target) {
|
|
||||||
helpers.logger.warn({provider}, 'refreshGifFeaturedCategories: unknown provider');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (!(await target.isAvailable())) {
|
|
||||||
helpers.logger.debug({provider}, 'refreshGifFeaturedCategories: provider not configured, skipping');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
helpers.logger.debug({provider, locale, country}, 'Refreshing enriched GIF featured categories');
|
|
||||||
await target.refreshFeaturedCategories({locale, country});
|
|
||||||
};
|
|
||||||
|
|
||||||
export default refreshGifFeaturedCategories;
|
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
[package]
|
||||||
|
name = "fluxer-gifs"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition.workspace = true
|
||||||
|
license.workspace = true
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
anyhow = "1.0.102"
|
||||||
|
base64 = "0.22.1"
|
||||||
|
fluxer-svc = { path = "../fluxer_svc", default-features = false }
|
||||||
|
hmac = "0.13.0"
|
||||||
|
moka = { version = "0.12.15", features = ["future", "sync"] }
|
||||||
|
reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls"] }
|
||||||
|
serde = { version = "1.0.228", features = ["derive"] }
|
||||||
|
serde_json = "1.0.150"
|
||||||
|
sha2 = "0.11.0"
|
||||||
|
tokio = { version = "1.52.3", features = ["macros", "rt-multi-thread", "signal", "sync", "time"] }
|
||||||
|
tracing = "0.1.44"
|
||||||
|
url = "2.5"
|
||||||
|
urlencoding = "2.1"
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
FROM rust:1-bookworm AS builder
|
||||||
|
|
||||||
|
WORKDIR /usr/src/app
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
RUN cargo build --release -p fluxer-gifs
|
||||||
|
|
||||||
|
FROM debian:bookworm-slim
|
||||||
|
|
||||||
|
ARG BUILD_VERSION=""
|
||||||
|
|
||||||
|
WORKDIR /usr/local/bin
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
COPY --from=builder /usr/src/app/target/release/fluxer-gifs /usr/local/bin/fluxer-gifs
|
||||||
|
|
||||||
|
ENV BUILD_VERSION="${BUILD_VERSION}"
|
||||||
|
|
||||||
|
USER 65532:65532
|
||||||
|
|
||||||
|
EXPOSE 8090
|
||||||
|
|
||||||
|
CMD ["/usr/local/bin/fluxer-gifs"]
|
||||||
@@ -0,0 +1,721 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
use crate::media_proxy::MediaProxyUrlBuilder;
|
||||||
|
use crate::types::{GifCategoryTag, GifItem, GifMediaFormat};
|
||||||
|
use anyhow::Context;
|
||||||
|
use reqwest::Url;
|
||||||
|
use serde::Deserialize;
|
||||||
|
use serde_json::Value;
|
||||||
|
use std::collections::{BTreeMap, HashSet};
|
||||||
|
use std::time::Duration;
|
||||||
|
use tokio::time::sleep;
|
||||||
|
|
||||||
|
const KLIPY_BASE_URL: &str = "https://api.klipy.com/v2";
|
||||||
|
const KLIPY_DIRECT_BASE_URL: &str = "https://api.klipy.com/api/v1";
|
||||||
|
const DEFAULT_CONTENT_FILTER: &str = "low";
|
||||||
|
const CLIENT_KEY: &str = "fluxer";
|
||||||
|
const MAX_RETRIES: usize = 3;
|
||||||
|
const BACKOFF_BASE_DELAY: Duration = Duration::from_secs(1);
|
||||||
|
const KLIPY_RESPONSE_LIMIT_BYTES: usize = 512 * 1024;
|
||||||
|
const FLUXER_USER_AGENT: &str = "Fluxerbot/1.0 (+https://fluxer.app)";
|
||||||
|
const KLIPY_PROVIDER_NAME: &str = "klipy";
|
||||||
|
const KLIPY_FEATURED_CATEGORY_REFRESH_COUNTRY: &str = "US";
|
||||||
|
|
||||||
|
const SIZE_PREFERENCE: [&str; 4] = ["hd", "md", "sm", "xs"];
|
||||||
|
const FORMAT_PREFERENCE: [&str; 4] = ["webm", "mp4", "webp", "gif"];
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct KlipyClient {
|
||||||
|
http_client: reqwest::Client,
|
||||||
|
media_proxy: MediaProxyUrlBuilder,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct ResultsResponse {
|
||||||
|
results: Vec<Value>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct TagsResponse {
|
||||||
|
tags: Vec<Value>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct KlipyGif {
|
||||||
|
id: Value,
|
||||||
|
#[serde(default)]
|
||||||
|
slug: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
title: String,
|
||||||
|
#[serde(default)]
|
||||||
|
itemurl: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
file: Option<BTreeMap<String, BTreeMap<String, KlipyFileEntry>>>,
|
||||||
|
#[serde(default)]
|
||||||
|
media_formats: Option<KlipyMediaFormats>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct KlipyMediaFormats {
|
||||||
|
#[serde(default)]
|
||||||
|
webm: Option<KlipyFallbackMediaFormat>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct KlipyFallbackMediaFormat {
|
||||||
|
url: String,
|
||||||
|
dims: [i32; 2],
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct KlipyFileEntry {
|
||||||
|
#[serde(default)]
|
||||||
|
url: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
width: Option<i32>,
|
||||||
|
#[serde(default)]
|
||||||
|
height: Option<i32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct KlipyCategoryTag {
|
||||||
|
searchterm: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct DirectGifResponse {
|
||||||
|
#[serde(default)]
|
||||||
|
data: Option<KlipyGif>,
|
||||||
|
}
|
||||||
|
|
||||||
|
enum KlipyJsonFetch<T> {
|
||||||
|
Found(T),
|
||||||
|
NotFound,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct KlipyPath {
|
||||||
|
path_type: KlipyPathType,
|
||||||
|
slug: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
enum KlipyPathType {
|
||||||
|
Gif,
|
||||||
|
Clip,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl KlipyClient {
|
||||||
|
pub fn new(media_proxy: MediaProxyUrlBuilder) -> anyhow::Result<Self> {
|
||||||
|
let http_client = reqwest::Client::builder()
|
||||||
|
.user_agent(FLUXER_USER_AGENT)
|
||||||
|
.timeout(Duration::from_secs(30))
|
||||||
|
.build()
|
||||||
|
.context("failed to build KLIPY HTTP client")?;
|
||||||
|
Ok(Self {
|
||||||
|
http_client,
|
||||||
|
media_proxy,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn search(
|
||||||
|
&self,
|
||||||
|
api_key: &str,
|
||||||
|
q: &str,
|
||||||
|
locale: &str,
|
||||||
|
country: &str,
|
||||||
|
limit: u32,
|
||||||
|
) -> anyhow::Result<Vec<GifItem>> {
|
||||||
|
let locale = normalize_locale(locale);
|
||||||
|
let limit = limit.to_string();
|
||||||
|
self.fetch_gifs(
|
||||||
|
"search",
|
||||||
|
&[
|
||||||
|
("key", api_key),
|
||||||
|
("q", q),
|
||||||
|
("country", country),
|
||||||
|
("locale", &locale),
|
||||||
|
("limit", &limit),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn featured_gifs(
|
||||||
|
&self,
|
||||||
|
api_key: &str,
|
||||||
|
locale: &str,
|
||||||
|
country: &str,
|
||||||
|
) -> anyhow::Result<Vec<GifItem>> {
|
||||||
|
let locale = normalize_locale(locale);
|
||||||
|
self.fetch_gifs(
|
||||||
|
"featured",
|
||||||
|
&[
|
||||||
|
("key", api_key),
|
||||||
|
("country", country),
|
||||||
|
("locale", &locale),
|
||||||
|
("limit", "1"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn trending_gifs(
|
||||||
|
&self,
|
||||||
|
api_key: &str,
|
||||||
|
locale: &str,
|
||||||
|
country: &str,
|
||||||
|
) -> anyhow::Result<Vec<GifItem>> {
|
||||||
|
let locale = normalize_locale(locale);
|
||||||
|
self.fetch_gifs(
|
||||||
|
"featured",
|
||||||
|
&[
|
||||||
|
("key", api_key),
|
||||||
|
("country", country),
|
||||||
|
("locale", &locale),
|
||||||
|
("limit", "50"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn suggestions(
|
||||||
|
&self,
|
||||||
|
api_key: &str,
|
||||||
|
q: &str,
|
||||||
|
locale: &str,
|
||||||
|
) -> anyhow::Result<Vec<String>> {
|
||||||
|
let locale = normalize_locale(locale);
|
||||||
|
let response: ResultsResponse = self
|
||||||
|
.fetch_json(
|
||||||
|
"autocomplete",
|
||||||
|
&[("key", api_key), ("q", q), ("locale", &locale)],
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(response
|
||||||
|
.results
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|value| value.as_str().map(ToOwned::to_owned))
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn register_share(
|
||||||
|
&self,
|
||||||
|
api_key: &str,
|
||||||
|
id: &str,
|
||||||
|
q: &str,
|
||||||
|
locale: &str,
|
||||||
|
country: &str,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let locale = normalize_locale(locale);
|
||||||
|
let url = self.create_url(
|
||||||
|
"registershare",
|
||||||
|
&[
|
||||||
|
("key", api_key),
|
||||||
|
("id", id),
|
||||||
|
("country", country),
|
||||||
|
("locale", &locale),
|
||||||
|
("q", q),
|
||||||
|
],
|
||||||
|
)?;
|
||||||
|
let response = self.http_client.get(url).send().await?;
|
||||||
|
if !response.status().is_success() {
|
||||||
|
anyhow::bail!(
|
||||||
|
"KLIPY registershare failed with status {}",
|
||||||
|
response.status()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn resolve_by_url(
|
||||||
|
&self,
|
||||||
|
api_key: &str,
|
||||||
|
url: &str,
|
||||||
|
_locale: &str,
|
||||||
|
_country: &str,
|
||||||
|
) -> anyhow::Result<Option<GifItem>> {
|
||||||
|
let Some(path) = parse_klipy_path(url) else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
self.fetch_direct_gif(api_key, &path).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn featured_categories(
|
||||||
|
&self,
|
||||||
|
api_key: &str,
|
||||||
|
locale: &str,
|
||||||
|
) -> anyhow::Result<Vec<GifCategoryTag>> {
|
||||||
|
let normalized_locale = normalize_locale(locale);
|
||||||
|
let response: TagsResponse = self
|
||||||
|
.fetch_json(
|
||||||
|
"categories",
|
||||||
|
&[
|
||||||
|
("key", api_key),
|
||||||
|
("country", KLIPY_FEATURED_CATEGORY_REFRESH_COUNTRY),
|
||||||
|
("locale", &normalized_locale),
|
||||||
|
("type", "featured"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let mut seen = HashSet::new();
|
||||||
|
let search_terms = response
|
||||||
|
.tags
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|value| serde_json::from_value::<KlipyCategoryTag>(value).ok())
|
||||||
|
.map(|tag| tag.searchterm.trim().to_owned())
|
||||||
|
.filter(|term| !term.is_empty())
|
||||||
|
.filter(|term| seen.insert(term.clone()))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
let mut categories = Vec::with_capacity(search_terms.len());
|
||||||
|
for search_term in search_terms {
|
||||||
|
let gif = match self
|
||||||
|
.search(
|
||||||
|
api_key,
|
||||||
|
&search_term,
|
||||||
|
&normalized_locale,
|
||||||
|
KLIPY_FEATURED_CATEGORY_REFRESH_COUNTRY,
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(mut gifs) => gifs.drain(..).next(),
|
||||||
|
Err(err) => {
|
||||||
|
tracing::debug!(
|
||||||
|
error = %err,
|
||||||
|
search_term = %search_term,
|
||||||
|
locale = %normalized_locale,
|
||||||
|
"failed to fetch KLIPY category preview GIF"
|
||||||
|
);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
};
|
||||||
|
categories.push(category_response(search_term, gif));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(categories)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fetch_gifs(
|
||||||
|
&self,
|
||||||
|
endpoint: &str,
|
||||||
|
params: &[(&str, &str)],
|
||||||
|
) -> anyhow::Result<Vec<GifItem>> {
|
||||||
|
let response: ResultsResponse = self.fetch_json(endpoint, params).await?;
|
||||||
|
Ok(response
|
||||||
|
.results
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|value| serde_json::from_value::<KlipyGif>(value).ok())
|
||||||
|
.filter_map(|gif| self.transform_gif(gif))
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fetch_direct_gif(
|
||||||
|
&self,
|
||||||
|
api_key: &str,
|
||||||
|
path: &KlipyPath,
|
||||||
|
) -> anyhow::Result<Option<GifItem>> {
|
||||||
|
let url = self.create_direct_url(api_key, path)?;
|
||||||
|
let mut last_error = None;
|
||||||
|
for attempt in 0..MAX_RETRIES {
|
||||||
|
match self.fetch_direct_gif_once(url.clone(), path).await {
|
||||||
|
Ok(value) => return Ok(value),
|
||||||
|
Err(error) if attempt + 1 < MAX_RETRIES => {
|
||||||
|
last_error = Some(error);
|
||||||
|
sleep(BACKOFF_BASE_DELAY * 2_u32.pow(attempt as u32)).await;
|
||||||
|
}
|
||||||
|
Err(error) => return Err(error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(last_error.unwrap_or_else(|| anyhow::anyhow!("exceeded KLIPY retry limit")))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fetch_direct_gif_once(
|
||||||
|
&self,
|
||||||
|
url: Url,
|
||||||
|
path: &KlipyPath,
|
||||||
|
) -> anyhow::Result<Option<GifItem>> {
|
||||||
|
match self.fetch_json_response::<DirectGifResponse>(url).await? {
|
||||||
|
KlipyJsonFetch::NotFound => Ok(None),
|
||||||
|
KlipyJsonFetch::Found(response) => Ok(response
|
||||||
|
.data
|
||||||
|
.and_then(|gif| self.transform_gif_with_path(gif, Some(path)))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fetch_json<T>(&self, endpoint: &str, params: &[(&str, &str)]) -> anyhow::Result<T>
|
||||||
|
where
|
||||||
|
T: serde::de::DeserializeOwned,
|
||||||
|
{
|
||||||
|
let url = self.create_url(endpoint, params)?;
|
||||||
|
let mut last_error = None;
|
||||||
|
for attempt in 0..MAX_RETRIES {
|
||||||
|
match self.fetch_json_once(url.clone()).await {
|
||||||
|
Ok(value) => return Ok(value),
|
||||||
|
Err(error) if attempt + 1 < MAX_RETRIES => {
|
||||||
|
last_error = Some(error);
|
||||||
|
sleep(BACKOFF_BASE_DELAY * 2_u32.pow(attempt as u32)).await;
|
||||||
|
}
|
||||||
|
Err(error) => return Err(error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(last_error.unwrap_or_else(|| anyhow::anyhow!("exceeded KLIPY retry limit")))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fetch_json_once<T>(&self, url: Url) -> anyhow::Result<T>
|
||||||
|
where
|
||||||
|
T: serde::de::DeserializeOwned,
|
||||||
|
{
|
||||||
|
match self.fetch_json_response(url).await? {
|
||||||
|
KlipyJsonFetch::Found(value) => Ok(value),
|
||||||
|
KlipyJsonFetch::NotFound => anyhow::bail!("KLIPY request returned not found"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fetch_json_response<T>(&self, url: Url) -> anyhow::Result<KlipyJsonFetch<T>>
|
||||||
|
where
|
||||||
|
T: serde::de::DeserializeOwned,
|
||||||
|
{
|
||||||
|
let response = self.http_client.get(url.clone()).send().await?;
|
||||||
|
if response.status() == reqwest::StatusCode::NOT_FOUND {
|
||||||
|
return Ok(KlipyJsonFetch::NotFound);
|
||||||
|
}
|
||||||
|
if !response.status().is_success() {
|
||||||
|
anyhow::bail!("KLIPY request failed with status {}", response.status());
|
||||||
|
}
|
||||||
|
if response
|
||||||
|
.content_length()
|
||||||
|
.is_some_and(|len| len > KLIPY_RESPONSE_LIMIT_BYTES as u64)
|
||||||
|
{
|
||||||
|
anyhow::bail!("KLIPY response declared more than {KLIPY_RESPONSE_LIMIT_BYTES} bytes");
|
||||||
|
}
|
||||||
|
let bytes = response.bytes().await?;
|
||||||
|
if bytes.len() > KLIPY_RESPONSE_LIMIT_BYTES {
|
||||||
|
anyhow::bail!("KLIPY response exceeded {KLIPY_RESPONSE_LIMIT_BYTES} bytes");
|
||||||
|
}
|
||||||
|
serde_json::from_slice(&bytes)
|
||||||
|
.with_context(|| format!("failed to parse KLIPY response from {url}"))
|
||||||
|
.map(KlipyJsonFetch::Found)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_url(&self, endpoint: &str, params: &[(&str, &str)]) -> anyhow::Result<Url> {
|
||||||
|
let mut url = Url::parse(&format!("{KLIPY_BASE_URL}/{endpoint}"))?;
|
||||||
|
{
|
||||||
|
let mut query = url.query_pairs_mut();
|
||||||
|
query.append_pair("client_key", CLIENT_KEY);
|
||||||
|
query.append_pair("contentfilter", DEFAULT_CONTENT_FILTER);
|
||||||
|
for (key, value) in params {
|
||||||
|
query.append_pair(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(url)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_direct_url(&self, api_key: &str, path: &KlipyPath) -> anyhow::Result<Url> {
|
||||||
|
let mut url = Url::parse(&format!("{KLIPY_DIRECT_BASE_URL}/"))?;
|
||||||
|
{
|
||||||
|
let mut segments = url
|
||||||
|
.path_segments_mut()
|
||||||
|
.map_err(|_| anyhow::anyhow!("KLIPY direct base URL cannot be a base"))?;
|
||||||
|
segments
|
||||||
|
.push(api_key)
|
||||||
|
.push(klipy_resource(path.path_type))
|
||||||
|
.push(&path.slug);
|
||||||
|
}
|
||||||
|
Ok(url)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn transform_gif(&self, input: KlipyGif) -> Option<GifItem> {
|
||||||
|
self.transform_gif_with_path(input, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn transform_gif_with_path(
|
||||||
|
&self,
|
||||||
|
input: KlipyGif,
|
||||||
|
fallback_path: Option<&KlipyPath>,
|
||||||
|
) -> Option<GifItem> {
|
||||||
|
let parsed_path = input.itemurl.as_deref().and_then(parse_klipy_path);
|
||||||
|
let resolved_path = parsed_path.as_ref().or(fallback_path);
|
||||||
|
let explicit_slug = input
|
||||||
|
.slug
|
||||||
|
.as_deref()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|slug| !slug.is_empty());
|
||||||
|
let fallback_id = klipy_id_as_string(&input.id)?;
|
||||||
|
let normalized_slug = explicit_slug
|
||||||
|
.or_else(|| resolved_path.map(|path| path.slug.as_str()))
|
||||||
|
.unwrap_or(fallback_id.as_str())
|
||||||
|
.to_owned();
|
||||||
|
let normalized_type = resolved_path
|
||||||
|
.map(|path| path.path_type)
|
||||||
|
.unwrap_or(KlipyPathType::Gif);
|
||||||
|
let normalized_url = if resolved_path.is_some() || explicit_slug.is_some() {
|
||||||
|
build_share_url_with_type(normalized_type, &normalized_slug)
|
||||||
|
} else {
|
||||||
|
input
|
||||||
|
.itemurl
|
||||||
|
.clone()
|
||||||
|
.unwrap_or_else(|| build_share_url_with_type(normalized_type, &normalized_slug))
|
||||||
|
};
|
||||||
|
let (media, preferred) = self.collect_media(&input);
|
||||||
|
let top = media.get("webm").cloned().or(preferred)?;
|
||||||
|
Some(GifItem {
|
||||||
|
id: normalized_slug.clone(),
|
||||||
|
slug: normalized_slug,
|
||||||
|
provider: KLIPY_PROVIDER_NAME.to_owned(),
|
||||||
|
title: input.title,
|
||||||
|
url: normalized_url,
|
||||||
|
src: top.src.clone(),
|
||||||
|
proxy_src: top.proxy_src.clone(),
|
||||||
|
width: top.width,
|
||||||
|
height: top.height,
|
||||||
|
media,
|
||||||
|
placeholder: None,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_media(
|
||||||
|
&self,
|
||||||
|
input: &KlipyGif,
|
||||||
|
) -> (BTreeMap<String, GifMediaFormat>, Option<GifMediaFormat>) {
|
||||||
|
let mut media = BTreeMap::new();
|
||||||
|
let mut preferred = None;
|
||||||
|
for size in SIZE_PREFERENCE {
|
||||||
|
let Some(bucket) = input.file.as_ref().and_then(|files| files.get(size)) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
for format in FORMAT_PREFERENCE {
|
||||||
|
let Some(entry) = bucket.get(format) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let Some(media_format) = self.to_media_format(entry) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let public_key = public_format_key(size, format);
|
||||||
|
media.insert(public_key, media_format.clone());
|
||||||
|
if preferred.is_none() {
|
||||||
|
preferred = Some(media_format);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if media.is_empty()
|
||||||
|
&& let Some(webm) = input
|
||||||
|
.media_formats
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|formats| formats.webm.as_ref())
|
||||||
|
&& webm.dims[0] > 0
|
||||||
|
&& webm.dims[1] > 0
|
||||||
|
&& let Some(proxy_src) = self.media_proxy.external_proxy_url(&webm.url)
|
||||||
|
{
|
||||||
|
let fallback = GifMediaFormat {
|
||||||
|
src: webm.url.clone(),
|
||||||
|
proxy_src,
|
||||||
|
width: webm.dims[0],
|
||||||
|
height: webm.dims[1],
|
||||||
|
};
|
||||||
|
media.insert("webm".to_owned(), fallback.clone());
|
||||||
|
preferred = Some(fallback);
|
||||||
|
}
|
||||||
|
(media, preferred)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn to_media_format(&self, entry: &KlipyFileEntry) -> Option<GifMediaFormat> {
|
||||||
|
let src = entry.url.as_ref()?;
|
||||||
|
let width = entry.width.filter(|width| *width > 0)?;
|
||||||
|
let height = entry.height.filter(|height| *height > 0)?;
|
||||||
|
let proxy_src = self.media_proxy.external_proxy_url(src)?;
|
||||||
|
Some(GifMediaFormat {
|
||||||
|
src: src.clone(),
|
||||||
|
proxy_src,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn normalize_locale(locale: &str) -> String {
|
||||||
|
locale.replace('-', "_")
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn build_share_url(slug: &str) -> String {
|
||||||
|
let trimmed = slug.trim();
|
||||||
|
if trimmed.is_empty() {
|
||||||
|
return "https://klipy.com/gifs".to_owned();
|
||||||
|
}
|
||||||
|
build_share_url_with_type(KlipyPathType::Gif, trimmed)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn extract_slug_from_url(url: &str) -> Option<String> {
|
||||||
|
parse_klipy_path(url).map(|path| path.slug)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn klipy_id_as_string(value: &Value) -> Option<String> {
|
||||||
|
match value {
|
||||||
|
Value::String(value) => {
|
||||||
|
let trimmed = value.trim();
|
||||||
|
(!trimmed.is_empty()).then(|| trimmed.to_owned())
|
||||||
|
}
|
||||||
|
Value::Number(value) => Some(value.to_string()),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_klipy_path(raw_url: &str) -> Option<KlipyPath> {
|
||||||
|
let parsed = Url::parse(raw_url).ok()?;
|
||||||
|
let hostname = parsed.host_str()?.to_ascii_lowercase();
|
||||||
|
if hostname != "klipy.com" && hostname != "www.klipy.com" {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let mut segments = parsed.path_segments()?;
|
||||||
|
let kind = segments.next()?.to_ascii_lowercase();
|
||||||
|
let slug = segments.next()?.trim().to_owned();
|
||||||
|
if slug.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let path_type = match kind.as_str() {
|
||||||
|
"gif" | "gifs" => KlipyPathType::Gif,
|
||||||
|
"clip" | "clips" => KlipyPathType::Clip,
|
||||||
|
_ => return None,
|
||||||
|
};
|
||||||
|
Some(KlipyPath { path_type, slug })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_share_url_with_type(path_type: KlipyPathType, slug: &str) -> String {
|
||||||
|
let base_path = match path_type {
|
||||||
|
KlipyPathType::Gif => "gifs",
|
||||||
|
KlipyPathType::Clip => "clips",
|
||||||
|
};
|
||||||
|
let encoded_slug = urlencoding::encode(slug);
|
||||||
|
format!("https://klipy.com/{base_path}/{encoded_slug}")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn klipy_resource(path_type: KlipyPathType) -> &'static str {
|
||||||
|
match path_type {
|
||||||
|
KlipyPathType::Gif => "gifs",
|
||||||
|
KlipyPathType::Clip => "clips",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn public_format_key(size: &str, format: &str) -> String {
|
||||||
|
match (size, format) {
|
||||||
|
("hd", "webm") => "webm",
|
||||||
|
("hd", "mp4") => "mp4",
|
||||||
|
("hd", "webp") => "webp",
|
||||||
|
("hd", "gif") => "gif",
|
||||||
|
("md", "webm") => "mediumwebm",
|
||||||
|
("md", "mp4") => "mediummp4",
|
||||||
|
("md", "webp") => "mediumwebp",
|
||||||
|
("md", "gif") => "mediumgif",
|
||||||
|
("sm", "webm") => "tinywebm",
|
||||||
|
("sm", "mp4") => "tinymp4",
|
||||||
|
("sm", "webp") => "tinywebp",
|
||||||
|
("sm", "gif") => "tinygif",
|
||||||
|
("xs", "webm") => "nanowebm",
|
||||||
|
("xs", "mp4") => "nanomp4",
|
||||||
|
("xs", "webp") => "nanowebp",
|
||||||
|
("xs", "gif") => "nanogif",
|
||||||
|
_ => format,
|
||||||
|
}
|
||||||
|
.to_owned()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn category_response(name: String, gif: Option<GifItem>) -> GifCategoryTag {
|
||||||
|
GifCategoryTag {
|
||||||
|
src: gif.as_ref().map(|gif| gif.src.clone()).unwrap_or_default(),
|
||||||
|
proxy_src: gif
|
||||||
|
.as_ref()
|
||||||
|
.map(|gif| gif.proxy_src.clone())
|
||||||
|
.unwrap_or_default(),
|
||||||
|
gif,
|
||||||
|
name,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn locale_uses_klipy_underscore_form() {
|
||||||
|
assert_eq!(normalize_locale("en-US"), "en_US");
|
||||||
|
assert_eq!(normalize_locale("sv_SE"), "sv_SE");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extracts_klipy_slug_only_from_klipy_hosts() {
|
||||||
|
assert_eq!(
|
||||||
|
extract_slug_from_url("https://klipy.com/gifs/funny-123").as_deref(),
|
||||||
|
Some("funny-123")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
extract_slug_from_url("https://www.klipy.com/clip/abc").as_deref(),
|
||||||
|
Some("abc")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
extract_slug_from_url("https://notklipy.com/gifs/funny"),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_share_url_uses_gifs_path() {
|
||||||
|
assert_eq!(build_share_url("hello"), "https://klipy.com/gifs/hello");
|
||||||
|
assert_eq!(build_share_url(" "), "https://klipy.com/gifs");
|
||||||
|
assert_eq!(
|
||||||
|
build_share_url_with_type(KlipyPathType::Clip, "hello"),
|
||||||
|
"https://klipy.com/clips/hello"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stringifies_numeric_klipy_ids() {
|
||||||
|
assert_eq!(
|
||||||
|
klipy_id_as_string(&serde_json::json!(2484942301552561_i64)).as_deref(),
|
||||||
|
Some("2484942301552561")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
klipy_id_as_string(&serde_json::json!(" abc ")).as_deref(),
|
||||||
|
Some("abc")
|
||||||
|
);
|
||||||
|
assert_eq!(klipy_id_as_string(&serde_json::json!(" ")), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn maps_provider_format_keys() {
|
||||||
|
assert_eq!(public_format_key("hd", "webm"), "webm");
|
||||||
|
assert_eq!(public_format_key("sm", "gif"), "tinygif");
|
||||||
|
assert_eq!(public_format_key("xs", "webp"), "nanowebp");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore]
|
||||||
|
async fn live_resolves_klipy_url_with_direct_lookup() {
|
||||||
|
let api_key = std::env::var("FLUXER_KLIPY_API_KEY")
|
||||||
|
.or_else(|_| std::env::var("KLIPY_API_KEY"))
|
||||||
|
.expect("FLUXER_KLIPY_API_KEY or KLIPY_API_KEY set");
|
||||||
|
let client =
|
||||||
|
KlipyClient::new(MediaProxyUrlBuilder::from_env().expect("media proxy env configured"))
|
||||||
|
.expect("KLIPY client");
|
||||||
|
|
||||||
|
let gif = client
|
||||||
|
.resolve_by_url(
|
||||||
|
&api_key,
|
||||||
|
"https://klipy.com/gifs/goatplaybanjo-chat-4",
|
||||||
|
"en-US",
|
||||||
|
"US",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("KLIPY direct lookup")
|
||||||
|
.expect("resolved GIF");
|
||||||
|
|
||||||
|
assert_eq!(gif.slug, "goatplaybanjo-chat-4");
|
||||||
|
assert_eq!(gif.provider, KLIPY_PROVIDER_NAME);
|
||||||
|
assert_eq!(gif.url, "https://klipy.com/gifs/goatplaybanjo-chat-4");
|
||||||
|
assert!(gif.width > 0);
|
||||||
|
assert!(gif.height > 0);
|
||||||
|
assert!(gif.media.contains_key("webm") || gif.media.contains_key("mp4"));
|
||||||
|
assert!(gif.proxy_src.starts_with("http"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
mod klipy;
|
||||||
|
mod media_proxy;
|
||||||
|
mod router_impl;
|
||||||
|
mod shard_impl;
|
||||||
|
mod types;
|
||||||
|
|
||||||
|
use fluxer_svc::config::{Mode, ServiceConfig};
|
||||||
|
use fluxer_svc::transport::NatsTransport;
|
||||||
|
use router_impl::GifsRouter;
|
||||||
|
use shard_impl::GifsShard;
|
||||||
|
|
||||||
|
#[tokio::main]
|
||||||
|
async fn main() -> anyhow::Result<()> {
|
||||||
|
fluxer_svc::init_tracing();
|
||||||
|
let config = ServiceConfig::from_env()?;
|
||||||
|
let transport = NatsTransport::connect(&config.nats_url).await?;
|
||||||
|
|
||||||
|
tracing::info!(
|
||||||
|
service = config.service_name,
|
||||||
|
mode = ?config.mode,
|
||||||
|
shard_id = config.shard_id,
|
||||||
|
shard_count = config.shard_count,
|
||||||
|
listen_addr = %config.listen_addr,
|
||||||
|
"starting gifs service"
|
||||||
|
);
|
||||||
|
|
||||||
|
match config.mode {
|
||||||
|
Mode::Router => {
|
||||||
|
let router = GifsRouter::new(config.cache_max_entries, config.cache_ttl);
|
||||||
|
fluxer_svc::router::run_router(&config, router, transport).await
|
||||||
|
}
|
||||||
|
Mode::Shard => {
|
||||||
|
let shard = GifsShard::new(&config)?;
|
||||||
|
fluxer_svc::shard::run_shard(&config, shard, transport).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,216 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
use base64::prelude::*;
|
||||||
|
use hmac::{Hmac, KeyInit, Mac};
|
||||||
|
use sha2::Sha256;
|
||||||
|
use url::Url;
|
||||||
|
|
||||||
|
const V2_PATH_PREFIX: &str = "v2/";
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct MediaProxyUrlBuilder {
|
||||||
|
endpoint: String,
|
||||||
|
endpoint_host: Option<String>,
|
||||||
|
secret_key: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MediaProxyUrlBuilder {
|
||||||
|
pub fn from_env() -> anyhow::Result<Self> {
|
||||||
|
let endpoint = std::env::var("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT")
|
||||||
|
.or_else(|_| std::env::var("FLUXER_MEDIA_ENDPOINT"))
|
||||||
|
.unwrap_or_default();
|
||||||
|
if endpoint.trim().is_empty() {
|
||||||
|
anyhow::bail!(
|
||||||
|
"gifs shard requires FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT or FLUXER_MEDIA_ENDPOINT"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let secret_key = std::env::var("FLUXER_MEDIA_PROXY_SECRET_KEY").unwrap_or_default();
|
||||||
|
if secret_key.trim().is_empty() {
|
||||||
|
anyhow::bail!("gifs shard requires FLUXER_MEDIA_PROXY_SECRET_KEY");
|
||||||
|
}
|
||||||
|
|
||||||
|
let endpoint = endpoint.trim_end_matches('/').to_owned();
|
||||||
|
let endpoint_host = Url::parse(&endpoint)
|
||||||
|
.ok()
|
||||||
|
.and_then(|parsed| parsed.host_str().map(ToOwned::to_owned));
|
||||||
|
|
||||||
|
Ok(Self {
|
||||||
|
endpoint,
|
||||||
|
endpoint_host,
|
||||||
|
secret_key,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn external_proxy_url(&self, input_url: &str) -> Option<String> {
|
||||||
|
let parsed = Url::parse(input_url).ok()?;
|
||||||
|
if self
|
||||||
|
.endpoint_host
|
||||||
|
.as_deref()
|
||||||
|
.is_some_and(|host| parsed.host_str() == Some(host))
|
||||||
|
{
|
||||||
|
return Some(input_url.to_owned());
|
||||||
|
}
|
||||||
|
|
||||||
|
let proxy_path = build_external_media_proxy_path(parsed.as_str());
|
||||||
|
let signature = create_signature(&proxy_path, &self.secret_key);
|
||||||
|
Some(format!(
|
||||||
|
"{}/external/{signature}/{proxy_path}",
|
||||||
|
self.endpoint
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_external_media_proxy_path(input_url: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"{V2_PATH_PREFIX}{}",
|
||||||
|
BASE64_URL_SAFE_NO_PAD.encode(input_url)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_signature(input: &str, secret: &str) -> String {
|
||||||
|
let mut mac = Hmac::<Sha256>::new_from_slice(secret.as_bytes()).expect("HMAC accepts any key");
|
||||||
|
mac.update(input.as_bytes());
|
||||||
|
BASE64_URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::sync::Mutex;
|
||||||
|
|
||||||
|
static ENV_LOCK: Mutex<()> = Mutex::new(());
|
||||||
|
|
||||||
|
fn with_media_proxy_env(
|
||||||
|
vars: &[(&str, Option<&str>)],
|
||||||
|
test: impl FnOnce() -> anyhow::Result<()>,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let _guard = ENV_LOCK.lock().unwrap();
|
||||||
|
let keys = [
|
||||||
|
"FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT",
|
||||||
|
"FLUXER_MEDIA_ENDPOINT",
|
||||||
|
"FLUXER_MEDIA_PROXY_ENDPOINT",
|
||||||
|
"FLUXER_MEDIA_PROXY_SECRET_KEY",
|
||||||
|
];
|
||||||
|
let saved = keys
|
||||||
|
.iter()
|
||||||
|
.map(|key| (*key, std::env::var(key).ok()))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
for key in keys {
|
||||||
|
unsafe {
|
||||||
|
std::env::remove_var(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (key, value) in vars {
|
||||||
|
if let Some(value) = value {
|
||||||
|
unsafe {
|
||||||
|
std::env::set_var(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let result = test();
|
||||||
|
|
||||||
|
for (key, value) in saved {
|
||||||
|
match value {
|
||||||
|
Some(value) => unsafe {
|
||||||
|
std::env::set_var(key, value);
|
||||||
|
},
|
||||||
|
None => unsafe {
|
||||||
|
std::env::remove_var(key);
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn external_proxy_url_builds_v2_signed_url() {
|
||||||
|
let builder = MediaProxyUrlBuilder {
|
||||||
|
endpoint: "https://media.example.test".to_owned(),
|
||||||
|
endpoint_host: Some("media.example.test".to_owned()),
|
||||||
|
secret_key: "secret".to_owned(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let url = builder
|
||||||
|
.external_proxy_url("https://img.klipy.com/a.webp?x=1")
|
||||||
|
.expect("proxy url");
|
||||||
|
|
||||||
|
assert!(url.starts_with("https://media.example.test/external/"));
|
||||||
|
assert!(url.contains("/v2/"));
|
||||||
|
assert_eq!(
|
||||||
|
builder.external_proxy_url("https://media.example.test/external/existing"),
|
||||||
|
Some("https://media.example.test/external/existing".to_owned())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn from_env_uses_public_endpoint_when_internal_proxy_endpoint_is_set() -> anyhow::Result<()> {
|
||||||
|
with_media_proxy_env(
|
||||||
|
&[
|
||||||
|
(
|
||||||
|
"FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT",
|
||||||
|
Some("https://media.example.test/"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"FLUXER_MEDIA_PROXY_ENDPOINT",
|
||||||
|
Some("http://media-proxy:8080"),
|
||||||
|
),
|
||||||
|
("FLUXER_MEDIA_PROXY_SECRET_KEY", Some("secret")),
|
||||||
|
],
|
||||||
|
|| {
|
||||||
|
let builder = MediaProxyUrlBuilder::from_env()?;
|
||||||
|
|
||||||
|
assert_eq!(builder.endpoint, "https://media.example.test");
|
||||||
|
Ok(())
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn from_env_accepts_legacy_public_media_endpoint() -> anyhow::Result<()> {
|
||||||
|
with_media_proxy_env(
|
||||||
|
&[
|
||||||
|
(
|
||||||
|
"FLUXER_MEDIA_ENDPOINT",
|
||||||
|
Some("https://media.example.test/media"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"FLUXER_MEDIA_PROXY_ENDPOINT",
|
||||||
|
Some("http://media-proxy:8080"),
|
||||||
|
),
|
||||||
|
("FLUXER_MEDIA_PROXY_SECRET_KEY", Some("secret")),
|
||||||
|
],
|
||||||
|
|| {
|
||||||
|
let builder = MediaProxyUrlBuilder::from_env()?;
|
||||||
|
|
||||||
|
assert_eq!(builder.endpoint, "https://media.example.test/media");
|
||||||
|
Ok(())
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn from_env_rejects_internal_proxy_endpoint_without_public_endpoint() -> anyhow::Result<()> {
|
||||||
|
with_media_proxy_env(
|
||||||
|
&[
|
||||||
|
(
|
||||||
|
"FLUXER_MEDIA_PROXY_ENDPOINT",
|
||||||
|
Some("http://media-proxy:8080"),
|
||||||
|
),
|
||||||
|
("FLUXER_MEDIA_PROXY_SECRET_KEY", Some("secret")),
|
||||||
|
],
|
||||||
|
|| {
|
||||||
|
let err = MediaProxyUrlBuilder::from_env()
|
||||||
|
.err()
|
||||||
|
.expect("internal endpoint must not be accepted as public endpoint")
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
assert!(err.contains("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT"));
|
||||||
|
Ok(())
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
use crate::types::{GifRequest, GifServiceResponse};
|
||||||
|
use fluxer_svc::router::RouterService;
|
||||||
|
use moka::sync::Cache;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
pub struct GifsRouter {
|
||||||
|
l1: Cache<String, GifServiceResponse>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl GifsRouter {
|
||||||
|
pub fn new(max_entries: u64, ttl: Duration) -> Self {
|
||||||
|
Self {
|
||||||
|
l1: Cache::builder()
|
||||||
|
.max_capacity(max_entries)
|
||||||
|
.time_to_live(ttl)
|
||||||
|
.build(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RouterService for GifsRouter {
|
||||||
|
type Request = GifRequest;
|
||||||
|
type Response = GifServiceResponse;
|
||||||
|
|
||||||
|
fn service_name(&self) -> &str {
|
||||||
|
"gifs"
|
||||||
|
}
|
||||||
|
|
||||||
|
fn route_key(req: &GifRequest) -> String {
|
||||||
|
request_key(req).unwrap_or_else(|| "uncached".to_owned())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn coalesce_key(req: &GifRequest) -> Option<String> {
|
||||||
|
request_key(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn l1_lookup(&self, req: &GifRequest) -> Option<GifServiceResponse> {
|
||||||
|
request_l1_key(req).and_then(|key| self.l1.get(&key))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn l1_insert(&self, req: &GifRequest, resp: &GifServiceResponse) {
|
||||||
|
if response_is_cacheable(resp)
|
||||||
|
&& let Some(key) = request_l1_key(req)
|
||||||
|
{
|
||||||
|
self.l1.insert(key, resp.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn l1_invalidate(&self, key: &str) {
|
||||||
|
self.l1.invalidate(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request_l1_key(req: &GifRequest) -> Option<String> {
|
||||||
|
match req {
|
||||||
|
GifRequest::Search { .. }
|
||||||
|
| GifRequest::GetFeatured { .. }
|
||||||
|
| GifRequest::GetTrendingGifs { .. }
|
||||||
|
| GifRequest::Suggest { .. }
|
||||||
|
| GifRequest::ResolveByUrl { .. }
|
||||||
|
| GifRequest::BuildShareUrl { .. }
|
||||||
|
| GifRequest::ExtractSlugFromUrl { .. } => request_key(req),
|
||||||
|
GifRequest::IsAvailable { .. } | GifRequest::RegisterShare { .. } => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request_key(req: &GifRequest) -> Option<String> {
|
||||||
|
match req {
|
||||||
|
GifRequest::IsAvailable { .. } => None,
|
||||||
|
GifRequest::Search {
|
||||||
|
q, locale, country, ..
|
||||||
|
} => Some(format!("search:{locale}:{country}:{q}")),
|
||||||
|
GifRequest::GetFeatured {
|
||||||
|
locale, country, ..
|
||||||
|
} => Some(format!("featured:{locale}:{country}")),
|
||||||
|
GifRequest::GetTrendingGifs {
|
||||||
|
locale, country, ..
|
||||||
|
} => Some(format!("trending:{locale}:{country}")),
|
||||||
|
GifRequest::Suggest { q, locale, .. } => Some(format!("suggest:{locale}:{q}")),
|
||||||
|
GifRequest::RegisterShare { .. } => None,
|
||||||
|
GifRequest::ResolveByUrl {
|
||||||
|
url,
|
||||||
|
locale,
|
||||||
|
country,
|
||||||
|
..
|
||||||
|
} => Some(format!("resolve:{locale}:{country}:{url}")),
|
||||||
|
GifRequest::BuildShareUrl { slug } => Some(format!("share-url:{slug}")),
|
||||||
|
GifRequest::ExtractSlugFromUrl { url } => Some(format!("extract-slug:{url}")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn response_is_cacheable(resp: &GifServiceResponse) -> bool {
|
||||||
|
!matches!(
|
||||||
|
resp,
|
||||||
|
GifServiceResponse::Available { .. }
|
||||||
|
| GifServiceResponse::Registered
|
||||||
|
| GifServiceResponse::Failed { .. }
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn coalesce_key_excludes_api_key() {
|
||||||
|
let first = GifRequest::Search {
|
||||||
|
api_key: "a".to_owned(),
|
||||||
|
q: "wave".to_owned(),
|
||||||
|
locale: "en_US".to_owned(),
|
||||||
|
country: "US".to_owned(),
|
||||||
|
};
|
||||||
|
let second = GifRequest::Search {
|
||||||
|
api_key: "b".to_owned(),
|
||||||
|
q: "wave".to_owned(),
|
||||||
|
locale: "en_US".to_owned(),
|
||||||
|
country: "US".to_owned(),
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
GifsRouter::coalesce_key(&first),
|
||||||
|
GifsRouter::coalesce_key(&second)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn trending_key_is_locale_country_scoped_and_api_key_free() {
|
||||||
|
let base = GifRequest::GetTrendingGifs {
|
||||||
|
api_key: "a".to_owned(),
|
||||||
|
locale: "en_US".to_owned(),
|
||||||
|
country: "US".to_owned(),
|
||||||
|
};
|
||||||
|
let different_api_key = GifRequest::GetTrendingGifs {
|
||||||
|
api_key: "b".to_owned(),
|
||||||
|
locale: "en_US".to_owned(),
|
||||||
|
country: "US".to_owned(),
|
||||||
|
};
|
||||||
|
let different_locale = GifRequest::GetTrendingGifs {
|
||||||
|
api_key: "a".to_owned(),
|
||||||
|
locale: "sv_SE".to_owned(),
|
||||||
|
country: "US".to_owned(),
|
||||||
|
};
|
||||||
|
let different_country = GifRequest::GetTrendingGifs {
|
||||||
|
api_key: "a".to_owned(),
|
||||||
|
locale: "en_US".to_owned(),
|
||||||
|
country: "SE".to_owned(),
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
GifsRouter::coalesce_key(&base),
|
||||||
|
Some("trending:en_US:US".to_owned())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
GifsRouter::coalesce_key(&base),
|
||||||
|
GifsRouter::coalesce_key(&different_api_key)
|
||||||
|
);
|
||||||
|
assert_ne!(
|
||||||
|
GifsRouter::coalesce_key(&base),
|
||||||
|
GifsRouter::coalesce_key(&different_locale)
|
||||||
|
);
|
||||||
|
assert_ne!(
|
||||||
|
GifsRouter::coalesce_key(&base),
|
||||||
|
GifsRouter::coalesce_key(&different_country)
|
||||||
|
);
|
||||||
|
assert_eq!(request_l1_key(&base), Some("trending:en_US:US".to_owned()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn register_share_is_not_cached_or_coalesced() {
|
||||||
|
let request = GifRequest::RegisterShare {
|
||||||
|
api_key: "key".to_owned(),
|
||||||
|
id: "gif".to_owned(),
|
||||||
|
q: "wave".to_owned(),
|
||||||
|
locale: "en_US".to_owned(),
|
||||||
|
country: "US".to_owned(),
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(GifsRouter::coalesce_key(&request), None);
|
||||||
|
assert_eq!(request_l1_key(&request), None);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,410 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
use crate::klipy::{KlipyClient, build_share_url, extract_slug_from_url};
|
||||||
|
use crate::media_proxy::MediaProxyUrlBuilder;
|
||||||
|
use crate::types::{GifCategoryTag, GifItem, GifRequest, GifServiceResponse};
|
||||||
|
use fluxer_svc::config::ServiceConfig;
|
||||||
|
use fluxer_svc::shard::ShardService;
|
||||||
|
use moka::future::Cache;
|
||||||
|
use std::collections::HashSet;
|
||||||
|
use std::future::Future;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
use tokio::sync::Mutex;
|
||||||
|
|
||||||
|
const SEARCH_SOFT_TTL: Duration = Duration::from_secs(30);
|
||||||
|
const SEARCH_HARD_TTL: Duration = Duration::from_secs(5 * 60);
|
||||||
|
const SUGGEST_SOFT_TTL: Duration = Duration::from_secs(60);
|
||||||
|
const SUGGEST_HARD_TTL: Duration = Duration::from_secs(10 * 60);
|
||||||
|
const FEATURED_GIFS_SOFT_TTL: Duration = Duration::from_secs(5 * 60);
|
||||||
|
const FEATURED_GIFS_HARD_TTL: Duration = Duration::from_secs(30 * 60);
|
||||||
|
const CATEGORIES_SOFT_TTL: Duration = Duration::from_secs(24 * 60 * 60);
|
||||||
|
const CATEGORIES_HARD_TTL: Duration = Duration::from_secs(48 * 60 * 60);
|
||||||
|
const RESOLVE_SOFT_TTL: Duration = Duration::from_secs(30 * 60);
|
||||||
|
const RESOLVE_HARD_TTL: Duration = Duration::from_secs(2 * 60 * 60);
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct GifsShard {
|
||||||
|
inner: Arc<GifsShardInner>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct GifsShardInner {
|
||||||
|
klipy: KlipyClient,
|
||||||
|
gif_lists: Cache<String, Cached<Vec<GifItem>>>,
|
||||||
|
categories: Cache<String, Cached<Vec<GifCategoryTag>>>,
|
||||||
|
suggestions: Cache<String, Cached<Vec<String>>>,
|
||||||
|
resolved: Cache<String, Cached<Option<GifItem>>>,
|
||||||
|
refreshing: Mutex<HashSet<String>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct Cached<T> {
|
||||||
|
data: T,
|
||||||
|
stored_at: Instant,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
struct CachePolicy {
|
||||||
|
soft_ttl: Duration,
|
||||||
|
hard_ttl: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CachePolicy {
|
||||||
|
const fn new(soft_ttl: Duration, hard_ttl: Duration) -> Self {
|
||||||
|
Self { soft_ttl, hard_ttl }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> Cached<T> {
|
||||||
|
fn new(data: T) -> Self {
|
||||||
|
Self {
|
||||||
|
data,
|
||||||
|
stored_at: Instant::now(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn age(&self) -> Duration {
|
||||||
|
self.stored_at.elapsed()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl GifsShard {
|
||||||
|
pub fn new(config: &ServiceConfig) -> anyhow::Result<Self> {
|
||||||
|
let media_proxy = MediaProxyUrlBuilder::from_env()?;
|
||||||
|
let klipy = KlipyClient::new(media_proxy)?;
|
||||||
|
let max_capacity = config.cache_max_entries;
|
||||||
|
let max_cache_ttl = CATEGORIES_HARD_TTL;
|
||||||
|
Ok(Self {
|
||||||
|
inner: Arc::new(GifsShardInner {
|
||||||
|
klipy,
|
||||||
|
gif_lists: build_cache(max_capacity, max_cache_ttl),
|
||||||
|
categories: build_cache(max_capacity, max_cache_ttl),
|
||||||
|
suggestions: build_cache(max_capacity, max_cache_ttl),
|
||||||
|
resolved: build_cache(max_capacity, max_cache_ttl),
|
||||||
|
refreshing: Mutex::new(HashSet::new()),
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_cached<T, Fetch, Fut>(
|
||||||
|
&self,
|
||||||
|
cache: Cache<String, Cached<T>>,
|
||||||
|
key: String,
|
||||||
|
policy: CachePolicy,
|
||||||
|
fetch: Fetch,
|
||||||
|
) -> anyhow::Result<T>
|
||||||
|
where
|
||||||
|
T: Clone + Send + Sync + 'static,
|
||||||
|
Fetch: Fn() -> Fut + Clone + Send + Sync + 'static,
|
||||||
|
Fut: Future<Output = anyhow::Result<T>> + Send + 'static,
|
||||||
|
{
|
||||||
|
if let Some(cached) = cache.get(&key).await {
|
||||||
|
let age = cached.age();
|
||||||
|
if age <= policy.soft_ttl {
|
||||||
|
return Ok(cached.data);
|
||||||
|
}
|
||||||
|
if age <= policy.hard_ttl {
|
||||||
|
self.trigger_background_refresh(cache, key, policy, fetch);
|
||||||
|
return Ok(cached.data);
|
||||||
|
}
|
||||||
|
cache.invalidate(&key).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
let fetch_for_load = fetch.clone();
|
||||||
|
let cached = cache
|
||||||
|
.try_get_with(key, async move {
|
||||||
|
let data = fetch_for_load().await?;
|
||||||
|
Ok::<Cached<T>, anyhow::Error>(Cached::new(data))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|error| anyhow::anyhow!("{}", error.as_ref()))?;
|
||||||
|
Ok(cached.data)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn trigger_background_refresh<T, Fetch, Fut>(
|
||||||
|
&self,
|
||||||
|
cache: Cache<String, Cached<T>>,
|
||||||
|
key: String,
|
||||||
|
_policy: CachePolicy,
|
||||||
|
fetch: Fetch,
|
||||||
|
) where
|
||||||
|
T: Clone + Send + Sync + 'static,
|
||||||
|
Fetch: Fn() -> Fut + Clone + Send + Sync + 'static,
|
||||||
|
Fut: Future<Output = anyhow::Result<T>> + Send + 'static,
|
||||||
|
{
|
||||||
|
let this = self.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
{
|
||||||
|
let mut refreshing = this.inner.refreshing.lock().await;
|
||||||
|
if !refreshing.insert(key.clone()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let result = fetch().await;
|
||||||
|
match result {
|
||||||
|
Ok(data) => {
|
||||||
|
cache.insert(key.clone(), Cached::new(data)).await;
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
tracing::debug!(error = %error, cache_key = %key, "background GIF cache refresh failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut refreshing = this.inner.refreshing.lock().await;
|
||||||
|
refreshing.remove(&key);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_available(&self, api_key: Option<String>) -> GifServiceResponse {
|
||||||
|
GifServiceResponse::Available {
|
||||||
|
available: api_key.as_deref().is_some_and(|key| !key.trim().is_empty()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_search(
|
||||||
|
&self,
|
||||||
|
api_key: String,
|
||||||
|
q: String,
|
||||||
|
locale: String,
|
||||||
|
country: String,
|
||||||
|
) -> anyhow::Result<GifServiceResponse> {
|
||||||
|
let key = format!("search:{locale}:{country}:{q}");
|
||||||
|
let this = self.clone();
|
||||||
|
let gifs = self
|
||||||
|
.get_cached(
|
||||||
|
self.inner.gif_lists.clone(),
|
||||||
|
key,
|
||||||
|
CachePolicy::new(SEARCH_SOFT_TTL, SEARCH_HARD_TTL),
|
||||||
|
move || {
|
||||||
|
let this = this.clone();
|
||||||
|
let api_key = api_key.clone();
|
||||||
|
let q = q.clone();
|
||||||
|
let locale = locale.clone();
|
||||||
|
let country = country.clone();
|
||||||
|
async move {
|
||||||
|
this.inner
|
||||||
|
.klipy
|
||||||
|
.search(&api_key, &q, &locale, &country, 50)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(GifServiceResponse::SearchResults(gifs))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_featured(
|
||||||
|
&self,
|
||||||
|
api_key: String,
|
||||||
|
locale: String,
|
||||||
|
country: String,
|
||||||
|
) -> anyhow::Result<GifServiceResponse> {
|
||||||
|
let gifs_key = format!("featured_gifs:{locale}:{country}");
|
||||||
|
let categories_key = format!("featured_categories:{locale}");
|
||||||
|
let gifs_this = self.clone();
|
||||||
|
let categories_this = self.clone();
|
||||||
|
let api_key_for_gifs = api_key.clone();
|
||||||
|
let locale_for_gifs = locale.clone();
|
||||||
|
let country_for_gifs = country.clone();
|
||||||
|
let api_key_for_categories = api_key;
|
||||||
|
let locale_for_categories = locale;
|
||||||
|
|
||||||
|
let gifs_future = self.get_cached(
|
||||||
|
self.inner.gif_lists.clone(),
|
||||||
|
gifs_key,
|
||||||
|
CachePolicy::new(FEATURED_GIFS_SOFT_TTL, FEATURED_GIFS_HARD_TTL),
|
||||||
|
move || {
|
||||||
|
let this = gifs_this.clone();
|
||||||
|
let api_key = api_key_for_gifs.clone();
|
||||||
|
let locale = locale_for_gifs.clone();
|
||||||
|
let country = country_for_gifs.clone();
|
||||||
|
async move {
|
||||||
|
this.inner
|
||||||
|
.klipy
|
||||||
|
.featured_gifs(&api_key, &locale, &country)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let categories_future = self.get_cached(
|
||||||
|
self.inner.categories.clone(),
|
||||||
|
categories_key,
|
||||||
|
CachePolicy::new(CATEGORIES_SOFT_TTL, CATEGORIES_HARD_TTL),
|
||||||
|
move || {
|
||||||
|
let this = categories_this.clone();
|
||||||
|
let api_key = api_key_for_categories.clone();
|
||||||
|
let locale = locale_for_categories.clone();
|
||||||
|
async move {
|
||||||
|
this.inner
|
||||||
|
.klipy
|
||||||
|
.featured_categories(&api_key, &locale)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
let (gifs, categories) = tokio::try_join!(gifs_future, categories_future)?;
|
||||||
|
Ok(GifServiceResponse::Featured { gifs, categories })
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_trending(
|
||||||
|
&self,
|
||||||
|
api_key: String,
|
||||||
|
locale: String,
|
||||||
|
country: String,
|
||||||
|
) -> anyhow::Result<GifServiceResponse> {
|
||||||
|
let key = format!("trending:{locale}:{country}");
|
||||||
|
let this = self.clone();
|
||||||
|
let gifs = self
|
||||||
|
.get_cached(
|
||||||
|
self.inner.gif_lists.clone(),
|
||||||
|
key,
|
||||||
|
CachePolicy::new(FEATURED_GIFS_SOFT_TTL, FEATURED_GIFS_HARD_TTL),
|
||||||
|
move || {
|
||||||
|
let this = this.clone();
|
||||||
|
let api_key = api_key.clone();
|
||||||
|
let locale = locale.clone();
|
||||||
|
let country = country.clone();
|
||||||
|
async move {
|
||||||
|
this.inner
|
||||||
|
.klipy
|
||||||
|
.trending_gifs(&api_key, &locale, &country)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(GifServiceResponse::TrendingResults(gifs))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_suggest(
|
||||||
|
&self,
|
||||||
|
api_key: String,
|
||||||
|
q: String,
|
||||||
|
locale: String,
|
||||||
|
) -> anyhow::Result<GifServiceResponse> {
|
||||||
|
let key = format!("suggest:{locale}:{q}");
|
||||||
|
let this = self.clone();
|
||||||
|
let suggestions = self
|
||||||
|
.get_cached(
|
||||||
|
self.inner.suggestions.clone(),
|
||||||
|
key,
|
||||||
|
CachePolicy::new(SUGGEST_SOFT_TTL, SUGGEST_HARD_TTL),
|
||||||
|
move || {
|
||||||
|
let this = this.clone();
|
||||||
|
let api_key = api_key.clone();
|
||||||
|
let q = q.clone();
|
||||||
|
let locale = locale.clone();
|
||||||
|
async move { this.inner.klipy.suggestions(&api_key, &q, &locale).await }
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(GifServiceResponse::Suggestions(suggestions))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_resolve_by_url(
|
||||||
|
&self,
|
||||||
|
api_key: String,
|
||||||
|
url: String,
|
||||||
|
locale: String,
|
||||||
|
country: String,
|
||||||
|
) -> anyhow::Result<GifServiceResponse> {
|
||||||
|
let key = format!("resolve:{locale}:{country}:{url}");
|
||||||
|
let this = self.clone();
|
||||||
|
let gif = self
|
||||||
|
.get_cached(
|
||||||
|
self.inner.resolved.clone(),
|
||||||
|
key,
|
||||||
|
CachePolicy::new(RESOLVE_SOFT_TTL, RESOLVE_HARD_TTL),
|
||||||
|
move || {
|
||||||
|
let this = this.clone();
|
||||||
|
let api_key = api_key.clone();
|
||||||
|
let url = url.clone();
|
||||||
|
let locale = locale.clone();
|
||||||
|
let country = country.clone();
|
||||||
|
async move {
|
||||||
|
this.inner
|
||||||
|
.klipy
|
||||||
|
.resolve_by_url(&api_key, &url, &locale, &country)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(GifServiceResponse::Resolved { gif })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ShardService for GifsShard {
|
||||||
|
type Request = GifRequest;
|
||||||
|
type Response = GifServiceResponse;
|
||||||
|
|
||||||
|
fn service_name(&self) -> &str {
|
||||||
|
"gifs"
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle(&self, request: GifRequest) -> anyhow::Result<GifServiceResponse> {
|
||||||
|
let response = match request {
|
||||||
|
GifRequest::IsAvailable { api_key } => self.handle_available(api_key).await,
|
||||||
|
GifRequest::Search {
|
||||||
|
api_key,
|
||||||
|
q,
|
||||||
|
locale,
|
||||||
|
country,
|
||||||
|
} => self.handle_search(api_key, q, locale, country).await?,
|
||||||
|
GifRequest::GetFeatured {
|
||||||
|
api_key,
|
||||||
|
locale,
|
||||||
|
country,
|
||||||
|
} => self.handle_featured(api_key, locale, country).await?,
|
||||||
|
GifRequest::GetTrendingGifs {
|
||||||
|
api_key,
|
||||||
|
locale,
|
||||||
|
country,
|
||||||
|
} => self.handle_trending(api_key, locale, country).await?,
|
||||||
|
GifRequest::Suggest { api_key, q, locale } => {
|
||||||
|
self.handle_suggest(api_key, q, locale).await?
|
||||||
|
}
|
||||||
|
GifRequest::RegisterShare {
|
||||||
|
api_key,
|
||||||
|
id,
|
||||||
|
q,
|
||||||
|
locale,
|
||||||
|
country,
|
||||||
|
} => {
|
||||||
|
self.inner
|
||||||
|
.klipy
|
||||||
|
.register_share(&api_key, &id, &q, &locale, &country)
|
||||||
|
.await?;
|
||||||
|
GifServiceResponse::Registered
|
||||||
|
}
|
||||||
|
GifRequest::ResolveByUrl {
|
||||||
|
api_key,
|
||||||
|
url,
|
||||||
|
locale,
|
||||||
|
country,
|
||||||
|
} => {
|
||||||
|
self.handle_resolve_by_url(api_key, url, locale, country)
|
||||||
|
.await?
|
||||||
|
}
|
||||||
|
GifRequest::BuildShareUrl { slug } => GifServiceResponse::ShareUrl {
|
||||||
|
url: build_share_url(&slug),
|
||||||
|
},
|
||||||
|
GifRequest::ExtractSlugFromUrl { url } => GifServiceResponse::ExtractedSlug {
|
||||||
|
slug: extract_slug_from_url(&url),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
Ok(response)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_cache<T>(max_capacity: u64, time_to_live: Duration) -> Cache<String, Cached<T>>
|
||||||
|
where
|
||||||
|
T: Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
Cache::builder()
|
||||||
|
.max_capacity(max_capacity)
|
||||||
|
.time_to_live(time_to_live)
|
||||||
|
.build()
|
||||||
|
}
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
#[serde(tag = "op", rename_all_fields = "snake_case")]
|
||||||
|
pub enum GifRequest {
|
||||||
|
IsAvailable {
|
||||||
|
api_key: Option<String>,
|
||||||
|
},
|
||||||
|
Search {
|
||||||
|
api_key: String,
|
||||||
|
q: String,
|
||||||
|
locale: String,
|
||||||
|
country: String,
|
||||||
|
},
|
||||||
|
GetFeatured {
|
||||||
|
api_key: String,
|
||||||
|
locale: String,
|
||||||
|
country: String,
|
||||||
|
},
|
||||||
|
GetTrendingGifs {
|
||||||
|
api_key: String,
|
||||||
|
locale: String,
|
||||||
|
country: String,
|
||||||
|
},
|
||||||
|
Suggest {
|
||||||
|
api_key: String,
|
||||||
|
q: String,
|
||||||
|
locale: String,
|
||||||
|
},
|
||||||
|
RegisterShare {
|
||||||
|
api_key: String,
|
||||||
|
id: String,
|
||||||
|
q: String,
|
||||||
|
locale: String,
|
||||||
|
country: String,
|
||||||
|
},
|
||||||
|
ResolveByUrl {
|
||||||
|
api_key: String,
|
||||||
|
url: String,
|
||||||
|
locale: String,
|
||||||
|
country: String,
|
||||||
|
},
|
||||||
|
BuildShareUrl {
|
||||||
|
slug: String,
|
||||||
|
},
|
||||||
|
ExtractSlugFromUrl {
|
||||||
|
url: String,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub enum GifServiceResponse {
|
||||||
|
Available {
|
||||||
|
available: bool,
|
||||||
|
},
|
||||||
|
SearchResults(Vec<GifItem>),
|
||||||
|
Featured {
|
||||||
|
gifs: Vec<GifItem>,
|
||||||
|
categories: Vec<GifCategoryTag>,
|
||||||
|
},
|
||||||
|
TrendingResults(Vec<GifItem>),
|
||||||
|
Suggestions(Vec<String>),
|
||||||
|
Registered,
|
||||||
|
Resolved {
|
||||||
|
gif: Option<GifItem>,
|
||||||
|
},
|
||||||
|
ShareUrl {
|
||||||
|
url: String,
|
||||||
|
},
|
||||||
|
ExtractedSlug {
|
||||||
|
slug: Option<String>,
|
||||||
|
},
|
||||||
|
Failed {
|
||||||
|
message: String,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct GifMediaFormat {
|
||||||
|
pub src: String,
|
||||||
|
pub proxy_src: String,
|
||||||
|
pub width: i32,
|
||||||
|
pub height: i32,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct GifItem {
|
||||||
|
pub id: String,
|
||||||
|
pub slug: String,
|
||||||
|
pub provider: String,
|
||||||
|
pub title: String,
|
||||||
|
pub url: String,
|
||||||
|
pub src: String,
|
||||||
|
pub proxy_src: String,
|
||||||
|
pub width: i32,
|
||||||
|
pub height: i32,
|
||||||
|
pub media: BTreeMap<String, GifMediaFormat>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub placeholder: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct GifCategoryTag {
|
||||||
|
pub name: String,
|
||||||
|
pub src: String,
|
||||||
|
pub proxy_src: String,
|
||||||
|
pub gif: Option<GifItem>,
|
||||||
|
}
|
||||||
@@ -13,11 +13,9 @@ entities = "1.0.1"
|
|||||||
fluxer-svc = { path = "../fluxer_svc", default-features = false }
|
fluxer-svc = { path = "../fluxer_svc", default-features = false }
|
||||||
hmac = "0.13.0"
|
hmac = "0.13.0"
|
||||||
infer = "0.19.0"
|
infer = "0.19.0"
|
||||||
mime_guess = "2.0.5"
|
|
||||||
moka = { version = "0.12.15", features = ["future", "sync"] }
|
moka = { version = "0.12.15", features = ["future", "sync"] }
|
||||||
regex = "1.12"
|
regex = "1.12"
|
||||||
reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls", "stream"] }
|
reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls", "stream"] }
|
||||||
rmp-serde = "1.3"
|
|
||||||
scraper = "0.27"
|
scraper = "0.27"
|
||||||
serde = { version = "1.0.228", features = ["derive", "rc"] }
|
serde = { version = "1.0.228", features = ["derive", "rc"] }
|
||||||
serde_json = "1.0.150"
|
serde_json = "1.0.150"
|
||||||
|
|||||||
@@ -81,6 +81,7 @@ async fn resolve_html(ctx: &ResolveContext<'_>) -> anyhow::Result<ResolverResult
|
|||||||
media_proxy: ctx.media_proxy,
|
media_proxy: ctx.media_proxy,
|
||||||
static_cdn_endpoint: ctx.static_cdn_endpoint,
|
static_cdn_endpoint: ctx.static_cdn_endpoint,
|
||||||
youtube_api_key: ctx.youtube_api_key.clone(),
|
youtube_api_key: ctx.youtube_api_key.clone(),
|
||||||
|
klipy_api_key: ctx.klipy_api_key.clone(),
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some(embeds) = activity_pub::try_resolve(
|
if let Some(embeds) = activity_pub::try_resolve(
|
||||||
|
|||||||
@@ -1075,6 +1075,7 @@ mod tests {
|
|||||||
media_proxy,
|
media_proxy,
|
||||||
static_cdn_endpoint: "https://static.example.test",
|
static_cdn_endpoint: "https://static.example.test",
|
||||||
youtube_api_key: None,
|
youtube_api_key: None,
|
||||||
|
klipy_api_key: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,12 @@ use std::pin::Pin;
|
|||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
const KLIPY_FLIGHT_CHUNK_MAX_BYTES: usize = 512 * 1024;
|
const KLIPY_API_V1_BASE_URL: &str = "https://api.klipy.com/api/v1";
|
||||||
|
const KLIPY_API_MAX_BYTES: usize = 512 * 1024;
|
||||||
|
const KLIPY_API_TIMEOUT: Duration = Duration::from_secs(10);
|
||||||
|
const KLIPY_SIZE_PREFERENCE: &[&str] = &["hd", "md", "sm", "xs"];
|
||||||
|
const KLIPY_THUMBNAIL_FORMATS: &[&str] = &["webp", "gif"];
|
||||||
|
const KLIPY_VIDEO_FORMATS: &[&str] = &["webm", "mp4"];
|
||||||
|
|
||||||
pub struct KlipyResolver;
|
pub struct KlipyResolver;
|
||||||
|
|
||||||
@@ -28,36 +33,13 @@ struct KlipyMediaFormats {
|
|||||||
|
|
||||||
impl Resolver for KlipyResolver {
|
impl Resolver for KlipyResolver {
|
||||||
fn matches(&self, url: &Url) -> bool {
|
fn matches(&self, url: &Url) -> bool {
|
||||||
url.host_str()
|
is_klipy_host(url)
|
||||||
.is_some_and(|h| h.eq_ignore_ascii_case("klipy.com"))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn transform_url(&self, url: &Url) -> Option<Url> {
|
fn transform_url(&self, url: &Url) -> Option<Url> {
|
||||||
if !url
|
let (kind, slug) = klipy_path(url)?;
|
||||||
.host_str()
|
let resource = klipy_resource(&kind);
|
||||||
.is_some_and(|h| h.eq_ignore_ascii_case("klipy.com"))
|
Url::parse(&format!("https://klipy.com/{resource}/{slug}/player")).ok()
|
||||||
{
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
|
|
||||||
let path = url.path();
|
|
||||||
static PATH_RE: std::sync::LazyLock<regex::Regex> = std::sync::LazyLock::new(|| {
|
|
||||||
regex::Regex::new(r"^/(gif|gifs|clip|clips)/([^/]+)").expect("valid regex")
|
|
||||||
});
|
|
||||||
let caps = PATH_RE.captures(path)?;
|
|
||||||
let kind = caps.get(1)?.as_str();
|
|
||||||
let slug = caps.get(2)?.as_str();
|
|
||||||
|
|
||||||
let normalized_kind = if kind.starts_with("clip") {
|
|
||||||
"clips"
|
|
||||||
} else {
|
|
||||||
"gifs"
|
|
||||||
};
|
|
||||||
|
|
||||||
Url::parse(&format!(
|
|
||||||
"https://klipy.com/{normalized_kind}/{slug}/player"
|
|
||||||
))
|
|
||||||
.ok()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn resolve<'a>(
|
fn resolve<'a>(
|
||||||
@@ -65,26 +47,21 @@ impl Resolver for KlipyResolver {
|
|||||||
ctx: &'a ResolveContext<'_>,
|
ctx: &'a ResolveContext<'_>,
|
||||||
) -> Pin<Box<dyn Future<Output = anyhow::Result<ResolverResult>> + Send + 'a>> {
|
) -> Pin<Box<dyn Future<Output = anyhow::Result<ResolverResult>> + Send + 'a>> {
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
let result = http_fetch::fetch_url(
|
let Some(api_key) = ctx.klipy_api_key.clone().or_else(klipy_api_key) else {
|
||||||
&ctx.http_client,
|
|
||||||
ctx.url.as_str(),
|
|
||||||
http_fetch::DEFAULT_HTML_MAX_BYTES,
|
|
||||||
Duration::from_secs(10),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
if result.status != 200 {
|
|
||||||
return Ok(ResolverResult { embeds: vec![] });
|
return Ok(ResolverResult { embeds: vec![] });
|
||||||
}
|
};
|
||||||
|
let formats = match resolve_media_via_api(ctx, &api_key).await {
|
||||||
let html = String::from_utf8_lossy(&result.bytes);
|
Ok(formats) => formats,
|
||||||
let formats = match extract_klipy_media(&html) {
|
Err(err) => {
|
||||||
Some(formats) => formats,
|
tracing::warn!(error = %err, "KLIPY API resolution failed");
|
||||||
None => {
|
None
|
||||||
return Ok(ResolverResult { embeds: vec![] });
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let Some(formats) = formats else {
|
||||||
|
return Ok(ResolverResult { embeds: vec![] });
|
||||||
|
};
|
||||||
|
|
||||||
let mut embed = MessageEmbed::new("gifv");
|
let mut embed = MessageEmbed::new("gifv");
|
||||||
embed.url = Some(ctx.original_url.to_string());
|
embed.url = Some(ctx.original_url.to_string());
|
||||||
embed.provider = Some(EmbedProvider {
|
embed.provider = Some(EmbedProvider {
|
||||||
@@ -108,6 +85,12 @@ impl Resolver for KlipyResolver {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn is_klipy_host(url: &Url) -> bool {
|
||||||
|
url.host_str().is_some_and(|h| {
|
||||||
|
h.eq_ignore_ascii_case("klipy.com") || h.eq_ignore_ascii_case("www.klipy.com")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async fn resolve_klipy_media(
|
async fn resolve_klipy_media(
|
||||||
ctx: &ResolveContext<'_>,
|
ctx: &ResolveContext<'_>,
|
||||||
format: &KlipyMediaFormat,
|
format: &KlipyMediaFormat,
|
||||||
@@ -130,6 +113,129 @@ async fn resolve_klipy_media(
|
|||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn klipy_path(url: &Url) -> Option<(String, String)> {
|
||||||
|
if !is_klipy_host(url) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
static PATH_RE: std::sync::LazyLock<regex::Regex> = std::sync::LazyLock::new(|| {
|
||||||
|
regex::Regex::new(r"^/(gif|gifs|clip|clips)/([^/]+)").expect("valid regex")
|
||||||
|
});
|
||||||
|
let caps = PATH_RE.captures(url.path())?;
|
||||||
|
Some((
|
||||||
|
caps.get(1)?.as_str().to_owned(),
|
||||||
|
caps.get(2)?.as_str().to_owned(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn klipy_resource(kind: &str) -> &'static str {
|
||||||
|
if kind.starts_with("clip") {
|
||||||
|
"clips"
|
||||||
|
} else {
|
||||||
|
"gifs"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn klipy_api_key() -> Option<String> {
|
||||||
|
std::env::var("FLUXER_KLIPY_API_KEY")
|
||||||
|
.ok()
|
||||||
|
.filter(|key| !key.is_empty())
|
||||||
|
.or_else(|| {
|
||||||
|
std::env::var("KLIPY_API_KEY")
|
||||||
|
.ok()
|
||||||
|
.filter(|key| !key.is_empty())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn resolve_media_via_api(
|
||||||
|
ctx: &ResolveContext<'_>,
|
||||||
|
api_key: &str,
|
||||||
|
) -> anyhow::Result<Option<KlipyMediaFormats>> {
|
||||||
|
let Some((kind, slug)) = klipy_path(&ctx.original_url) else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let resource = klipy_resource(&kind);
|
||||||
|
let url = klipy_direct_url(api_key, resource, &slug)?;
|
||||||
|
let response = http_fetch::fetch_url(
|
||||||
|
&ctx.http_client,
|
||||||
|
url.as_str(),
|
||||||
|
KLIPY_API_MAX_BYTES,
|
||||||
|
KLIPY_API_TIMEOUT,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
if response.status != 200 {
|
||||||
|
tracing::warn!(
|
||||||
|
status = response.status,
|
||||||
|
"KLIPY direct API lookup returned non-200 status"
|
||||||
|
);
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
let payload: serde_json::Value = serde_json::from_slice(&response.bytes)?;
|
||||||
|
Ok(payload.get("data").and_then(extract_klipy_api_media))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn klipy_direct_url(api_key: &str, resource: &str, slug: &str) -> anyhow::Result<Url> {
|
||||||
|
Ok(Url::parse(&format!(
|
||||||
|
"{KLIPY_API_V1_BASE_URL}/{api_key}/{resource}/{slug}"
|
||||||
|
))?)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn extract_klipy_api_media(item: &serde_json::Value) -> Option<KlipyMediaFormats> {
|
||||||
|
let file = item.get("file");
|
||||||
|
let thumbnail = file.and_then(|file| pick_klipy_file_format(file, KLIPY_THUMBNAIL_FORMATS));
|
||||||
|
let video = file
|
||||||
|
.and_then(|file| pick_klipy_file_format(file, KLIPY_VIDEO_FORMATS))
|
||||||
|
.or_else(|| extract_klipy_fallback_webm(item.pointer("/media_formats/webm")));
|
||||||
|
|
||||||
|
if thumbnail.is_none() && video.is_none() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(KlipyMediaFormats { thumbnail, video })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn pick_klipy_file_format(file: &serde_json::Value, formats: &[&str]) -> Option<KlipyMediaFormat> {
|
||||||
|
for size in KLIPY_SIZE_PREFERENCE {
|
||||||
|
for media_format in formats {
|
||||||
|
if let Some(media) =
|
||||||
|
extract_media_format(file.pointer(&format!("/{size}/{media_format}")))
|
||||||
|
{
|
||||||
|
return Some(media);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for media_format in formats {
|
||||||
|
if let Some(media) = extract_media_format(file.get(*media_format)) {
|
||||||
|
return Some(media);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn extract_klipy_fallback_webm(value: Option<&serde_json::Value>) -> Option<KlipyMediaFormat> {
|
||||||
|
let value = value?;
|
||||||
|
let url = value
|
||||||
|
.get("url")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.filter(|url| !url.is_empty())?;
|
||||||
|
let dims = value.get("dims")?.as_array()?;
|
||||||
|
let width = dims
|
||||||
|
.first()
|
||||||
|
.and_then(|value| value.as_i64())
|
||||||
|
.filter(|value| *value > 0)
|
||||||
|
.and_then(|value| u32::try_from(value).ok());
|
||||||
|
let height = dims
|
||||||
|
.get(1)
|
||||||
|
.and_then(|value| value.as_i64())
|
||||||
|
.filter(|value| *value > 0)
|
||||||
|
.and_then(|value| u32::try_from(value).ok());
|
||||||
|
Some(KlipyMediaFormat {
|
||||||
|
url: Some(url.to_owned()),
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn resolve_relative_url(base_url: &Url, media_url: &str) -> Option<String> {
|
fn resolve_relative_url(base_url: &Url, media_url: &str) -> Option<String> {
|
||||||
let url = base_url.join(media_url).ok()?;
|
let url = base_url.join(media_url).ok()?;
|
||||||
if matches!(url.scheme(), "http" | "https") {
|
if matches!(url.scheme(), "http" | "https") {
|
||||||
@@ -158,55 +264,21 @@ fn build_embed_media_payload(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn extract_klipy_media(html: &str) -> Option<KlipyMediaFormats> {
|
|
||||||
static FLIGHT_RE: std::sync::LazyLock<regex::Regex> = std::sync::LazyLock::new(|| {
|
|
||||||
regex::Regex::new(r#"(?s)self\.__next_f\.push\(\[1,"(.*?)"\]\)"#).expect("valid regex")
|
|
||||||
});
|
|
||||||
|
|
||||||
for cap in FLIGHT_RE.captures_iter(html) {
|
|
||||||
let encoded = cap.get(1)?.as_str();
|
|
||||||
if encoded.len() > KLIPY_FLIGHT_CHUNK_MAX_BYTES {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if let Some(media) = parse_next_flight_data(encoded) {
|
|
||||||
return Some(media);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
None
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parse_next_flight_data(encoded: &str) -> Option<KlipyMediaFormats> {
|
|
||||||
let unescaped = serde_json::from_str::<String>(&format!("\"{encoded}\"")).ok()?;
|
|
||||||
let colon_idx = unescaped.find(':')?;
|
|
||||||
let json_str = &unescaped[colon_idx + 1..];
|
|
||||||
if json_str.len() > KLIPY_FLIGHT_CHUNK_MAX_BYTES {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
|
|
||||||
let arr: Vec<serde_json::Value> = serde_json::from_str(json_str).ok()?;
|
|
||||||
|
|
||||||
for item in &arr {
|
|
||||||
if let Some(media) = item.get("media")
|
|
||||||
&& media.get("file").is_some()
|
|
||||||
{
|
|
||||||
return Some(KlipyMediaFormats {
|
|
||||||
thumbnail: extract_media_format(media.pointer("/file/hd/webp")),
|
|
||||||
video: extract_media_format(media.pointer("/file/hd/mp4")),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
None
|
|
||||||
}
|
|
||||||
|
|
||||||
fn extract_media_format(value: Option<&serde_json::Value>) -> Option<KlipyMediaFormat> {
|
fn extract_media_format(value: Option<&serde_json::Value>) -> Option<KlipyMediaFormat> {
|
||||||
let value = value?;
|
let value = value?;
|
||||||
|
if let Some(url) = value.as_str().filter(|url| !url.is_empty()) {
|
||||||
|
return Some(KlipyMediaFormat {
|
||||||
|
url: Some(url.to_owned()),
|
||||||
|
width: None,
|
||||||
|
height: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let url = value
|
||||||
|
.get("url")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.filter(|url| !url.is_empty())?;
|
||||||
Some(KlipyMediaFormat {
|
Some(KlipyMediaFormat {
|
||||||
url: value
|
url: Some(url.to_owned()),
|
||||||
.get("url")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.map(|url| url.to_owned()),
|
|
||||||
width: value
|
width: value
|
||||||
.get("width")
|
.get("width")
|
||||||
.and_then(|v| v.as_u64())
|
.and_then(|v| v.as_u64())
|
||||||
@@ -222,35 +294,6 @@ fn extract_media_format(value: Option<&serde_json::Value>) -> Option<KlipyMediaF
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn parse_next_flight_data_extracts_media_formats() {
|
|
||||||
let encoded = r#"0:[{\"media\":{\"file\":{\"hd\":{\"webp\":{\"url\":\"https://img.example/a.webp\",\"width\":320,\"height\":180},\"mp4\":{\"url\":\"https://img.example/a.mp4\",\"width\":640,\"height\":360}}}}}]"#;
|
|
||||||
assert_eq!(
|
|
||||||
parse_next_flight_data(encoded),
|
|
||||||
Some(KlipyMediaFormats {
|
|
||||||
thumbnail: Some(KlipyMediaFormat {
|
|
||||||
url: Some("https://img.example/a.webp".to_owned()),
|
|
||||||
width: Some(320),
|
|
||||||
height: Some(180),
|
|
||||||
}),
|
|
||||||
video: Some(KlipyMediaFormat {
|
|
||||||
url: Some("https://img.example/a.mp4".to_owned()),
|
|
||||||
width: Some(640),
|
|
||||||
height: Some(360),
|
|
||||||
}),
|
|
||||||
})
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn extract_klipy_media_skips_oversized_flight_chunks() {
|
|
||||||
let html = format!(
|
|
||||||
r#"self.__next_f.push([1,"{}"])"#,
|
|
||||||
"x".repeat(KLIPY_FLIGHT_CHUNK_MAX_BYTES + 1)
|
|
||||||
);
|
|
||||||
assert_eq!(extract_klipy_media(&html), None);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn resolve_relative_url_uses_original_url_like_ts() {
|
fn resolve_relative_url_uses_original_url_like_ts() {
|
||||||
let base = Url::parse("https://klipy.com/gifs/funny").unwrap();
|
let base = Url::parse("https://klipy.com/gifs/funny").unwrap();
|
||||||
@@ -332,27 +375,152 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn extract_klipy_media_from_realistic_html() {
|
fn klipy_path_extracts_kind_and_slug() {
|
||||||
let html = r#"
|
let (kind, slug) =
|
||||||
<script>self.__next_f.push([1,"0:[{\"media\":{\"file\":{\"hd\":{\"webp\":{\"url\":\"https://img.klipy.com/hd.webp\",\"width\":640,\"height\":360},\"mp4\":{\"url\":\"https://img.klipy.com/hd.mp4\",\"width\":1280,\"height\":720}}}}}]"])</script>
|
klipy_path(&Url::parse("https://www.klipy.com/gifs/funny-cat-123").unwrap()).unwrap();
|
||||||
"#;
|
assert_eq!(kind, "gifs");
|
||||||
let result = extract_klipy_media(html);
|
assert_eq!(slug, "funny-cat-123");
|
||||||
assert!(result.is_some());
|
assert!(klipy_path(&Url::parse("https://klipy.com/about").unwrap()).is_none());
|
||||||
let formats = result.unwrap();
|
assert!(klipy_path(&Url::parse("https://notklipy.com/gifs/x").unwrap()).is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn klipy_resource_maps_kind_to_api_segment() {
|
||||||
|
assert_eq!(klipy_resource("gif"), "gifs");
|
||||||
|
assert_eq!(klipy_resource("gifs"), "gifs");
|
||||||
|
assert_eq!(klipy_resource("clip"), "clips");
|
||||||
|
assert_eq!(klipy_resource("clips"), "clips");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pick_klipy_file_format_prefers_hd_and_format_order() {
|
||||||
|
let file = serde_json::json!({
|
||||||
|
"hd": {
|
||||||
|
"webp": {"url": "https://img.klipy.com/hd.webp", "width": 254, "height": 450},
|
||||||
|
"webm": {"url": "https://img.klipy.com/hd.webm", "width": 254, "height": 450},
|
||||||
|
"mp4": {"url": "https://img.klipy.com/hd.mp4", "width": 254, "height": 450}
|
||||||
|
},
|
||||||
|
"sm": {
|
||||||
|
"webp": {"url": "https://img.klipy.com/sm.webp", "width": 165, "height": 294}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let thumbnail = pick_klipy_file_format(&file, KLIPY_THUMBNAIL_FORMATS).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
formats.thumbnail.as_ref().unwrap().url.as_deref(),
|
thumbnail.url.as_deref(),
|
||||||
Some("https://img.klipy.com/hd.webp")
|
Some("https://img.klipy.com/hd.webp")
|
||||||
);
|
);
|
||||||
|
assert_eq!(thumbnail.width, Some(254));
|
||||||
|
assert_eq!(thumbnail.height, Some(450));
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
formats.video.as_ref().unwrap().url.as_deref(),
|
pick_klipy_file_format(&file, KLIPY_VIDEO_FORMATS)
|
||||||
Some("https://img.klipy.com/hd.mp4")
|
.unwrap()
|
||||||
|
.url
|
||||||
|
.as_deref(),
|
||||||
|
Some("https://img.klipy.com/hd.webm")
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn extract_klipy_media_returns_none_for_non_media_chunks() {
|
fn pick_klipy_file_format_handles_string_shape() {
|
||||||
let html = r#"<script>self.__next_f.push([1,"0:[{\"status\":\"ok\"}]"])</script>"#;
|
let file = serde_json::json!({
|
||||||
assert!(extract_klipy_media(html).is_none());
|
"mp4": "https://img.klipy.com/c.mp4",
|
||||||
|
"gif": "https://img.klipy.com/c.gif",
|
||||||
|
"webp": "https://img.klipy.com/c.webp"
|
||||||
|
});
|
||||||
|
let thumbnail = pick_klipy_file_format(&file, KLIPY_THUMBNAIL_FORMATS).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
thumbnail.url.as_deref(),
|
||||||
|
Some("https://img.klipy.com/c.webp")
|
||||||
|
);
|
||||||
|
assert_eq!(thumbnail.width, None);
|
||||||
|
assert_eq!(
|
||||||
|
pick_klipy_file_format(&file, KLIPY_VIDEO_FORMATS)
|
||||||
|
.unwrap()
|
||||||
|
.url
|
||||||
|
.as_deref(),
|
||||||
|
Some("https://img.klipy.com/c.mp4")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_klipy_api_media_uses_fallback_webm_shape() {
|
||||||
|
let item = serde_json::json!({
|
||||||
|
"media_formats": {
|
||||||
|
"webm": {
|
||||||
|
"url": "https://img.klipy.com/fallback.webm",
|
||||||
|
"dims": [320, 180]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let media = extract_klipy_api_media(&item).unwrap();
|
||||||
|
let video = media.video.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
video.url.as_deref(),
|
||||||
|
Some("https://img.klipy.com/fallback.webm")
|
||||||
|
);
|
||||||
|
assert_eq!(video.width, Some(320));
|
||||||
|
assert_eq!(video.height, Some(180));
|
||||||
|
assert!(media.thumbnail.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore = "hits the live KLIPY API and local media proxy"]
|
||||||
|
async fn live_klipy_embed_resolves_real_media() {
|
||||||
|
let api_key = std::env::var("FLUXER_KLIPY_API_KEY").expect("FLUXER_KLIPY_API_KEY set");
|
||||||
|
let media_proxy_endpoint =
|
||||||
|
std::env::var("FLUXER_MEDIA_PROXY_ENDPOINT").expect("FLUXER_MEDIA_PROXY_ENDPOINT set");
|
||||||
|
let media_proxy_secret = std::env::var("FLUXER_MEDIA_PROXY_SECRET_KEY")
|
||||||
|
.expect("FLUXER_MEDIA_PROXY_SECRET_KEY set");
|
||||||
|
let media_proxy_public_endpoint = std::env::var("FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT").ok();
|
||||||
|
let raw_url = std::env::var("FLUXER_KLIPY_LIVE_URL")
|
||||||
|
.unwrap_or_else(|_| "https://klipy.com/gifs/goatplaybanjo-chat-4".to_owned());
|
||||||
|
|
||||||
|
let resolver = KlipyResolver;
|
||||||
|
let original_url = Url::parse(&raw_url).expect("valid live KLIPY URL");
|
||||||
|
let url = resolver
|
||||||
|
.transform_url(&original_url)
|
||||||
|
.unwrap_or_else(|| original_url.clone());
|
||||||
|
let media_proxy = crate::media_proxy::MediaProxyClient::new_with_public_endpoint(
|
||||||
|
&media_proxy_endpoint,
|
||||||
|
&media_proxy_secret,
|
||||||
|
media_proxy_public_endpoint.as_deref(),
|
||||||
|
reqwest::Client::new(),
|
||||||
|
);
|
||||||
|
let ctx = ResolveContext {
|
||||||
|
url,
|
||||||
|
original_url: original_url.clone(),
|
||||||
|
http_client: reqwest::Client::new(),
|
||||||
|
nsfw_mode: crate::types::NsfwMode::Allow,
|
||||||
|
media_proxy: &media_proxy,
|
||||||
|
static_cdn_endpoint: "",
|
||||||
|
youtube_api_key: None,
|
||||||
|
klipy_api_key: Some(api_key),
|
||||||
|
};
|
||||||
|
|
||||||
|
let result = resolver.resolve(&ctx).await.expect("resolve KLIPY embed");
|
||||||
|
assert_eq!(result.embeds.len(), 1);
|
||||||
|
let embed = &result.embeds[0];
|
||||||
|
assert_eq!(embed.embed_type, "gifv");
|
||||||
|
assert_eq!(embed.url.as_deref(), Some(original_url.as_str()));
|
||||||
|
assert_eq!(
|
||||||
|
embed
|
||||||
|
.provider
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|provider| provider.name.as_deref()),
|
||||||
|
Some("KLIPY")
|
||||||
|
);
|
||||||
|
let video = embed.video.as_ref().expect("video media resolved");
|
||||||
|
assert!(
|
||||||
|
video
|
||||||
|
.url
|
||||||
|
.as_deref()
|
||||||
|
.is_some_and(|url| url.starts_with("https://"))
|
||||||
|
);
|
||||||
|
assert!(video.width.is_some_and(|width| width > 0));
|
||||||
|
assert!(video.height.is_some_and(|height| height > 0));
|
||||||
|
assert!(video.content_type.as_deref().is_some_and(|content_type| {
|
||||||
|
content_type.starts_with("video/") || content_type == "image/gif"
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ pub struct ResolveContext<'mp> {
|
|||||||
pub media_proxy: &'mp MediaProxyClient,
|
pub media_proxy: &'mp MediaProxyClient,
|
||||||
pub static_cdn_endpoint: &'mp str,
|
pub static_cdn_endpoint: &'mp str,
|
||||||
pub youtube_api_key: Option<String>,
|
pub youtube_api_key: Option<String>,
|
||||||
|
pub klipy_api_key: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ResolveContext<'_> {
|
impl ResolveContext<'_> {
|
||||||
|
|||||||
@@ -125,6 +125,7 @@ mod tests {
|
|||||||
bypass_cache: false,
|
bypass_cache: false,
|
||||||
cache_only,
|
cache_only,
|
||||||
youtube_api_key: None,
|
youtube_api_key: None,
|
||||||
|
klipy_api_key: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -91,6 +91,7 @@ impl UnfurlShard {
|
|||||||
url_str: &str,
|
url_str: &str,
|
||||||
nsfw_mode: NsfwMode,
|
nsfw_mode: NsfwMode,
|
||||||
youtube_api_key: Option<&str>,
|
youtube_api_key: Option<&str>,
|
||||||
|
klipy_api_key: Option<&str>,
|
||||||
) -> anyhow::Result<UnfurlResult> {
|
) -> anyhow::Result<UnfurlResult> {
|
||||||
let parsed = Url::parse(url_str)?;
|
let parsed = Url::parse(url_str)?;
|
||||||
|
|
||||||
@@ -104,6 +105,7 @@ impl UnfurlShard {
|
|||||||
media_proxy: &self.media_proxy,
|
media_proxy: &self.media_proxy,
|
||||||
static_cdn_endpoint: &self.static_cdn_endpoint,
|
static_cdn_endpoint: &self.static_cdn_endpoint,
|
||||||
youtube_api_key: youtube_api_key.map(str::to_owned),
|
youtube_api_key: youtube_api_key.map(str::to_owned),
|
||||||
|
klipy_api_key: klipy_api_key.map(str::to_owned),
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some(idx) = matched_resolver_idx {
|
if let Some(idx) = matched_resolver_idx {
|
||||||
@@ -204,6 +206,7 @@ impl ShardService for UnfurlShard {
|
|||||||
bypass_cache,
|
bypass_cache,
|
||||||
cache_only,
|
cache_only,
|
||||||
ref youtube_api_key,
|
ref youtube_api_key,
|
||||||
|
ref klipy_api_key,
|
||||||
} => {
|
} => {
|
||||||
let nsfw = nsfw_mode.unwrap_or_default();
|
let nsfw = nsfw_mode.unwrap_or_default();
|
||||||
let cache_key = unfurl_cache_key(url, nsfw);
|
let cache_key = unfurl_cache_key(url, nsfw);
|
||||||
@@ -219,7 +222,12 @@ impl ShardService for UnfurlShard {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let result = match self
|
let result = match self
|
||||||
.resolve_url(url, nsfw, youtube_api_key.as_deref())
|
.resolve_url(
|
||||||
|
url,
|
||||||
|
nsfw,
|
||||||
|
youtube_api_key.as_deref(),
|
||||||
|
klipy_api_key.as_deref(),
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(r) => Arc::new(r),
|
Ok(r) => Arc::new(r),
|
||||||
|
|||||||
@@ -15,6 +15,8 @@ pub enum UnfurlRequest {
|
|||||||
cache_only: bool,
|
cache_only: bool,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
youtube_api_key: Option<String>,
|
youtube_api_key: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
klipy_api_key: Option<String>,
|
||||||
},
|
},
|
||||||
Invalidate {
|
Invalidate {
|
||||||
url: String,
|
url: String,
|
||||||
@@ -203,12 +205,14 @@ mod tests {
|
|||||||
bypass_cache,
|
bypass_cache,
|
||||||
cache_only,
|
cache_only,
|
||||||
youtube_api_key,
|
youtube_api_key,
|
||||||
|
klipy_api_key,
|
||||||
} => {
|
} => {
|
||||||
assert_eq!(url, "https://fxtwitter.com/example/status/1");
|
assert_eq!(url, "https://fxtwitter.com/example/status/1");
|
||||||
assert_eq!(nsfw_mode, Some(NsfwMode::Block));
|
assert_eq!(nsfw_mode, Some(NsfwMode::Block));
|
||||||
assert!(!bypass_cache);
|
assert!(!bypass_cache);
|
||||||
assert!(!cache_only);
|
assert!(!cache_only);
|
||||||
assert!(youtube_api_key.is_none());
|
assert!(youtube_api_key.is_none());
|
||||||
|
assert!(klipy_api_key.is_none());
|
||||||
}
|
}
|
||||||
UnfurlRequest::Invalidate { .. } => panic!("expected unfurl request"),
|
UnfurlRequest::Invalidate { .. } => panic!("expected unfurl request"),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -219,15 +219,9 @@ function defaultConfig(): MasterConfig {
|
|||||||
port: 3310,
|
port: 3310,
|
||||||
fail_open: false,
|
fail_open: false,
|
||||||
},
|
},
|
||||||
gif: {
|
|
||||||
provider: 'tenor',
|
|
||||||
},
|
|
||||||
klipy: {
|
klipy: {
|
||||||
api_key: '',
|
api_key: '',
|
||||||
},
|
},
|
||||||
tenor: {
|
|
||||||
api_key: '',
|
|
||||||
},
|
|
||||||
youtube: {
|
youtube: {
|
||||||
api_key: '',
|
api_key: '',
|
||||||
},
|
},
|
||||||
@@ -399,7 +393,6 @@ function normalizeConfig(config: MasterConfig): MasterConfig {
|
|||||||
assertOneOf(config.integrations.email.provider, ['smtp', 'none'], 'FLUXER_EMAIL_PROVIDER');
|
assertOneOf(config.integrations.email.provider, ['smtp', 'none'], 'FLUXER_EMAIL_PROVIDER');
|
||||||
assertOneOf(config.integrations.captcha.provider, ['hcaptcha', 'turnstile', 'none'], 'FLUXER_CAPTCHA_PROVIDER');
|
assertOneOf(config.integrations.captcha.provider, ['hcaptcha', 'turnstile', 'none'], 'FLUXER_CAPTCHA_PROVIDER');
|
||||||
assertOneOf(config.integrations.search.engine, ['elasticsearch', 'meilisearch'], 'FLUXER_SEARCH_ENGINE');
|
assertOneOf(config.integrations.search.engine, ['elasticsearch', 'meilisearch'], 'FLUXER_SEARCH_ENGINE');
|
||||||
assertOneOf(config.integrations.gif.provider, ['tenor', 'klipy'], 'FLUXER_GIF_PROVIDER');
|
|
||||||
assertOneOf(
|
assertOneOf(
|
||||||
config.instance.abuse_policy.direct_contact_spam.action,
|
config.instance.abuse_policy.direct_contact_spam.action,
|
||||||
['flag_spammer', 'suppress_delivery'],
|
['flag_spammer', 'suppress_delivery'],
|
||||||
|
|||||||
@@ -259,15 +259,9 @@ export interface MasterConfig {
|
|||||||
port: number;
|
port: number;
|
||||||
fail_open: boolean;
|
fail_open: boolean;
|
||||||
};
|
};
|
||||||
gif: {
|
|
||||||
provider: 'tenor' | 'klipy';
|
|
||||||
};
|
|
||||||
klipy: {
|
klipy: {
|
||||||
api_key: string;
|
api_key: string;
|
||||||
};
|
};
|
||||||
tenor: {
|
|
||||||
api_key: string;
|
|
||||||
};
|
|
||||||
youtube: {
|
youtube: {
|
||||||
api_key: string;
|
api_key: string;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -278,9 +278,7 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
|
|||||||
FLUXER_CLAMAV_HOST: {path: ['integrations', 'clamav', 'host']},
|
FLUXER_CLAMAV_HOST: {path: ['integrations', 'clamav', 'host']},
|
||||||
FLUXER_CLAMAV_PORT: {path: ['integrations', 'clamav', 'port'], parse: parseEnvValue},
|
FLUXER_CLAMAV_PORT: {path: ['integrations', 'clamav', 'port'], parse: parseEnvValue},
|
||||||
FLUXER_CLAMAV_FAIL_OPEN: {path: ['integrations', 'clamav', 'fail_open'], parse: parseEnvValue},
|
FLUXER_CLAMAV_FAIL_OPEN: {path: ['integrations', 'clamav', 'fail_open'], parse: parseEnvValue},
|
||||||
FLUXER_GIF_PROVIDER: {path: ['integrations', 'gif', 'provider']},
|
|
||||||
FLUXER_KLIPY_API_KEY: {path: ['integrations', 'klipy', 'api_key']},
|
FLUXER_KLIPY_API_KEY: {path: ['integrations', 'klipy', 'api_key']},
|
||||||
FLUXER_TENOR_API_KEY: {path: ['integrations', 'tenor', 'api_key']},
|
|
||||||
FLUXER_YOUTUBE_API_KEY: {path: ['integrations', 'youtube', 'api_key']},
|
FLUXER_YOUTUBE_API_KEY: {path: ['integrations', 'youtube', 'api_key']},
|
||||||
FLUXER_BUNNY_PURGE_ENABLED: {path: ['integrations', 'bunny', 'purge_enabled'], parse: parseEnvValue},
|
FLUXER_BUNNY_PURGE_ENABLED: {path: ['integrations', 'bunny', 'purge_enabled'], parse: parseEnvValue},
|
||||||
FLUXER_BUNNY_API_KEY: {path: ['integrations', 'bunny', 'api_key']},
|
FLUXER_BUNNY_API_KEY: {path: ['integrations', 'bunny', 'api_key']},
|
||||||
|
|||||||
@@ -21,13 +21,11 @@ describe('extractClientIp', () => {
|
|||||||
it('prioritises configured header', () => {
|
it('prioritises configured header', () => {
|
||||||
const request = new Request('http://example.com', {
|
const request = new Request('http://example.com', {
|
||||||
headers: {
|
headers: {
|
||||||
'Cf-Connecting-Ip': '203.0.113.40',
|
'X-Real-Ip': '203.0.113.40',
|
||||||
'X-Forwarded-For': '203.0.113.60',
|
'X-Forwarded-For': '203.0.113.60',
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
expect(extractClientIp(request, {trustClientIpHeader: true, clientIpHeaderName: 'cf-connecting-ip'})).toBe(
|
expect(extractClientIp(request, {trustClientIpHeader: true, clientIpHeaderName: 'x-real-ip'})).toBe('203.0.113.40');
|
||||||
'203.0.113.40',
|
|
||||||
);
|
|
||||||
});
|
});
|
||||||
it('uses default header name (x-forwarded-for) when clientIpHeaderName is not specified', () => {
|
it('uses default header name (x-forwarded-for) when clientIpHeaderName is not specified', () => {
|
||||||
const request = new Request('http://example.com', {
|
const request = new Request('http://example.com', {
|
||||||
@@ -39,7 +37,7 @@ describe('extractClientIp', () => {
|
|||||||
const request = new Request('http://example.com', {
|
const request = new Request('http://example.com', {
|
||||||
headers: {'X-Forwarded-For': '192.168.1.3'},
|
headers: {'X-Forwarded-For': '192.168.1.3'},
|
||||||
});
|
});
|
||||||
expect(extractClientIp(request, {trustClientIpHeader: true, clientIpHeaderName: 'cf-connecting-ip'})).toBeNull();
|
expect(extractClientIp(request, {trustClientIpHeader: true, clientIpHeaderName: 'x-real-ip'})).toBeNull();
|
||||||
});
|
});
|
||||||
it('extracts first hop from x-forwarded-for', () => {
|
it('extracts first hop from x-forwarded-for', () => {
|
||||||
const request = new Request('http://example.com', {
|
const request = new Request('http://example.com', {
|
||||||
@@ -108,7 +106,7 @@ describe('resolveClientIpHeaderName', () => {
|
|||||||
expect(resolveClientIpHeaderName()).toBe('x-forwarded-for');
|
expect(resolveClientIpHeaderName()).toBe('x-forwarded-for');
|
||||||
});
|
});
|
||||||
it('returns the configured header name normalised to lowercase', () => {
|
it('returns the configured header name normalised to lowercase', () => {
|
||||||
expect(resolveClientIpHeaderName('cf-connecting-ip')).toBe('cf-connecting-ip');
|
expect(resolveClientIpHeaderName('x-client-ip')).toBe('x-client-ip');
|
||||||
expect(resolveClientIpHeaderName('X-Real-Ip')).toBe('x-real-ip');
|
expect(resolveClientIpHeaderName('X-Real-Ip')).toBe('x-real-ip');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -526,7 +526,6 @@ const InstancePolicyResponse = z.object({
|
|||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
|
|
||||||
const GifProviderSchema = z.enum(['tenor', 'klipy']);
|
|
||||||
const CaptchaProviderSchema = z.enum(['hcaptcha', 'turnstile', 'none']);
|
const CaptchaProviderSchema = z.enum(['hcaptcha', 'turnstile', 'none']);
|
||||||
const EmailProviderSchema = z.enum(['smtp', 'none']);
|
const EmailProviderSchema = z.enum(['smtp', 'none']);
|
||||||
|
|
||||||
@@ -559,9 +558,6 @@ const InstanceMediaResponse = z.object({
|
|||||||
|
|
||||||
const InstanceIntegrationsResponse = z.object({
|
const InstanceIntegrationsResponse = z.object({
|
||||||
gif: z.object({
|
gif: z.object({
|
||||||
provider: GifProviderSchema.nullable(),
|
|
||||||
effective_provider: GifProviderSchema,
|
|
||||||
tenor_api_key_set: z.boolean(),
|
|
||||||
klipy_api_key_set: z.boolean(),
|
klipy_api_key_set: z.boolean(),
|
||||||
effective_available: z.boolean(),
|
effective_available: z.boolean(),
|
||||||
}),
|
}),
|
||||||
@@ -648,8 +644,6 @@ export const InstanceConfigUpdateRequest = z.object({
|
|||||||
.object({
|
.object({
|
||||||
gif: z
|
gif: z
|
||||||
.object({
|
.object({
|
||||||
provider: GifProviderSchema.nullish(),
|
|
||||||
tenor_api_key: z.string().trim().max(4096).nullish(),
|
|
||||||
klipy_api_key: z.string().trim().max(4096).nullish(),
|
klipy_api_key: z.string().trim().max(4096).nullish(),
|
||||||
})
|
})
|
||||||
.nullish(),
|
.nullish(),
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ export const GIF_PROVIDER_DISPLAY_NAME_HEADER = 'X-Fluxer-GIF-Provider-Display-N
|
|||||||
export const GIF_PROVIDER_ATTRIBUTION_HEADER = 'X-Fluxer-GIF-Provider-Attribution-Required';
|
export const GIF_PROVIDER_ATTRIBUTION_HEADER = 'X-Fluxer-GIF-Provider-Attribution-Required';
|
||||||
const LocaleType = LocaleSchema.default('en-US').transform((v) => v.replace('-', '_'));
|
const LocaleType = LocaleSchema.default('en-US').transform((v) => v.replace('-', '_'));
|
||||||
const GifProviderName = createStringType(1, 32).describe(
|
const GifProviderName = createStringType(1, 32).describe(
|
||||||
'Identifier of the active GIF provider (e.g. "klipy", "tenor"). Vendor names are opaque to clients.',
|
'Identifier of the active GIF provider. KLIPY is currently the only supported provider.',
|
||||||
);
|
);
|
||||||
|
|
||||||
export const GifSearchQuery = z.object({
|
export const GifSearchQuery = z.object({
|
||||||
@@ -69,10 +69,10 @@ export type GifResponse = z.infer<typeof GifResponse>;
|
|||||||
|
|
||||||
export const GifCategoryTagResponse = z.object({
|
export const GifCategoryTagResponse = z.object({
|
||||||
name: z.string().describe('Category search term (locale-translated label suitable for display).'),
|
name: z.string().describe('Category search term (locale-translated label suitable for display).'),
|
||||||
src: z.string().describe('URL to the category preview image (legacy / fallback).'),
|
src: z.string().describe('Category preview image URL from the top GIF for this category search term.'),
|
||||||
proxy_src: z.string().describe('Proxied URL to the category preview image (legacy / fallback).'),
|
proxy_src: z.string().describe('Proxied category preview image URL from the top GIF for this category search term.'),
|
||||||
gif: GifResponse.nullable().describe(
|
gif: GifResponse.nullable().describe(
|
||||||
'Full enriched GIF for the category (first search result for `name`). Null until the background cache is populated for this locale; fall back to `src` / `proxy_src` in that case.',
|
'Enriched category preview GIF from the top search result for this category. Null only when no preview GIF was available.',
|
||||||
),
|
),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -91,7 +91,7 @@ export const WellKnownFluxerResponse = z.object({
|
|||||||
.describe('Feature flags for this instance'),
|
.describe('Feature flags for this instance'),
|
||||||
gif: z
|
gif: z
|
||||||
.object({
|
.object({
|
||||||
provider: z.string().describe('Stable machine name of the active GIF provider (e.g. "klipy", "tenor")'),
|
provider: z.string().describe('Stable machine name of the active GIF provider.'),
|
||||||
display_name: z.string().describe('Human-readable provider name shown in the UI'),
|
display_name: z.string().describe('Human-readable provider name shown in the UI'),
|
||||||
attribution_required: z
|
attribution_required: z
|
||||||
.boolean()
|
.boolean()
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ export const CreateFavoriteMemeFromUrlBodySchema = FavoriteMemeBase.extend({
|
|||||||
.describe('Provider-issued slug or slug-id token for the GIF, when sourced from a provider'),
|
.describe('Provider-issued slug or slug-id token for the GIF, when sourced from a provider'),
|
||||||
gif_provider: createStringType(1, 32)
|
gif_provider: createStringType(1, 32)
|
||||||
.nullish()
|
.nullish()
|
||||||
.describe('Stable name of the GIF provider that issued gif_slug (e.g. "klipy", "tenor")'),
|
.describe('Stable name of the GIF provider that issued gif_slug. New provider GIFs are sourced from KLIPY.'),
|
||||||
media: z
|
media: z
|
||||||
.record(z.string(), GifMediaFormat)
|
.record(z.string(), GifMediaFormat)
|
||||||
.nullish()
|
.nullish()
|
||||||
@@ -77,7 +77,9 @@ export const FavoriteMemeResponse = z.object({
|
|||||||
gif_provider: z
|
gif_provider: z
|
||||||
.string()
|
.string()
|
||||||
.nullish()
|
.nullish()
|
||||||
.describe('Stable name of the GIF provider that issued gif_slug (e.g. "klipy", "tenor"), if any'),
|
.describe(
|
||||||
|
'Stable name of the GIF provider that issued gif_slug, if any. Legacy records may contain older provider names.',
|
||||||
|
),
|
||||||
media: z
|
media: z
|
||||||
.record(z.string(), GifMediaFormat)
|
.record(z.string(), GifMediaFormat)
|
||||||
.nullish()
|
.nullish()
|
||||||
|
|||||||
@@ -189,6 +189,12 @@ pub struct RustServiceSpec {
|
|||||||
|
|
||||||
pub fn rust_services() -> Vec<RustServiceSpec> {
|
pub fn rust_services() -> Vec<RustServiceSpec> {
|
||||||
vec![
|
vec![
|
||||||
|
RustServiceSpec {
|
||||||
|
name: "gifs",
|
||||||
|
package: "fluxer-gifs",
|
||||||
|
path: ROOT.join("fluxer_gifs"),
|
||||||
|
port_base: 8110,
|
||||||
|
},
|
||||||
RustServiceSpec {
|
RustServiceSpec {
|
||||||
name: "messages",
|
name: "messages",
|
||||||
package: "fluxer-messages",
|
package: "fluxer-messages",
|
||||||
|
|||||||
Reference in New Issue
Block a user