feat(media): emit external proxy urls with a readable path and extension (#1793)

fluxer_api/pkgs/media_proxy_utils/package.json

@@ -29,12 +29,16 @@
 	"main": "./src/MediaProxyUtils.ts",
 	"types": "./src/MediaProxyUtils.ts",
 	"scripts": {
+		"test": "vitest run",
+		"test:watch": "vitest",
 		"typecheck": "tsgo --noEmit"
 	},
 	"dependencies": {
 		"@types/node": "catalog:"
 	},
 	"devDependencies": {
-		"@typescript/native-preview": "catalog:"
+		"@typescript/native-preview": "catalog:",
+		"vitest": "catalog:",
+		"vite-tsconfig-paths": "catalog:"
 	}
 }

fluxer_api/pkgs/media_proxy_utils/src/ExternalMediaProxyPathCodec.test.ts

@@ -0,0 +1,106 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {
+	buildExternalMediaProxyPath,
+	buildV2ExternalMediaProxyPath,
+	reconstructOriginalUrl,
+} from '@pkgs/media_proxy_utils/src/ExternalMediaProxyPathCodec';
+import {describe, expect, it} from 'vitest';
+
+const ROUND_TRIP_URLS = [
+	'https://static.klipy.com/ii/c8/28/HkAKKCzZ.webp',
+	'https://static.klipy.com/ii/HkAKKCzZ.webp?v=query_param&goes=here',
+	'https://example.com:8443/a.png',
+	'https://avatars.githubusercontent.com/u/241303489?v=4',
+	'http://example.com/plain.gif',
+	'https://example.com/deep/nested/path/to/file.jpeg',
+	'https://example.com/file.png?a=1&b=2&c=3',
+	'https://example.com/spaced%20name.png',
+	'https://example.com/unicode/%C3%A5%C3%A4%C3%B6.png',
+	'https://example.com/file.png?redirect=https%3A%2F%2Fother.example%2Fx.png',
+	'https://sub.domain.example.co.uk/a/b.webp',
+];
+
+describe('buildExternalMediaProxyPath', () => {
+	it('emits the plain path shape with the extension last', () => {
+		expect(buildExternalMediaProxyPath('https://static.klipy.com/ii/c8/28/HkAKKCzZ.webp')).toBe(
+			'https/static.klipy.com/ii/c8/28/HkAKKCzZ.webp',
+		);
+	});
+
+	it('puts an encoded query, leading question mark included, ahead of the protocol', () => {
+		expect(buildExternalMediaProxyPath('https://static.klipy.com/ii/HkAKKCzZ.webp?v=query_param&goes=here')).toBe(
+			'%3Fv%3Dquery_param%26goes%3Dhere/https/static.klipy.com/ii/HkAKKCzZ.webp',
+		);
+	});
+
+	it('keeps a non default port on the host segment', () => {
+		expect(buildExternalMediaProxyPath('https://example.com:8443/a.png')).toBe('https/example.com:8443/a.png');
+	});
+
+	it('preserves the http scheme', () => {
+		expect(buildExternalMediaProxyPath('http://example.com/a.gif')).toBe('http/example.com/a.gif');
+	});
+
+	it('handles a root url with no path', () => {
+		expect(buildExternalMediaProxyPath('https://example.com/')).toBe('https/example.com');
+	});
+
+	it('never emits the v2 prefix any more', () => {
+		for (const url of ROUND_TRIP_URLS) {
+			expect(buildExternalMediaProxyPath(url).startsWith('v2/')).toBe(false);
+		}
+	});
+
+	it('ends in the source file extension so extension based cdn caching applies', () => {
+		for (const [url, ext] of [
+			['https://example.com/a.webp', '.webp'],
+			['https://example.com/a.png?x=1', '.png'],
+			['https://example.com/a/b/c.jpeg', '.jpeg'],
+		] as const) {
+			expect(buildExternalMediaProxyPath(url).endsWith(ext)).toBe(true);
+		}
+	});
+
+	it('rejects a url it cannot parse', () => {
+		expect(() => buildExternalMediaProxyPath('not a url')).toThrow();
+	});
+});
+
+describe('reconstructOriginalUrl', () => {
+	it('round trips every supported shape', () => {
+		for (const url of ROUND_TRIP_URLS) {
+			expect(reconstructOriginalUrl(buildExternalMediaProxyPath(url))).toBe(url);
+		}
+	});
+
+	it('decodes an externally produced path verbatim', () => {
+		expect(
+			reconstructOriginalUrl('%3Fv%3Dquery_param%26goes%3Dhere/https/static.klipy.com/ii/c8/28/HkAKKCzZ.webp'),
+		).toBe('https://static.klipy.com/ii/c8/28/HkAKKCzZ.webp?v=query_param&goes=here');
+	});
+
+	it('does not double the question mark when the query segment carries one', () => {
+		const decoded = reconstructOriginalUrl('%3Fa%3D1/https/example.com/x.png');
+		expect(decoded).toBe('https://example.com/x.png?a=1');
+		expect(decoded).not.toContain('??');
+	});
+
+	it('still accepts a query segment without a leading question mark', () => {
+		expect(reconstructOriginalUrl('a%3D1/https/example.com/x.png')).toBe('https://example.com/x.png?a=1');
+	});
+
+	it('still decodes v2 paths so links already sent keep working', () => {
+		expect(reconstructOriginalUrl(buildV2ExternalMediaProxyPath('https://example.com/a.png?x=1'))).toBe(
+			'https://example.com/a.png?x=1',
+		);
+	});
+
+	it('rejects a path that has no host after the protocol', () => {
+		expect(() => reconstructOriginalUrl('https')).toThrow();
+	});
+
+	it('rejects an empty v2 payload', () => {
+		expect(() => reconstructOriginalUrl('v2/')).toThrow();
+	});
+});

fluxer_api/pkgs/media_proxy_utils/src/ExternalMediaProxyPathCodec.ts

@@ -74,7 +74,8 @@ function reconstructLegacyOriginalUrl(proxyUrlPath: string): string {
 	const query = encodedQuery ? decodeLegacyComponent(encodedQuery) : '';
 	const path = decodeLegacyComponent(encodedPath);
 	const {hostname, port} = decodeLegacyHostAndPort(hostPart);
-	return `${protocol}://${hostname}${port ? `:${port}` : ''}/${path}${query ? `?${query}` : ''}`;
+	const normalizedQuery = query.startsWith('?') ? query.slice(1) : query;
+	return `${protocol}://${hostname}${port ? `:${port}` : ''}/${path}${normalizedQuery ? `?${normalizedQuery}` : ''}`;
 }
 
 function reconstructV2OriginalUrl(proxyUrlPath: string): string {
@@ -85,11 +86,28 @@ function reconstructV2OriginalUrl(proxyUrlPath: string): string {
 	return decodeV2PathComponent(encodedOriginalUrl);
 }
 
-export function buildExternalMediaProxyPath(inputUrl: string): string {
+export function buildV2ExternalMediaProxyPath(inputUrl: string): string {
 	const parsedUrl = new URL(inputUrl);
 	return `${V2_PATH_PREFIX}${encodeV2PathComponent(parsedUrl.toString())}`;
 }
 
+export function buildExternalMediaProxyPath(inputUrl: string): string {
+	const parsedUrl = new URL(inputUrl);
+	const protocol = parsedUrl.protocol.replace(/:$/u, '');
+	const host = parsedUrl.port ? `${parsedUrl.hostname}:${parsedUrl.port}` : parsedUrl.hostname;
+	const path = parsedUrl.pathname
+		.replace(/^\//u, '')
+		.split('/')
+		.map((segment) => encodeURIComponent(segment))
+		.join('/');
+	const segments = parsedUrl.search ? [encodeURIComponent(parsedUrl.search)] : [];
+	segments.push(protocol, host);
+	if (path) {
+		segments.push(path);
+	}
+	return segments.join('/');
+}
+
 export function reconstructOriginalUrl(proxyUrlPath: string): string {
 	const reconstructedUrl = proxyUrlPath.startsWith(V2_PATH_PREFIX)
 		? reconstructV2OriginalUrl(proxyUrlPath)

fluxer_api/pkgs/media_proxy_utils/tsconfig.json

@@ -0,0 +1,10 @@
+{
+	"extends": "../../../tsconfigs/package.json",
+	"compilerOptions": {
+		"paths": {
+			"@fluxer/*": ["../../../packages/*", "../../../packages/*/src/index.ts"],
+			"@pkgs/*": ["../*"]
+		}
+	},
+	"include": ["src/**/*"]
+}

fluxer_api/pkgs/media_proxy_utils/vitest.config.ts

@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+import tsconfigPaths from 'vite-tsconfig-paths';
+import {defineConfig} from 'vitest/config';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+export default defineConfig({
+	plugins: [
+		tsconfigPaths({
+			root: path.resolve(__dirname, '../..'),
+		}),
+	],
+	test: {
+		globals: true,
+		environment: 'node',
+		include: ['**/*.{test,spec}.{ts,tsx}'],
+		exclude: ['node_modules', 'dist'],
+		coverage: {
+			provider: 'v8',
+			reporter: ['text', 'json', 'html'],
+			exclude: ['**/*.test.tsx', '**/*.spec.tsx', 'node_modules/'],
+		},
+	},
+});

fluxer_media_proxy/src/external_path.rs

@@ -15,13 +15,70 @@ pub enum ExternalPathError {
     InvalidUtf8,
 }
 
-pub fn build_external_media_proxy_path(input_url: &str) -> String {
+pub fn build_v2_external_media_proxy_path(input_url: &str) -> String {
     format!(
         "{V2_PREFIX}{}",
         BASE64_URL_SAFE_NO_PAD.encode(input_url.as_bytes())
     )
 }
 
+fn percent_encode_component(value: &str) -> String {
+    let mut out = String::with_capacity(value.len());
+    for byte in value.bytes() {
+        match byte {
+            b'A'..=b'Z'
+            | b'a'..=b'z'
+            | b'0'..=b'9'
+            | b'-'
+            | b'_'
+            | b'.'
+            | b'!'
+            | b'~'
+            | b'*'
+            | b'\''
+            | b'('
+            | b')' => out.push(byte as char),
+            _ => out.push_str(&format!("%{byte:02X}")),
+        }
+    }
+    out
+}
+
+pub fn build_external_media_proxy_path(input_url: &str) -> Result<String, ExternalPathError> {
+    let (scheme, remainder) = input_url
+        .split_once("://")
+        .ok_or(ExternalPathError::InvalidExternalPath)?;
+    if scheme.is_empty() || remainder.is_empty() {
+        return Err(ExternalPathError::InvalidExternalPath);
+    }
+    let (authority_and_path, query) = match remainder.split_once('?') {
+        Some((head, tail)) => (head, Some(tail)),
+        None => (remainder, None),
+    };
+    let (host, path) = match authority_and_path.split_once('/') {
+        Some((host, path)) => (host, path),
+        None => (authority_and_path, ""),
+    };
+    if host.is_empty() {
+        return Err(ExternalPathError::InvalidExternalPath);
+    }
+    let mut segments: Vec<String> = Vec::new();
+    if let Some(query) = query {
+        segments.push(percent_encode_component(&format!("?{query}")));
+    }
+    segments.push(scheme.to_owned());
+    segments.push(host.to_owned());
+    if !path.is_empty() {
+        segments.push(
+            path.split('/')
+                .map(percent_encode_component)
+                .collect::<Vec<_>>()
+                .join("/"),
+        );
+    }
+    Ok(segments.join("/"))
+}
+
 fn decode_v2(proxy_path: &str) -> Result<String, ExternalPathError> {
     let encoded = proxy_path
         .strip_prefix(V2_PREFIX)
@@ -105,10 +162,10 @@ fn reconstruct_legacy(proxy_path: &str) -> Result<String, ExternalPathError> {
     let path_raw = parts[protocol_index + 2..].join("/");
     let query = percent_decode_string(&query_raw, false);
     let path = percent_decode_string(&path_raw, false);
+    let normalized_query = query.strip_prefix('?').unwrap_or(&query);
     Ok(format!(
-        "{protocol}://{host_port}/{path}{}{}",
+        "{protocol}://{host_port}/{path}{}{normalized_query}",
-        if query.is_empty() { "" } else { "?" },
+        if normalized_query.is_empty() { "" } else { "?" }
-        query
     ))
 }
 
@@ -124,9 +181,76 @@ pub fn reconstruct_original_url(proxy_path: &str) -> Result<String, ExternalPath
 mod tests {
     use super::*;
 
+    #[test]
+    fn builds_the_plain_path_shape() {
+        assert_eq!(
+            "https/static.klipy.com/ii/c8/28/HkAKKCzZ.webp",
+            build_external_media_proxy_path("https://static.klipy.com/ii/c8/28/HkAKKCzZ.webp")
+                .unwrap()
+        );
+    }
+
+    #[test]
+    fn builds_an_encoded_query_ahead_of_the_scheme() {
+        assert_eq!(
+            "%3Fv%3Dquery_param%26goes%3Dhere/https/static.klipy.com/ii/HkAKKCzZ.webp",
+            build_external_media_proxy_path(
+                "https://static.klipy.com/ii/HkAKKCzZ.webp?v=query_param&goes=here"
+            )
+            .unwrap()
+        );
+    }
+
+    #[test]
+    fn keeps_a_non_default_port() {
+        assert_eq!(
+            "https/example.com:8443/a.png",
+            build_external_media_proxy_path("https://example.com:8443/a.png").unwrap()
+        );
+    }
+
+    #[test]
+    fn round_trips_every_shape() {
+        for url in [
+            "https://static.klipy.com/ii/c8/28/HkAKKCzZ.webp",
+            "https://static.klipy.com/ii/HkAKKCzZ.webp?v=query_param&goes=here",
+            "https://example.com:8443/a.png",
+            "https://avatars.githubusercontent.com/u/241303489?v=4",
+            "http://example.com/plain.gif",
+            "https://example.com/file.png?a=1&b=2&c=3",
+        ] {
+            let path = build_external_media_proxy_path(url).unwrap();
+            assert_eq!(
+                url,
+                reconstruct_original_url(&path).unwrap(),
+                "round trip {url}"
+            );
+        }
+    }
+
+    #[test]
+    fn does_not_double_the_question_mark() {
+        let decoded = reconstruct_original_url("%3Fa%3D1/https/example.com/x.png").unwrap();
+        assert_eq!("https://example.com/x.png?a=1", decoded);
+        assert!(!decoded.contains("??"));
+    }
+
+    #[test]
+    fn still_accepts_a_query_without_a_leading_question_mark() {
+        assert_eq!(
+            "https://example.com/x.png?a=1",
+            reconstruct_original_url("a%3D1/https/example.com/x.png").unwrap()
+        );
+    }
+
+    #[test]
+    fn rejects_a_url_without_a_scheme() {
+        assert!(build_external_media_proxy_path("example.com/a.png").is_err());
+    }
+
     #[test]
     fn v2_path_roundtrip() {
-        let path = build_external_media_proxy_path("https://example.com/a b.png?x=1");
+        let path = build_v2_external_media_proxy_path("https://example.com/a b.png?x=1");
         let decoded = reconstruct_original_url(&path).unwrap();
         assert_eq!("https://example.com/a b.png?x=1", decoded);
     }

pnpm-lock.yaml

@@ -826,6 +826,12 @@ importers:
       '@typescript/native-preview':
         specifier: 'catalog:'
         version: 7.0.0-dev.20260224.1
+      vite-tsconfig-paths:
+        specifier: 'catalog:'
+        version: 6.1.1([email protected])([email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected]))
+      vitest:
+        specifier: 'catalog:'
+        version: 4.0.18(@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected](@types/[email protected])([email protected]))([email protected])([email protected])
 
   fluxer_api/pkgs/mime_utils:
     dependencies: