fix(media-proxy): stop proxy paths carrying fragments or credentials (#1835)

Cargo.lock

@@ -2053,6 +2053,7 @@ dependencies = [
  "thiserror",
  "time",
  "tracing",
+ "url",
  "urlencoding",
 ]
 

fluxer_api/pkgs/media_proxy_utils/src/ExternalMediaProxyPathCodec.test.ts

@@ -2,7 +2,7 @@
 
 import {
 	buildExternalMediaProxyPath,
-	buildV2ExternalMediaProxyPath,
+	buildOpaqueExternalMediaProxyPath,
 	reconstructOriginalUrl,
 } from '@pkgs/media_proxy_utils/src/ExternalMediaProxyPathCodec';
 import {describe, expect, it} from 'vitest';
@@ -91,7 +91,7 @@ describe('reconstructOriginalUrl', () => {
 	});
 
 	it('still decodes v2 paths so links already sent keep working', () => {
-		expect(reconstructOriginalUrl(buildV2ExternalMediaProxyPath('https://example.com/a.png?x=1'))).toBe(
+		expect(reconstructOriginalUrl(buildOpaqueExternalMediaProxyPath('https://example.com/a.png?x=1'))).toBe(
 			'https://example.com/a.png?x=1',
 		);
 	});

fluxer_api/pkgs/media_proxy_utils/src/ExternalMediaProxyPathCodec.ts

@@ -2,21 +2,21 @@
 
 const BASE64_URL_PADDING_REGEX = /=*$/;
 const LEGACY_PROTOCOL_REGEX = /^[A-Za-z][A-Za-z0-9+.-]*$/;
-const V2_PATH_PREFIX = 'v2/';
+const OPAQUE_PATH_PREFIX = 'v2/';
 
-function encodeV2PathComponent(value: string): string {
+function encodeOpaquePathComponent(value: string): string {
 	return Buffer.from(value, 'utf8').toString('base64url').replace(BASE64_URL_PADDING_REGEX, '');
 }
 
-function decodeV2PathComponent(value: string): string {
+function decodeOpaquePathComponent(value: string): string {
 	return Buffer.from(value, 'base64url').toString('utf8');
 }
 
-function decodeLegacyComponent(component: string): string {
+function decodeSegmentedComponent(component: string): string {
 	return decodeURIComponent(component);
 }
 
-function getLegacyProtocolIndex(parts: Array<string>): number {
+function getSegmentedProtocolIndex(parts: Array<string>): number {
 	const firstPart = parts[0];
 	if (firstPart && LEGACY_PROTOCOL_REGEX.test(firstPart)) {
 		return 0;
@@ -33,15 +33,15 @@ function getLegacyProtocolIndex(parts: Array<string>): number {
 	throw new Error('Protocol is missing in the proxy URL path.');
 }
 
-interface LegacyHostAndPort {
+interface SegmentedHostAndPort {
 	hostname: string;
 	port: string;
 }
 
-function decodeLegacyHostAndPort(hostPart: string): LegacyHostAndPort {
+function decodeSegmentedHostAndPort(hostPart: string): SegmentedHostAndPort {
 	const separatorIndex = hostPart.lastIndexOf(':');
 	if (separatorIndex === -1) {
-		const hostname = decodeLegacyComponent(hostPart);
+		const hostname = decodeSegmentedComponent(hostPart);
 		if (!hostname) {
 			throw new Error('Hostname is invalid in the proxy URL path.');
 		}
@@ -53,14 +53,14 @@ function decodeLegacyHostAndPort(hostPart: string): LegacyHostAndPort {
 		throw new Error('Hostname is invalid in the proxy URL path.');
 	}
 	return {
-		hostname: decodeLegacyComponent(encodedHostname),
-		port: encodedPort ? decodeLegacyComponent(encodedPort) : '',
+		hostname: decodeSegmentedComponent(encodedHostname),
+		port: encodedPort ? decodeSegmentedComponent(encodedPort) : '',
 	};
 }
 
-function reconstructLegacyOriginalUrl(proxyUrlPath: string): string {
+function reconstructSegmentedOriginalUrl(proxyUrlPath: string): string {
 	const parts = proxyUrlPath.split('/');
-	const protocolIndex = getLegacyProtocolIndex(parts);
+	const protocolIndex = getSegmentedProtocolIndex(parts);
 	const protocol = parts[protocolIndex];
 	if (!protocol) {
 		throw new Error('Protocol is missing in the proxy URL path.');
@@ -71,24 +71,24 @@ function reconstructLegacyOriginalUrl(proxyUrlPath: string): string {
 	}
 	const encodedQuery = parts.slice(0, protocolIndex).join('/');
 	const encodedPath = parts.slice(protocolIndex + 2).join('/');
-	const query = encodedQuery ? decodeLegacyComponent(encodedQuery) : '';
-	const path = decodeLegacyComponent(encodedPath);
-	const {hostname, port} = decodeLegacyHostAndPort(hostPart);
+	const query = encodedQuery ? decodeSegmentedComponent(encodedQuery) : '';
+	const path = decodeSegmentedComponent(encodedPath);
+	const {hostname, port} = decodeSegmentedHostAndPort(hostPart);
 	const normalizedQuery = query.startsWith('?') ? query.slice(1) : query;
 	return `${protocol}://${hostname}${port ? `:${port}` : ''}/${path}${normalizedQuery ? `?${normalizedQuery}` : ''}`;
 }
 
-function reconstructV2OriginalUrl(proxyUrlPath: string): string {
-	const encodedOriginalUrl = proxyUrlPath.slice(V2_PATH_PREFIX.length);
+function reconstructOpaqueOriginalUrl(proxyUrlPath: string): string {
+	const encodedOriginalUrl = proxyUrlPath.slice(OPAQUE_PATH_PREFIX.length);
 	if (!encodedOriginalUrl) {
 		throw new Error('Encoded URL is missing in the proxy URL path.');
 	}
-	return decodeV2PathComponent(encodedOriginalUrl);
+	return decodeOpaquePathComponent(encodedOriginalUrl);
 }
 
-export function buildV2ExternalMediaProxyPath(inputUrl: string): string {
+export function buildOpaqueExternalMediaProxyPath(inputUrl: string): string {
 	const parsedUrl = new URL(inputUrl);
-	return `${V2_PATH_PREFIX}${encodeV2PathComponent(parsedUrl.toString())}`;
+	return `${OPAQUE_PATH_PREFIX}${encodeOpaquePathComponent(parsedUrl.toString())}`;
 }
 
 export function buildExternalMediaProxyPath(inputUrl: string): string {
@@ -109,9 +109,9 @@ export function buildExternalMediaProxyPath(inputUrl: string): string {
 }
 
 export function reconstructOriginalUrl(proxyUrlPath: string): string {
-	const reconstructedUrl = proxyUrlPath.startsWith(V2_PATH_PREFIX)
-		? reconstructV2OriginalUrl(proxyUrlPath)
-		: reconstructLegacyOriginalUrl(proxyUrlPath);
+	const reconstructedUrl = proxyUrlPath.startsWith(OPAQUE_PATH_PREFIX)
+		? reconstructOpaqueOriginalUrl(proxyUrlPath)
+		: reconstructSegmentedOriginalUrl(proxyUrlPath);
 	new URL(reconstructedUrl);
 	return reconstructedUrl;
 }

fluxer_common/Cargo.toml

@@ -20,3 +20,6 @@ hmac = "0.13.0"
 sha2 = "0.11.0"
 thiserror = "2"
 urlencoding = "2.1.3"
+
+[dev-dependencies]
+url = "2.5.8"

fluxer_common/src/external_media_path.rs

@@ -3,7 +3,7 @@
 use base64::prelude::*;
 use thiserror::Error;
 
-const V2_PREFIX: &str = "v2/";
+const OPAQUE_PREFIX: &str = "v2/";
 
 #[derive(Debug, Error, Eq, PartialEq)]
 pub enum ExternalPathError {
@@ -15,9 +15,9 @@ pub enum ExternalPathError {
     InvalidUtf8,
 }
 
-pub fn build_v2_external_media_proxy_path(input_url: &str) -> String {
+pub fn build_opaque_external_media_proxy_path(input_url: &str) -> String {
     format!(
-        "{V2_PREFIX}{}",
+        "{OPAQUE_PREFIX}{}",
         BASE64_URL_SAFE_NO_PAD.encode(input_url.as_bytes())
     )
 }
@@ -51,14 +51,22 @@ pub fn build_external_media_proxy_path(input_url: &str) -> Result<String, Extern
     if scheme.is_empty() || remainder.is_empty() {
         return Err(ExternalPathError::InvalidExternalPath);
     }
-    let (authority_and_path, query) = match remainder.split_once('?') {
-        Some((head, tail)) => (head, Some(tail)),
-        None => (remainder, None),
+    let fetchable = match remainder.split_once('#') {
+        Some((head, _)) => head,
+        None => remainder,
     };
-    let (host, path) = match authority_and_path.split_once('/') {
-        Some((host, path)) => (host, path),
+    let (authority_and_path, query) = match fetchable.split_once('?') {
+        Some((head, tail)) => (head, (!tail.is_empty()).then_some(tail)),
+        None => (fetchable, None),
+    };
+    let (authority, path) = match authority_and_path.split_once('/') {
+        Some((authority, path)) => (authority, path),
         None => (authority_and_path, ""),
     };
+    let host = match authority.rsplit_once('@') {
+        Some((_, after_credentials)) => after_credentials,
+        None => authority,
+    };
     if host.is_empty() {
         return Err(ExternalPathError::InvalidExternalPath);
     }
@@ -79,9 +87,9 @@ pub fn build_external_media_proxy_path(input_url: &str) -> Result<String, Extern
     Ok(segments.join("/"))
 }
 
-fn decode_v2(proxy_path: &str) -> Result<String, ExternalPathError> {
+fn decode_opaque(proxy_path: &str) -> Result<String, ExternalPathError> {
     let encoded = proxy_path
-        .strip_prefix(V2_PREFIX)
+        .strip_prefix(OPAQUE_PREFIX)
         .ok_or(ExternalPathError::InvalidExternalPath)?;
     if encoded.is_empty() {
         return Err(ExternalPathError::InvalidExternalPath);
@@ -129,7 +137,7 @@ pub fn percent_decode_string(input: &str, plus_as_space: bool) -> String {
     String::from_utf8_lossy(&percent_decode(input, plus_as_space)).into_owned()
 }
 
-fn legacy_protocol_index(parts: &[&str]) -> Option<usize> {
+fn segmented_protocol_index(parts: &[&str]) -> Option<usize> {
     for (index, part) in parts.iter().enumerate() {
         if part.is_empty() {
             continue;
@@ -149,10 +157,10 @@ fn legacy_protocol_index(parts: &[&str]) -> Option<usize> {
     None
 }
 
-fn reconstruct_legacy(proxy_path: &str) -> Result<String, ExternalPathError> {
+fn reconstruct_segmented(proxy_path: &str) -> Result<String, ExternalPathError> {
     let parts: Vec<&str> = proxy_path.split('/').collect();
     let protocol_index =
-        legacy_protocol_index(&parts).ok_or(ExternalPathError::InvalidExternalPath)?;
+        segmented_protocol_index(&parts).ok_or(ExternalPathError::InvalidExternalPath)?;
     if protocol_index + 1 >= parts.len() {
         return Err(ExternalPathError::InvalidExternalPath);
     }
@@ -170,10 +178,10 @@ fn reconstruct_legacy(proxy_path: &str) -> Result<String, ExternalPathError> {
 }
 
 pub fn reconstruct_original_url(proxy_path: &str) -> Result<String, ExternalPathError> {
-    if proxy_path.starts_with(V2_PREFIX) {
-        decode_v2(proxy_path)
+    if proxy_path.starts_with(OPAQUE_PREFIX) {
+        decode_opaque(proxy_path)
     } else {
-        reconstruct_legacy(proxy_path)
+        reconstruct_segmented(proxy_path)
     }
 }
 
@@ -220,6 +228,56 @@ mod tests {
         );
     }
 
+    #[test]
+    fn leaves_the_fragment_out_of_the_signed_path() {
+        assert_eq!(
+            build_external_media_proxy_path("https://example.com/a.gif").unwrap(),
+            build_external_media_proxy_path("https://example.com/a.gif#anchor").unwrap()
+        );
+        assert_eq!(
+            build_external_media_proxy_path("https://example.com/a.gif?v=1").unwrap(),
+            build_external_media_proxy_path("https://example.com/a.gif?v=1#anchor").unwrap()
+        );
+    }
+
+    #[test]
+    fn leaves_credentials_out_of_the_host_segment() {
+        assert_eq!(
+            "https/cdn.example.com/a.gif",
+            build_external_media_proxy_path("https://reader:[email protected]/a.gif").unwrap()
+        );
+        assert_eq!(
+            "https/cdn.example.com/a.gif",
+            build_external_media_proxy_path("https://[email protected]/a.gif").unwrap()
+        );
+    }
+
+    #[test]
+    fn leaves_out_a_query_that_reconstructs_to_nothing() {
+        assert_eq!(
+            build_external_media_proxy_path("https://example.com/a.gif").unwrap(),
+            build_external_media_proxy_path("https://example.com/a.gif?").unwrap()
+        );
+    }
+
+    #[test]
+    fn matches_the_typescript_builder_on_the_shared_vectors() {
+        let raw = include_str!("testdata/external_media_proxy_path_vectors.json");
+        let vectors: serde_json::Value = serde_json::from_str(raw).expect("vectors parse as json");
+        let vectors = vectors.as_array().expect("vectors are an array");
+        assert!(!vectors.is_empty());
+        for vector in vectors {
+            let original = vector["url"].as_str().expect("vector carries a url");
+            let expected = vector["path"].as_str().expect("vector carries a path");
+            let normalized = url::Url::parse(original).expect("vector url parses");
+            assert_eq!(
+                expected,
+                build_external_media_proxy_path(normalized.as_str()).expect("path builds"),
+                "vector {original}"
+            );
+        }
+    }
+
     #[test]
     fn keeps_a_non_default_port() {
         assert_eq!(
@@ -269,7 +327,7 @@ mod tests {
 
     #[test]
     fn v2_path_roundtrip() {
-        let path = build_v2_external_media_proxy_path("https://example.com/a b.png?x=1");
+        let path = build_opaque_external_media_proxy_path("https://example.com/a b.png?x=1");
         let decoded = reconstruct_original_url(&path).unwrap();
         assert_eq!("https://example.com/a b.png?x=1", decoded);
     }

fluxer_common/src/testdata/external_media_proxy_path_vectors.json

@@ -0,0 +1,70 @@
+[
+	{
+		"url": "https://static.example.com/a.gif",
+		"path": "https/static.example.com/a.gif"
+	},
+	{
+		"url": "https://EXAMPLE.com/a.gif",
+		"path": "https/example.com/a.gif"
+	},
+	{
+		"url": "https://example.com:443/a.gif",
+		"path": "https/example.com/a.gif"
+	},
+	{
+		"url": "http://example.com:80/a.gif",
+		"path": "http/example.com/a.gif"
+	},
+	{
+		"url": "https://example.com:8443/a.gif",
+		"path": "https/example.com:8443/a.gif"
+	},
+	{
+		"url": "https://example.com/a/./b/../c.gif",
+		"path": "https/example.com/a/c.gif"
+	},
+	{
+		"url": "https://example.com/a b.gif",
+		"path": "https/example.com/a%2520b.gif"
+	},
+	{
+		"url": "https://example.com/café.gif",
+		"path": "https/example.com/caf%25C3%25A9.gif"
+	},
+	{
+		"url": "https://☃.example.com/a.gif",
+		"path": "https/xn--n3h.example.com/a.gif"
+	},
+	{
+		"url": "https://example.com/a.gif?v=1&x=2",
+		"path": "%3Fv%3D1%26x%3D2/https/example.com/a.gif"
+	},
+	{
+		"url": "https://example.com/a.gif#anchor",
+		"path": "https/example.com/a.gif"
+	},
+	{
+		"url": "https://example.com/a.gif?v=1#anchor",
+		"path": "%3Fv%3D1/https/example.com/a.gif"
+	},
+	{
+		"url": "https://reader:[email protected]/a.gif",
+		"path": "https/cdn.example.com/a.gif"
+	},
+	{
+		"url": "https://example.com/a.gif?",
+		"path": "https/example.com/a.gif"
+	},
+	{
+		"url": "https://[::1]:8443/a.gif",
+		"path": "https/[::1]:8443/a.gif"
+	},
+	{
+		"url": "https://example.com",
+		"path": "https/example.com"
+	},
+	{
+		"url": "https://example.com/p%2Fq.gif",
+		"path": "https/example.com/p%252Fq.gif"
+	}
+]

fluxer_media_proxy/src/external_path.rs

@@ -1,6 +1,44 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 pub use fluxer_common::external_media_path::{
-    ExternalPathError, build_external_media_proxy_path, build_v2_external_media_proxy_path,
+    ExternalPathError, build_external_media_proxy_path, build_opaque_external_media_proxy_path,
     percent_decode, percent_decode_string, reconstruct_original_url,
 };
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::public_net_policy;
+
+    #[test]
+    fn a_signed_path_never_decodes_into_a_url_the_fetch_policy_refuses() {
+        let credentialed = "https://reader:[email protected]/i.png";
+        assert_eq!(
+            Err(public_net_policy::Error::BlockedUrl),
+            public_net_policy::parse_url(credentialed)
+        );
+
+        let path = build_external_media_proxy_path(credentialed).expect("path builds");
+        let decoded = reconstruct_original_url(&path).expect("path decodes");
+
+        assert_eq!("https://cdn.example.com/i.png", decoded);
+        assert!(public_net_policy::parse_url(&decoded).is_ok());
+    }
+
+    #[test]
+    fn a_fragment_never_splits_the_path_for_bytes_the_fetch_would_not_see() {
+        assert_eq!(
+            build_external_media_proxy_path("https://cdn.example.com/i.png").expect("path builds"),
+            build_external_media_proxy_path("https://cdn.example.com/i.png#one")
+                .expect("path builds")
+        );
+        assert_eq!(
+            public_net_policy::parse_url("https://cdn.example.com/i.png")
+                .expect("plain url parses")
+                .path_query,
+            public_net_policy::parse_url("https://cdn.example.com/i.png#one")
+                .expect("anchored url parses")
+                .path_query
+        );
+    }
+}