perf(desktop): cache display media and retain shown images (#3264)

fluxer_app/src/features/channel/components/embeds/media/EmbedVideo.tsx

@@ -22,6 +22,7 @@ import {useDeleteAttachment} from '@app/features/messaging/hooks/useDeleteAttach
 import {useMatureMedia} from '@app/features/messaging/hooks/useMatureMedia';
 import {useMediaFavorite} from '@app/features/messaging/hooks/useMediaFavorite';
 import {useNearViewport} from '@app/features/messaging/hooks/useNearViewport';
+import {resolveDesktopCrossOriginMediaURL} from '@app/features/messaging/utils/DesktopResourceUrl';
 import {createDownloadHandler} from '@app/features/messaging/utils/FileDownloadUtils';
 import * as ImageCacheUtils from '@app/features/messaging/utils/ImageCacheUtils';
 import {buildMediaProxyURL, mediaDevicePixelRatio} from '@app/features/messaging/utils/MediaProxyUtils';
@@ -575,7 +576,7 @@ const EmbedVideo: FC<EmbedVideoProps> = observer(
 					muted
 					playsInline
 					preload="metadata"
-					src={effectiveSrc}
+					src={resolveDesktopCrossOriginMediaURL(effectiveSrc)}
 					tabIndex={-1}
 					onLoadedMetadata={handleMetadataProbeLoaded}
 					data-flx="channel.embeds.media.embed-video.metadata-probe"
@@ -609,7 +610,7 @@ const EmbedVideo: FC<EmbedVideoProps> = observer(
 								<video
 									ref={inlineVideoRef}
 									className={styles.inlineVideo}
-									src={effectiveSrc}
+									src={resolveDesktopCrossOriginMediaURL(effectiveSrc)}
 									poster={presentedPosterSrc ?? undefined}
 									autoPlay
 									playsInline

fluxer_app/src/features/channel/components/embeds/media/VoiceMessagePlayer.tsx

@@ -7,6 +7,7 @@ import {useMaybeMessageViewContext} from '@app/features/channel/components/Messa
 import {PAUSE_DESCRIPTOR, PLAY_DESCRIPTOR} from '@app/features/i18n/utils/CommonMessageDescriptors';
 import {getCachedNumberFormat} from '@app/features/i18n/utils/IntlCache';
 import {useAttachmentRefreshOnError} from '@app/features/messaging/hooks/useAttachmentRefreshOnError';
+import {resolveDesktopCrossOriginMediaURL} from '@app/features/messaging/utils/DesktopResourceUrl';
 import {buildMediaProxyURL} from '@app/features/messaging/utils/MediaProxyUtils';
 import {Logger} from '@app/features/platform/utils/AppLogger';
 import {remFromPx} from '@app/features/theme/layout/RemFromPx';
@@ -324,7 +325,7 @@ const VoiceMessagePlayer: React.FC<VoiceMessagePlayerProps> = observer(
 			>
 				<audio
 					ref={mediaRef as React.RefObject<HTMLAudioElement>}
-					src={hasStarted || wantsMetadata ? effectiveSrc : undefined}
+					src={hasStarted || wantsMetadata ? resolveDesktopCrossOriginMediaURL(effectiveSrc) : undefined}
 					preload={wantsMetadata ? 'metadata' : 'none'}
 					onError={handleMediaError}
 					data-flx="channel.embeds.media.voice-message-player.audio"

fluxer_app/src/features/messaging/components/modals/ImageCropModal.tsx

@@ -7,6 +7,7 @@ import type {AnimatedCropSourceFormat, CropParams} from '@app/features/expressio
 import {CropPipelineError, cropAnimatedImage} from '@app/features/expressions/workers/AnimatedImageCropWorkerManager';
 import {showMessagingErrorModal} from '@app/features/messaging/components/alerts/MessagingErrorModalUtils';
 import styles from '@app/features/messaging/components/modals/ImageCropModal.module.css';
+import {resolveDesktopCrossOriginMediaURL} from '@app/features/messaging/utils/DesktopResourceUrl';
 import {formatFileSize} from '@app/features/messaging/utils/FileUtils';
 import {Logger} from '@app/features/platform/utils/AppLogger';
 import {Button} from '@app/features/ui/button/Button';
@@ -720,7 +721,7 @@ export const ImageCropModal: React.FC<ImageCropModalProps> = observer(
 					width: displayDimensions.width * zoomRatio,
 					height: displayDimensions.height * zoomRatio,
 				};
-				const response = await fetch(imageUrl);
+				const response = await fetch(resolveDesktopCrossOriginMediaURL(imageUrl) ?? imageUrl);
 				if (controller.signal.aborted) return;
 				if (!response.ok) {
 					throw new Error('Failed to fetch image data');
@@ -869,7 +870,7 @@ export const ImageCropModal: React.FC<ImageCropModalProps> = observer(
 					>
 						<img
 							ref={imageRef}
-							src={imageUrl}
+							src={resolveDesktopCrossOriginMediaURL(imageUrl)}
 							alt={i18n._(CROP_PREVIEW_DESCRIPTOR)}
 							className={styles.image}
 							style={{

fluxer_app/src/features/messaging/utils/DesktopResourceUrl.ts

@@ -21,6 +21,7 @@ const UPLOAD_RELAY_MULTIPART_PARAMETERS: ReadonlySet<string> = new Set(['uploadI
 const UPLOAD_RELAY_BASE_PATH_PATTERN = /(?:^|\/)relay(?:\/|$)/u;
 const UPLOAD_RELAY_BASE_SUFFIX_PATTERN = /\/v1\/relay$/u;
 const TRAILING_SLASHES_PATTERN = /\/+$/u;
+const LOOPBACK_IPV4_PATTERN = /^127(?:\.\d{1,3}){3}$/u;
 
 class DesktopUploadRelayUrlError extends Error {
 	constructor(message: string) {
@@ -56,6 +57,44 @@ export function wrapDesktopLocalResourceURLForInstance(value: string, instanceKe
 	return buildDesktopLocalResourceProxyURL(target, instanceKey);
 }
 
+export function resolveDesktopDisplayResourceURL(value: string): string {
+	if (!isDesktopLocalAppDocument()) return value;
+	const target = parseDisplayTarget(value);
+	if (target == null) return value;
+	if (isDirectlyDisplayableTarget(target)) return target.toString();
+	return wrapResolvedDesktopLocalResourceURL(target.toString(), false);
+}
+
+export function resolveDesktopDisplayResourceURLForInstance(value: string, instanceKey: string): string {
+	if (!isDesktopLocalAppDocument()) return value;
+	const target = parseDisplayTarget(value);
+	if (target != null && isDirectlyDisplayableTarget(target)) return target.toString();
+	return wrapDesktopLocalResourceURLForInstance(value, instanceKey);
+}
+
+export function resolveDesktopCrossOriginMediaURL(value: string | undefined): string | undefined {
+	if (value == null || value.length === 0) return value;
+	return wrapDesktopLocalResourceURL(value);
+}
+
+function parseDisplayTarget(value: string): URL | null {
+	try {
+		return new URL(parseDesktopLocalResourceProxyTarget(value));
+	} catch {
+		return null;
+	}
+}
+
+function isDirectlyDisplayableTarget(target: URL): boolean {
+	if (target.protocol === 'https:') return true;
+	return target.protocol === 'http:' && isLoopbackHostname(target.hostname);
+}
+
+function isLoopbackHostname(hostname: string): boolean {
+	if (hostname === 'localhost' || hostname.endsWith('.localhost') || hostname === '[::1]') return true;
+	return LOOPBACK_IPV4_PATTERN.test(hostname);
+}
+
 export function updateDesktopLocalResourceURLTarget(value: string, updateTarget: (target: URL) => boolean): string {
 	const targetValue = parseDesktopLocalResourceProxyTarget(value);
 	let target: URL;

fluxer_app/src/features/messaging/utils/ImageCacheUtils.ts

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+import {retainImage} from '@app/features/messaging/utils/ImageRetention';
 import {LRUCache} from 'lru-cache';
 
 export interface CachedImageSize {
@@ -127,6 +128,7 @@ function failEntry(entry: ImageCacheEntry): void {
 }
 
 function settleLoaded(entry: ImageCacheEntry, image: HTMLImageElement): void {
+	retainImage(image);
 	clearRetryState(entry);
 	detachImageLoad(entry);
 	entry.loaded = true;

fluxer_app/src/features/messaging/utils/ImageRetention.ts

@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {LRUCache} from 'lru-cache';
+
+const RETAINED_IMAGE_BYTE_BUDGET = 192 * 1024 * 1024;
+const RETAINED_IMAGE_MAX_ENTRIES = 2000;
+const RETAINED_IMAGE_MIN_BYTES = 1024;
+const BYTES_PER_PIXEL = 4;
+const MAX_RETAINED_SOURCE_LENGTH = 16 * 1024;
+
+const retainedImages = new LRUCache<string, HTMLImageElement>({
+	max: RETAINED_IMAGE_MAX_ENTRIES,
+	maxSize: RETAINED_IMAGE_BYTE_BUDGET,
+	sizeCalculation: (image) => estimateDecodedBytes(image.naturalWidth, image.naturalHeight),
+});
+
+function estimateDecodedBytes(width: number, height: number): number {
+	return Math.min(RETAINED_IMAGE_BYTE_BUDGET, Math.max(RETAINED_IMAGE_MIN_BYTES, width * height * BYTES_PER_PIXEL));
+}
+
+function isRetainableSource(src: string): boolean {
+	return (
+		src.length > 0 && src.length <= MAX_RETAINED_SOURCE_LENGTH && !src.startsWith('data:') && !src.startsWith('blob:')
+	);
+}
+
+export function retainImage(image: HTMLImageElement): void {
+	if (!image.complete || image.naturalWidth <= 0) return;
+	const src = image.currentSrc || image.src;
+	if (!isRetainableSource(src)) return;
+	if (retainedImages.get(src) != null) return;
+	const holder = new Image();
+	holder.decoding = 'async';
+	holder.src = src;
+	retainedImages.set(src, holder, {size: estimateDecodedBytes(image.naturalWidth, image.naturalHeight)});
+}
+
+export function installImageRetention(target: Document): () => void {
+	const handleLoad = (event: Event): void => {
+		if (event.target instanceof HTMLImageElement) retainImage(event.target);
+	};
+	target.addEventListener('load', handleLoad, true);
+	return () => target.removeEventListener('load', handleLoad, true);
+}

fluxer_app/src/features/messaging/utils/MediaProxyUtils.desktop.test.ts

@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// @vitest-environment happy-dom
+// @vitest-environment-options {"url":"fluxer-app://app/channels/@me"}
+
+import {beforeEach, describe, expect, test, vi} from 'vitest';
+
+const runtime = vi.hoisted(() => ({
+	mediaEndpoint: 'https://media.example.com',
+	staticCdnEndpoint: null as string | null,
+}));
+
+vi.mock('@app/features/platform/DesktopLocalAppRuntime', async (importOriginal) => ({
+	...(await importOriginal<object>()),
+	isDesktopLocalAppDocument: () => true,
+}));
+vi.mock('@app/features/app/state/InstanceSnapshotStore', () => ({runtimeInstanceKey: () => 'instance-key'}));
+vi.mock('@app/features/app/state/RuntimeConfig', () => ({
+	default: {
+		get mediaEndpoint() {
+			return runtime.mediaEndpoint;
+		},
+		get staticCdnEndpoint() {
+			return runtime.staticCdnEndpoint;
+		},
+		uploadRelayEndpoint: null,
+		getSnapshot: () => ({}),
+	},
+}));
+
+const {buildMediaProxyURL} = await import('@app/features/messaging/utils/MediaProxyUtils');
+const {resolveDesktopCrossOriginMediaURL} = await import('@app/features/messaging/utils/DesktopResourceUrl');
+
+describe('desktop media URLs', () => {
+	beforeEach(() => {
+		runtime.mediaEndpoint = 'https://media.example.com';
+	});
+
+	test('images on https media load directly so the HTTP cache serves revisits', () => {
+		const url = buildMediaProxyURL('https://media.example.com/attachments/1/2/a.png', {format: 'webp', width: 320});
+		expect(url).toBe('https://media.example.com/attachments/1/2/a.png?format=webp&width=320');
+	});
+
+	test('images on loopback http media load directly', () => {
+		runtime.mediaEndpoint = 'http://localhost:48090/media';
+		expect(buildMediaProxyURL('http://localhost:48090/media/avatars/1/a.webp')).toBe(
+			'http://localhost:48090/media/avatars/1/a.webp',
+		);
+	});
+
+	test('images on cleartext non-loopback media stay behind the local proxy', () => {
+		runtime.mediaEndpoint = 'http://192.168.1.20/media';
+		expect(buildMediaProxyURL('http://192.168.1.20/media/avatars/1/a.webp')).toMatch(/^fluxer-app:\/\/app\/proxy\//);
+	});
+
+	test('cross-origin media elements keep a same-origin source', () => {
+		expect(resolveDesktopCrossOriginMediaURL('https://media.example.com/attachments/1/2/v.mp4')).toMatch(
+			/^fluxer-app:\/\/app\/proxy\/instance-key\?/,
+		);
+	});
+});

fluxer_app/src/features/messaging/utils/MediaProxyUtils.ts

@@ -1,9 +1,9 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import {
+	resolveDesktopDisplayResourceURL,
+	resolveDesktopDisplayResourceURLForInstance,
 	unwrapDesktopLocalResourceURL,
-	wrapDesktopLocalResourceURL,
-	wrapDesktopLocalResourceURLForInstance,
 } from '@app/features/messaging/utils/DesktopResourceUrl';
 import {MEDIA_PROXY_IMAGE_SIZES, type MediaProxyImageSize} from '@fluxer/constants/src/MediaProxyImageSizes';
 
@@ -99,7 +99,7 @@ export function buildMediaProxyURL(originalUrl: string, options: MediaProxyOptio
 	if (!originalUrl) return originalUrl;
 	const url = new URL(unwrapDesktopLocalResourceURL(originalUrl));
 	appendMediaProxyParams(url, options);
-	return wrapDesktopLocalResourceURL(url.toString());
+	return resolveDesktopDisplayResourceURL(url.toString());
 }
 
 export function buildMediaProxyURLForInstance(
@@ -110,14 +110,14 @@ export function buildMediaProxyURLForInstance(
 	if (!originalUrl) return originalUrl;
 	const url = new URL(unwrapDesktopLocalResourceURL(originalUrl));
 	appendMediaProxyParams(url, options);
-	return wrapDesktopLocalResourceURLForInstance(url.toString(), instanceKey);
+	return resolveDesktopDisplayResourceURLForInstance(url.toString(), instanceKey);
 }
 
 export function buildFitInsideMediaProxyURL(originalUrl: string, options: FitInsideMediaProxyOptions = {}): string {
 	if (!originalUrl) return originalUrl;
 	const url = new URL(unwrapDesktopLocalResourceURL(originalUrl));
 	appendMediaProxyParams(url, getFitInsideProxyOptions(options));
-	return wrapDesktopLocalResourceURL(url.toString());
+	return resolveDesktopDisplayResourceURL(url.toString());
 }
 
 function readProxyDimensionParam(url: URL, key: 'width' | 'height'): number | undefined {
@@ -150,7 +150,7 @@ export function stripMediaProxyParams(proxyURL: string): string {
 	url.searchParams.delete('format');
 	url.searchParams.delete('quality');
 	url.searchParams.delete('animated');
-	return wrapDesktopLocalResourceURL(url.toString());
+	return resolveDesktopDisplayResourceURL(url.toString());
 }
 
 export function buildAnimatedImageProxyURL(proxyURL: string, width?: number, height?: number): string {

fluxer_app/src/features/ui/action_menu/items/MediaMenuData.tsx

@@ -257,7 +257,7 @@ async function fetchFirstPngBlob(urls: Array<string>): Promise<Blob> {
 	let lastError: unknown = null;
 	for (const url of urls) {
 		try {
-			const response = await fetch(url);
+			const response = await fetch(wrapDesktopLocalResourceURL(url));
 			if (!response.ok) {
 				throw new Error(`Unexpected response status ${response.status} for ${url}`);
 			}

fluxer_app/src/features/voice/components/media_player/components/AudioPlayer.tsx

@@ -2,6 +2,7 @@
 
 import {PAUSE_DESCRIPTOR, PLAY_DESCRIPTOR} from '@app/features/i18n/utils/CommonMessageDescriptors';
 import {useAttachmentRefreshOnError} from '@app/features/messaging/hooks/useAttachmentRefreshOnError';
+import {resolveDesktopCrossOriginMediaURL} from '@app/features/messaging/utils/DesktopResourceUrl';
 import FocusRing from '@app/features/ui/focus_ring/FocusRing';
 import {Tooltip} from '@app/features/ui/tooltip/Tooltip';
 import styles from '@app/features/voice/components/media_player/AudioPlayer.module.css';
@@ -210,7 +211,7 @@ export function AudioPlayer({
 			{/* biome-ignore lint/a11y/useMediaCaption: voice/audio attachments have no caption track source */}
 			<audio
 				ref={mediaRef as React.RefObject<HTMLAudioElement>}
-				src={sourceAttribute}
+				src={resolveDesktopCrossOriginMediaURL(sourceAttribute)}
 				preload={preloadAttribute}
 				onError={handleMediaError}
 				data-flx="voice.media-player.audio-player.audio"

fluxer_app/src/features/voice/components/media_player/components/InlineAudioPlayer.tsx

@@ -9,6 +9,7 @@ import {
 	REMOVE_FROM_FAVORITES_DESCRIPTOR,
 } from '@app/features/i18n/utils/CommonMessageDescriptors';
 import {useAttachmentRefreshOnError} from '@app/features/messaging/hooks/useAttachmentRefreshOnError';
+import {resolveDesktopCrossOriginMediaURL} from '@app/features/messaging/utils/DesktopResourceUrl';
 import {formatFileSize} from '@app/features/messaging/utils/FileUtils';
 import {remFromPx} from '@app/features/theme/layout/RemFromPx';
 import FocusRing from '@app/features/ui/focus_ring/FocusRing';
@@ -205,7 +206,7 @@ export function InlineAudioPlayer({
 			>
 				<audio
 					ref={mediaRef as React.RefObject<HTMLAudioElement>}
-					src={sourceAttribute}
+					src={resolveDesktopCrossOriginMediaURL(sourceAttribute)}
 					preload={preloadAttribute}
 					onError={handleMediaError}
 					data-flx="voice.media-player.inline-audio-player.audio"
@@ -333,7 +334,7 @@ export function InlineAudioPlayer({
 		>
 			<audio
 				ref={mediaRef as React.RefObject<HTMLAudioElement>}
-				src={sourceAttribute}
+				src={resolveDesktopCrossOriginMediaURL(sourceAttribute)}
 				preload={preloadAttribute}
 				onError={handleMediaError}
 				data-flx="voice.media-player.inline-audio-player.audio--2"

fluxer_app/src/features/voice/components/media_player/components/VideoPlayer.tsx

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import Accessibility from '@app/features/accessibility/state/Accessibility';
+import {resolveDesktopCrossOriginMediaURL} from '@app/features/messaging/utils/DesktopResourceUrl';
 import * as ImageCacheUtils from '@app/features/messaging/utils/ImageCacheUtils';
 import {decodeThumbHashDataURL} from '@app/features/messaging/utils/ThumbHashUtils';
 import {observeResize} from '@app/features/platform/utils/SharedResizeObserver';
@@ -155,7 +156,7 @@ function VideoSeekPreviewThumbnail({src, poster, time}: VideoSeekPreviewThumbnai
 		<video
 			ref={videoRef}
 			className={styles.seekPreviewVideo}
-			src={src}
+			src={resolveDesktopCrossOriginMediaURL(src)}
 			poster={poster}
 			preload="metadata"
 			muted
@@ -606,7 +607,7 @@ export const VideoPlayer = observer(function VideoPlayer({
 					<video
 						ref={mediaRef as React.RefObject<HTMLVideoElement>}
 						className={clsx(styles.video, renderModel.shouldHideVideo && styles.videoHidden)}
-						src={renderModel.shouldAttachSource ? src : undefined}
+						src={renderModel.shouldAttachSource ? resolveDesktopCrossOriginMediaURL(src) : undefined}
 						preload={renderModel.preloadAttribute}
 						crossOrigin="anonymous"
 						playsInline

fluxer_app/src/features/voice/components/modals/MobileVideoViewer.tsx

@@ -11,6 +11,7 @@ import {getCachedNumberFormat} from '@app/features/i18n/utils/IntlCache';
 import {isKeyboardActivationKey} from '@app/features/input/utils/KeyboardUtils';
 import {PanZoomSurface} from '@app/features/messaging/components/modals/media_modal/pan_zoom/PanZoomSurface';
 import type {ZoomState} from '@app/features/messaging/components/modals/media_modal/shared';
+import {resolveDesktopCrossOriginMediaURL} from '@app/features/messaging/utils/DesktopResourceUrl';
 import {remFromPx} from '@app/features/theme/layout/RemFromPx';
 import {
 	clampMediaTime,
@@ -404,7 +405,7 @@ export const MobileVideoViewer = observer(function MobileVideoViewer({
 				<video
 					ref={videoRef}
 					className={styles.video}
-					src={src}
+					src={resolveDesktopCrossOriginMediaURL(src)}
 					autoPlay={initialTime === undefined}
 					playsInline
 					loop={loop}