perf(app): defer media capture routing and skip offscreen skeleton layout (#1732)

fluxer_app/src/features/app/components/skeleton/PlaceholderSpecs.ts

@@ -19,10 +19,12 @@ export interface PlaceholderMessageGroup {
 	readonly usernameWidth: number;
 	readonly timestampWidth: number;
 	readonly attachment: PlaceholderAttachmentSize | null;
+	readonly height: number;
 }
 
 interface MutablePlaceholderMessageGroup extends PlaceholderMessageGroup {
 	attachment: PlaceholderAttachmentSize | null;
+	height: number;
 }
 
 export interface PlaceholderSpecs {
@@ -100,20 +102,20 @@ function generatePlaceholderSpecs(options: PlaceholderGenerationOptions): Placeh
 		for (let line = 0; line < lineCount; line++) {
 			lineWidths.push(LINE_WIDTH_MIN + random() * LINE_WIDTH_RANGE);
 		}
+		const groupHeight = compact
+			? MESSAGE_HEIGHT_COMPACT * lineCount
+			: COZY_LEAD_MESSAGE_HEIGHT + COZY_GROUPED_MESSAGE_HEIGHT * (lineCount - 1);
 		groups.push({
 			lineWidths,
 			usernameWidth: USERNAME_WIDTH_MIN + random() * USERNAME_WIDTH_RANGE,
 			timestampWidth: TIMESTAMP_WIDTH_MIN + random() * TIMESTAMP_WIDTH_RANGE,
 			attachment: null,
+			height: groupHeight,
 		});
 		if (index > 0) {
 			totalHeight += groupSpacing;
 		}
-		if (compact) {
-			totalHeight += MESSAGE_HEIGHT_COMPACT * lineCount;
-		} else {
-			totalHeight += COZY_LEAD_MESSAGE_HEIGHT + COZY_GROUPED_MESSAGE_HEIGHT * (lineCount - 1);
-		}
+		totalHeight += groupHeight;
 	}
 	const availableGroupIndices = Array.from(Array(groups.length).keys());
 	for (let index = 0; index < attachments && availableGroupIndices.length > 0; index++) {
@@ -123,6 +125,7 @@ function generatePlaceholderSpecs(options: PlaceholderGenerationOptions): Placeh
 			height: randomInRange(random, ATTACHMENT_HEIGHT_MIN, ATTACHMENT_HEIGHT_MAX),
 		};
 		groups[groupIndex].attachment = attachment;
+		groups[groupIndex].height += attachment.height + ATTACHMENT_MARGIN;
 		totalHeight += attachment.height + ATTACHMENT_MARGIN;
 	}
 	return {compact, compactAvatarsVisible, groups, totalHeight, groupSpacing};

fluxer_app/src/features/app/components/skeleton/ScrollFillerSkeleton.module.css

@@ -44,6 +44,7 @@
 .messageGroup {
 	position: relative;
 	z-index: 0;
+	content-visibility: auto;
 }
 
 .group {

fluxer_app/src/features/app/components/skeleton/ScrollFillerSkeleton.tsx

@@ -52,7 +52,10 @@ function CozyScrollFillerSkeletonGroup({
 	return (
 		<flx-message-list-skeleton-group
 			className={flxElementClassName(styles.messageGroup)}
-			style={{marginBottom: resolveSkeletonGroupMarginBottom(groupIndex, groupCount, groupSpacing)}}
+			style={{
+				marginBottom: resolveSkeletonGroupMarginBottom(groupIndex, groupCount, groupSpacing),
+				containIntrinsicSize: `auto ${remFromPx(group.height)}`,
+			}}
 			data-flx="app.skeleton.scroll-filler-skeleton.cozy-scroll-filler-skeleton-group.message-group"
 		>
 			<flx-message-list-skeleton-group-row
@@ -170,7 +173,10 @@ function CompactScrollFillerSkeletonGroup({
 	return (
 		<flx-message-list-skeleton-group
 			className={flxElementClassName(styles.messageGroup)}
-			style={{marginBottom: resolveSkeletonGroupMarginBottom(groupIndex, groupCount, groupSpacing)}}
+			style={{
+				marginBottom: resolveSkeletonGroupMarginBottom(groupIndex, groupCount, groupSpacing),
+				containIntrinsicSize: `auto ${remFromPx(group.height)}`,
+			}}
 			data-flx="app.skeleton.scroll-filler-skeleton.compact-scroll-filler-skeleton-group.message-group"
 		>
 			<flx-message-list-skeleton-compact-messages

fluxer_app/src/features/app/state/Initialization.ts

@@ -16,7 +16,6 @@ class Initialization {
 	state: InitializationState = InitializationState.LOADING;
 	hasCompletedInitialLoad = false;
 	error: string | null = null;
-	readyPayload: unknown = null;
 
 	constructor() {
 		makeAutoObservable(this, {}, {autoBind: true});
@@ -46,29 +45,25 @@ class Initialization {
 	setLoading(): void {
 		this.state = InitializationState.LOADING;
 		this.error = null;
-		this.readyPayload = null;
 	}
 
 	@action
 	setConnecting(): void {
 		this.state = InitializationState.CONNECTING;
 		this.error = null;
-		this.readyPayload = null;
 	}
 
 	@action
-	setReady(payload: unknown): void {
+	setReady(): void {
 		this.state = InitializationState.READY;
 		this.hasCompletedInitialLoad = true;
 		this.error = null;
-		this.readyPayload = payload;
 	}
 
 	@action
 	setError(error: string): void {
 		this.state = InitializationState.ERROR;
 		this.error = error;
-		this.readyPayload = null;
 	}
 
 	@action
@@ -76,7 +71,6 @@ class Initialization {
 		this.state = InitializationState.LOADING;
 		this.hasCompletedInitialLoad = false;
 		this.error = null;
-		this.readyPayload = null;
 	}
 }
 

fluxer_app/src/features/gateway/events/GatewayReady.ts

@@ -183,7 +183,7 @@ function handleReadyInternal(data: ReadyPayload, context: GatewayHandlerContext)
 	GuildReadState.handleConnectionOpen();
 	Presence.handleConnectionOpen(data.user, guilds, data.presences);
 	MediaEngine.handleConnectionOpen(guilds);
-	Initialization.setReady(data);
+	Initialization.setReady();
 	context.setReady();
 	Messages.handleConnectionOpen((channelId) => ReadStates.getUnreadJumpAnchor(channelId));
 }

fluxer_app/src/features/notification/utils/SoundUtils.ts

@@ -236,10 +236,33 @@ export function getSoundCaptureMasterGainNode(): GainNode {
 	return getMasterGainNode();
 }
 
+const soundCaptureActivationListeners: Set<() => void> = new Set();
+
+export function isSoundCaptureActive(): boolean {
+	return captureTaps.size > 0;
+}
+
+export function onSoundCaptureActivated(listener: () => void): () => void {
+	soundCaptureActivationListeners.add(listener);
+	return () => {
+		soundCaptureActivationListeners.delete(listener);
+	};
+}
+
 export function addSoundCaptureDestination(node: AudioNode): void {
 	if (captureTaps.has(node)) return;
 	const master = getMasterGainNode();
+	const wasInactive = captureTaps.size === 0;
 	captureTaps.add(node);
+	if (wasInactive) {
+		for (const listener of soundCaptureActivationListeners) {
+			try {
+				listener();
+			} catch (error) {
+				logger.warn('Sound capture activation listener failed', error);
+			}
+		}
+	}
 	try {
 		master.connect(node);
 	} catch (error) {

fluxer_app/src/features/voice/utils/InAppMediaSoundCapture.test.ts

@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {beforeEach, describe, expect, it, vi} from 'vitest';
+
+const master = {connect: vi.fn(), disconnect: vi.fn()};
+const createdGains: Array<{connect: ReturnType<typeof vi.fn>; disconnect: ReturnType<typeof vi.fn>}> = [];
+let captureActive = true;
+
+vi.mock('@app/features/notification/utils/SoundUtils', () => ({
+	getSoundCaptureAudioContext: () => ({
+		createMediaElementSource: vi.fn(() => ({connect: vi.fn(), disconnect: vi.fn()})),
+		createGain: vi.fn(() => {
+			const gain = {connect: vi.fn(), disconnect: vi.fn(), gain: {value: 1}};
+			createdGains.push(gain);
+			return gain;
+		}),
+	}),
+	getSoundCaptureMasterGainNode: () => master,
+	isSoundCaptureActive: () => captureActive,
+	onSoundCaptureActivated: () => () => undefined,
+}));
+
+const {routeMediaElementForSoundCapture} = await import('@app/features/voice/utils/InAppMediaSoundCapture');
+
+function createElement(): HTMLMediaElement {
+	const listeners = new Map<string, () => void>();
+	return {
+		volume: 1,
+		muted: false,
+		addEventListener: (type: string, listener: () => void) => listeners.set(type, listener),
+		removeEventListener: (type: string) => listeners.delete(type),
+		listenerCount: () => listeners.size,
+	} as unknown as HTMLMediaElement & {listenerCount: () => number};
+}
+
+describe('in-app media sound capture', () => {
+	beforeEach(() => {
+		createdGains.length = 0;
+		captureActive = true;
+	});
+
+	it('disconnects the element from the master graph when the caller disposes', () => {
+		const element = createElement();
+		const dispose = routeMediaElementForSoundCapture(element);
+		const gain = createdGains.at(-1);
+		expect(gain?.connect).toHaveBeenCalledWith(master);
+		dispose();
+		expect(gain?.disconnect).toHaveBeenCalled();
+	});
+
+	it('removes the volume listener when the caller disposes', () => {
+		const element = createElement() as HTMLMediaElement & {listenerCount: () => number};
+		const dispose = routeMediaElementForSoundCapture(element);
+		expect(element.listenerCount()).toBe(1);
+		dispose();
+		expect(element.listenerCount()).toBe(0);
+	});
+
+	it('reuses the existing source node when the same element is routed again', () => {
+		const element = createElement();
+		routeMediaElementForSoundCapture(element)();
+		const gainsAfterFirst = createdGains.length;
+		const dispose = routeMediaElementForSoundCapture(element);
+		expect(createdGains.length).toBe(gainsAfterFirst);
+		expect(createdGains.at(-1)?.connect).toHaveBeenCalledTimes(2);
+		dispose();
+	});
+});

fluxer_app/src/features/voice/utils/InAppMediaSoundCapture.ts

@@ -1,17 +1,24 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-import {getSoundCaptureAudioContext, getSoundCaptureMasterGainNode} from '@app/features/notification/utils/SoundUtils';
+import {
+	getSoundCaptureAudioContext,
+	getSoundCaptureMasterGainNode,
+	isSoundCaptureActive,
+	onSoundCaptureActivated,
+} from '@app/features/notification/utils/SoundUtils';
 import {Logger} from '@app/features/platform/utils/AppLogger';
 
 const logger = new Logger('InAppMediaSoundCapture');
 
 interface RoutedElement {
+	connected: boolean;
 	source: MediaElementAudioSourceNode;
 	gain: GainNode;
 	volumeListener: () => void;
 }
 
 const routedElements: WeakMap<HTMLMediaElement, RoutedElement> = new WeakMap();
+const pendingElements: Set<WeakRef<HTMLMediaElement>> = new Set();
 
 function syncGainFromElement(el: HTMLMediaElement, gain: GainNode): void {
 	const volume = el.muted ? 0 : Math.max(0, Math.min(1, el.volume));
@@ -20,9 +27,20 @@ function syncGainFromElement(el: HTMLMediaElement, gain: GainNode): void {
 	} catch {}
 }
 
-export function routeMediaElementForSoundCapture(element: HTMLMediaElement): () => void {
-	if (routedElements.has(element)) {
-		return () => undefined;
+function attachToCaptureGraph(element: HTMLMediaElement): void {
+	const existing = routedElements.get(element);
+	if (existing) {
+		if (existing.connected) return;
+		try {
+			existing.gain.connect(getSoundCaptureMasterGainNode());
+		} catch (error) {
+			logger.debug('Failed to reconnect media element to sound capture graph', {error});
+			return;
+		}
+		element.addEventListener('volumechange', existing.volumeListener);
+		syncGainFromElement(element, existing.gain);
+		existing.connected = true;
+		return;
 	}
 	let ctx: AudioContext;
 	let master: GainNode;
@@ -31,7 +49,7 @@ export function routeMediaElementForSoundCapture(element: HTMLMediaElement): ()
 		master = getSoundCaptureMasterGainNode();
 	} catch (error) {
 		logger.debug('Sound capture context unavailable; leaving media element on native playback', {error});
-		return () => undefined;
+		return;
 	}
 	let source: MediaElementAudioSourceNode;
 	let gain: GainNode;
@@ -42,23 +60,45 @@ export function routeMediaElementForSoundCapture(element: HTMLMediaElement): ()
 		gain.connect(master);
 	} catch (error) {
 		logger.debug('Failed to route media element through sound capture graph', {error});
-		return () => undefined;
+		return;
 	}
 	syncGainFromElement(element, gain);
 	const volumeListener = (): void => syncGainFromElement(element, gain);
 	element.addEventListener('volumechange', volumeListener);
-	const routed: RoutedElement = {source, gain, volumeListener};
-	routedElements.set(element, routed);
+	routedElements.set(element, {source, gain, volumeListener, connected: true});
+}
+
+function detachFromCaptureGraph(element: HTMLMediaElement): void {
+	const entry = routedElements.get(element);
+	if (!entry || !entry.connected) return;
+	entry.connected = false;
+	element.removeEventListener('volumechange', entry.volumeListener);
+	try {
+		entry.gain.disconnect();
+	} catch {}
+}
+
+function flushPendingElements(): void {
+	for (const ref of pendingElements) {
+		const element = ref.deref();
+		pendingElements.delete(ref);
+		if (element) {
+			attachToCaptureGraph(element);
+		}
+	}
+}
+
+onSoundCaptureActivated(flushPendingElements);
+
+export function routeMediaElementForSoundCapture(element: HTMLMediaElement): () => void {
+	if (isSoundCaptureActive()) {
+		attachToCaptureGraph(element);
+		return () => detachFromCaptureGraph(element);
+	}
+	const ref = new WeakRef(element);
+	pendingElements.add(ref);
 	return () => {
-		const entry = routedElements.get(element);
-		if (!entry || entry !== routed) return;
-		routedElements.delete(element);
-		element.removeEventListener('volumechange', volumeListener);
-		try {
-			gain.disconnect();
-		} catch {}
-		try {
-			source.disconnect();
-		} catch {}
+		pendingElements.delete(ref);
+		detachFromCaptureGraph(element);
 	};
 }

fluxer_app/src/features/voice/utils/MediaDeviceStartupPreload.ts

@@ -27,7 +27,6 @@ export function startMediaDeviceStartupPreload(): () => void {
 			logger.debug('Failed to preload media devices', {error});
 		});
 	};
-	preloadDevices();
 	const disposePermissionListener = MediaPermission.addChangeListener(preloadDevices);
 	return () => {
 		stopped = true;