feat(voice): add screen share subscription debug helper (#2432)

fluxer_app/src/env.d.ts

@@ -16,6 +16,8 @@ interface FluxerDebugApi {
 	getClientInfoSync?: () => string;
 	getClientInfoObject?: () => Promise<unknown>;
 	getClientInfoObjectSync?: () => unknown;
+	getVoiceSubscriptionDebug?: () => Promise<unknown>;
+	getVoiceSubscriptionDebugJson?: () => Promise<string>;
 }
 
 type FluxerDebugGlobal = Record<string, unknown> & FluxerDebugApi;

fluxer_app/src/features/platform/utils/ClientInfo.ts

@@ -3,6 +3,7 @@
 import i18n from '@app/app/I18n';
 import Config from '@app/features/app/config/Config';
 import {Logger} from '@app/features/platform/utils/AppLogger';
+import {getFluxerDebugObject} from '@app/features/platform/utils/FluxerDebugGlobal';
 import {getElectronAPI, isDesktop} from '@app/features/ui/utils/NativeUtils';
 import type {DesktopInfo} from '@app/types/electron.d';
 import Bowser from 'bowser';
@@ -325,29 +326,6 @@ export async function getFormattedClientInfo(): Promise<string> {
 	return formatClientBuildInfo(await getClientInfo());
 }
 
-interface FluxerDebugApi {
-	getClientInfo?: () => Promise<string>;
-	getClientInfoSync?: () => string;
-	getClientInfoObject?: () => Promise<ClientInfo>;
-	getClientInfoObjectSync?: () => ClientInfo;
-}
-
-function getFluxerDebugObject(): (Record<string, unknown> & FluxerDebugApi) | null {
-	if (typeof window === 'undefined') {
-		return null;
-	}
-	const win = window as Window & {
-		__FLUXER_DEBUG__?: Record<string, unknown> & FluxerDebugApi;
-	};
-	if (win.__FLUXER_DEBUG__ === undefined || win.__FLUXER_DEBUG__ === null) {
-		win.__FLUXER_DEBUG__ = {};
-	}
-	if (typeof win.__FLUXER_DEBUG__ !== 'object' || Array.isArray(win.__FLUXER_DEBUG__)) {
-		return null;
-	}
-	return win.__FLUXER_DEBUG__;
-}
-
 export function installFluxerConfigDebugApi(): void {
 	const config = getFluxerDebugObject();
 	if (!config) {

fluxer_app/src/features/platform/utils/FluxerDebugGlobal.ts

@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+type FluxerDebugObject = NonNullable<Window['__FLUXER_DEBUG__']>;
+
+export function getFluxerDebugObject(): FluxerDebugObject | null {
+	if (typeof window === 'undefined') {
+		return null;
+	}
+	const existing = window.__FLUXER_DEBUG__;
+	if (existing === undefined || existing === null) {
+		const created: FluxerDebugObject = {};
+		window.__FLUXER_DEBUG__ = created;
+		return created;
+	}
+	if (typeof existing !== 'object' || Array.isArray(existing)) {
+		return null;
+	}
+	return existing;
+}

fluxer_app/src/features/voice/diagnostics/VoiceSubscriptionDebugApi.ts

@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {Logger} from '@app/features/platform/utils/AppLogger';
+import {getFluxerDebugObject} from '@app/features/platform/utils/FluxerDebugGlobal';
+import {loadLazyModule} from '@app/features/platform/utils/LazyModuleLoader';
+
+const logger = new Logger('VoiceSubscriptionDebugApi');
+
+async function collectReport(): Promise<unknown> {
+	const {collectVoiceSubscriptionDebugReport} = await loadLazyModule(
+		() => import('@app/features/voice/diagnostics/VoiceSubscriptionDebugReport'),
+	);
+	return collectVoiceSubscriptionDebugReport();
+}
+
+export function installVoiceSubscriptionDebugApi(): void {
+	const debugObject = getFluxerDebugObject();
+	if (!debugObject) {
+		return;
+	}
+	try {
+		debugObject.getVoiceSubscriptionDebug = collectReport;
+		debugObject.getVoiceSubscriptionDebugJson = async () => JSON.stringify(await collectReport(), null, 2);
+	} catch (error) {
+		logger.warn('Failed to install __FLUXER_DEBUG__ voice subscription helpers', error);
+	}
+}

fluxer_app/src/features/voice/diagnostics/VoiceSubscriptionDebugReport.ts

@@ -0,0 +1,236 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import Config from '@app/features/app/config/Config';
+import {
+	summarizeVoiceSubscriptionDebug,
+	type VoiceSubscriptionDebugMediaStreamTrack,
+	type VoiceSubscriptionDebugParticipant,
+	type VoiceSubscriptionDebugPublication,
+	type VoiceSubscriptionDebugSubscription,
+	type VoiceSubscriptionDebugSummary,
+	type VoiceSubscriptionDebugTransceiver,
+} from '@app/features/voice/diagnostics/VoiceSubscriptionDebugSummary';
+import MediaEngine from '@app/features/voice/engine/MediaEngineFacade';
+import type {VoiceMediaGraphSubscriptionCommand} from '@app/features/voice/engine/VoiceMediaGraph';
+import {voiceMediaGraphStore} from '@app/features/voice/engine/VoiceMediaGraphStore';
+import type {RemoteTrackPublication, Room} from 'livekit-client';
+
+interface VoiceSubscriptionDebugFailure {
+	key: string;
+	code: number;
+	reason: string;
+	reportedAt: number;
+	streamKey: string | null;
+	participantIdentity: string | null;
+	participantSid: string | null;
+	trackSid: string | null;
+	source: string | null;
+	generation: number | null;
+	error: string | null;
+}
+
+interface VoiceSubscriptionDebugReport {
+	capturedAt: string;
+	buildVersion: string;
+	graphNowMs: number;
+	connection: {
+		connected: boolean;
+		connecting: boolean;
+		reconnecting: boolean;
+		guildId: string | null;
+		channelId: string | null;
+		connectionId: string | null;
+		voiceServerEndpoint: string | null;
+		roomState: string | null;
+		localIdentity: string | null;
+	};
+	summary: VoiceSubscriptionDebugSummary;
+	publisherTransceivers: Array<VoiceSubscriptionDebugTransceiver>;
+	subscriberTransportPresent: boolean;
+	remoteParticipants: Array<VoiceSubscriptionDebugParticipant>;
+	subscriptions: Array<VoiceSubscriptionDebugSubscription>;
+	pendingSubscriptionCommands: Array<VoiceMediaGraphSubscriptionCommand>;
+	watchIntent: {viewerStreamKeys: Array<string>; deferredStopKeys: Array<string>};
+	watchAttempts: Array<{
+		streamKey: string;
+		attemptKey: string;
+		startedAt: number;
+		hasRenderedVideoFrame: boolean;
+		generation: number;
+	}>;
+	deadlines: Array<{
+		key: string;
+		kind: string;
+		streamKey: string | null;
+		subscriptionKey: string | null;
+		attemptKey: string | null;
+		generation: number;
+		dueAt: number;
+	}>;
+	failures: Array<VoiceSubscriptionDebugFailure>;
+}
+
+function describeMediaStreamTrack(
+	track: MediaStreamTrack | null | undefined,
+): VoiceSubscriptionDebugMediaStreamTrack | null {
+	if (!track) {
+		return null;
+	}
+	return {
+		id: track.id,
+		kind: track.kind,
+		enabled: track.enabled,
+		muted: track.muted,
+		readyState: track.readyState,
+	};
+}
+
+function describeTransceivers(room: Room | null): Array<VoiceSubscriptionDebugTransceiver> {
+	const publisher = room?.engine?.pcManager?.publisher;
+	if (!publisher) {
+		return [];
+	}
+	return publisher.getTransceivers().map((transceiver) => ({
+		mid: transceiver.mid,
+		direction: transceiver.direction,
+		currentDirection: transceiver.currentDirection,
+		receiver: describeMediaStreamTrack(transceiver.receiver?.track),
+		sender: describeMediaStreamTrack(transceiver.sender?.track),
+	}));
+}
+
+function describePublication(publication: RemoteTrackPublication): VoiceSubscriptionDebugPublication {
+	const dimensions = publication.dimensions;
+	return {
+		trackSid: publication.trackSid,
+		trackName: publication.trackName,
+		kind: publication.kind,
+		source: publication.source,
+		mimeType: publication.mimeType ?? null,
+		dimensions: dimensions ? {width: dimensions.width, height: dimensions.height} : null,
+		simulcasted: publication.simulcasted ?? null,
+		muted: publication.isMuted,
+		enabled: publication.isEnabled,
+		subscribed: publication.isSubscribed,
+		desired: publication.isDesired,
+		subscriptionStatus: publication.subscriptionStatus,
+		permissionStatus: publication.permissionStatus,
+		videoQuality: publication.videoQuality ?? null,
+		track: describeMediaStreamTrack(publication.track?.mediaStreamTrack),
+	};
+}
+
+function describeRemoteParticipants(room: Room | null): Array<VoiceSubscriptionDebugParticipant> {
+	if (!room) {
+		return [];
+	}
+	return Array.from(room.remoteParticipants.values()).map((participant) => ({
+		identity: participant.identity,
+		sid: participant.sid,
+		publications: Array.from(participant.trackPublications.values()).map(describePublication),
+	}));
+}
+
+function describeError(error: unknown): string | null {
+	if (error == null) {
+		return null;
+	}
+	if (error instanceof Error) {
+		return `${error.name}: ${error.message}`;
+	}
+	return String(error);
+}
+
+export function collectVoiceSubscriptionDebugReport(): VoiceSubscriptionDebugReport {
+	const room = MediaEngine.room;
+	const snapshot = voiceMediaGraphStore.getGraphSnapshot();
+	const transceivers = describeTransceivers(room);
+	const remoteParticipants = describeRemoteParticipants(room);
+	const subscriptions = Array.from(snapshot.subscriptionsByKey.entries()).map(([key, entry]) => ({
+		key,
+		participantIdentity: entry.participantIdentity,
+		source: entry.source,
+		desired: {
+			enabled: entry.desired.enabled,
+			quality: entry.desired.quality,
+			context: entry.desired.context,
+			isIntersecting: entry.desired.isIntersecting,
+			hasObservedElement: entry.desired.observedElement != null,
+		},
+		actual: {
+			subscribed: entry.actual.subscribed,
+			enabled: entry.actual.enabled,
+			quality: entry.actual.quality,
+			lastCommandAt: entry.actual.lastCommandAt,
+			lastError: entry.actual.lastError,
+		},
+		publication: {
+			available: entry.publication.available,
+			trackSid: entry.publication.trackSid,
+			observedAt: entry.publication.observedAt,
+		},
+		firstFrame: {renderedAt: entry.firstFrame.renderedAt},
+		subscribed: entry.subscribed,
+	}));
+	const pendingSubscriptionCommands = Array.from(snapshot.subscriptionCommands);
+	return {
+		capturedAt: new Date().toISOString(),
+		buildVersion: Config.PUBLIC_BUILD_VERSION ?? 'dev',
+		graphNowMs: voiceMediaGraphStore.nowMs(),
+		connection: {
+			connected: MediaEngine.connected,
+			connecting: MediaEngine.connecting,
+			reconnecting: MediaEngine.reconnecting,
+			guildId: MediaEngine.guildId,
+			channelId: MediaEngine.channelId,
+			connectionId: MediaEngine.connectionId,
+			voiceServerEndpoint: MediaEngine.voiceServerEndpoint,
+			roomState: room?.state ?? null,
+			localIdentity: room?.localParticipant.identity ?? null,
+		},
+		summary: summarizeVoiceSubscriptionDebug(
+			transceivers,
+			remoteParticipants,
+			subscriptions,
+			pendingSubscriptionCommands.length,
+		),
+		publisherTransceivers: transceivers,
+		subscriberTransportPresent: room?.engine?.pcManager?.subscriber != null,
+		remoteParticipants,
+		subscriptions,
+		pendingSubscriptionCommands,
+		watchIntent: {
+			viewerStreamKeys: Array.from(snapshot.watchIntent.viewerStreamKeys),
+			deferredStopKeys: Array.from(snapshot.watchIntent.deferredStopKeys),
+		},
+		watchAttempts: Array.from(snapshot.attemptsByStreamKey.entries()).map(([streamKey, attempt]) => ({
+			streamKey,
+			attemptKey: attempt.attemptKey,
+			startedAt: attempt.startedAt,
+			hasRenderedVideoFrame: attempt.hasRenderedVideoFrame,
+			generation: attempt.generation,
+		})),
+		deadlines: Array.from(snapshot.deadlinesByKey.entries()).map(([key, deadline]) => ({
+			key,
+			kind: deadline.kind,
+			streamKey: deadline.streamKey,
+			subscriptionKey: deadline.subscriptionKey,
+			attemptKey: deadline.attemptKey,
+			generation: deadline.generation,
+			dueAt: deadline.dueAt,
+		})),
+		failures: Array.from(snapshot.failuresByKey.entries()).map(([key, failure]) => ({
+			key,
+			code: failure.code,
+			reason: failure.reason,
+			reportedAt: failure.reportedAt,
+			streamKey: failure.streamKey ?? null,
+			participantIdentity: failure.participantIdentity ?? null,
+			participantSid: failure.participantSid ?? null,
+			trackSid: failure.trackSid ?? null,
+			source: failure.source ?? null,
+			generation: failure.generation ?? null,
+			error: describeError(failure.error),
+		})),
+	};
+}

fluxer_app/src/features/voice/diagnostics/VoiceSubscriptionDebugSummary.test.ts

@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {
+	summarizeVoiceSubscriptionDebug,
+	type VoiceSubscriptionDebugParticipant,
+	type VoiceSubscriptionDebugPublication,
+	type VoiceSubscriptionDebugSubscription,
+	type VoiceSubscriptionDebugTransceiver,
+} from '@app/features/voice/diagnostics/VoiceSubscriptionDebugSummary';
+import {describe, expect, it} from 'vitest';
+
+function transceiver(overrides: Partial<VoiceSubscriptionDebugTransceiver>): VoiceSubscriptionDebugTransceiver {
+	return {
+		mid: '0',
+		direction: 'recvonly',
+		currentDirection: 'recvonly',
+		receiver: {id: 'receiver', kind: 'video', enabled: true, muted: false, readyState: 'live'},
+		sender: null,
+		...overrides,
+	};
+}
+
+function publication(overrides: Partial<VoiceSubscriptionDebugPublication>): VoiceSubscriptionDebugPublication {
+	return {
+		trackSid: 'TR_1',
+		trackName: 'screen',
+		kind: 'video',
+		source: 'screen_share',
+		mimeType: 'video/VP9',
+		dimensions: {width: 1920, height: 1080},
+		simulcasted: false,
+		muted: false,
+		enabled: true,
+		subscribed: false,
+		desired: true,
+		subscriptionStatus: 'desired',
+		permissionStatus: 'allowed',
+		videoQuality: 2,
+		track: null,
+		...overrides,
+	};
+}
+
+function participant(publications: Array<VoiceSubscriptionDebugPublication>): VoiceSubscriptionDebugParticipant {
+	return {identity: 'user_1_connection-a', sid: 'PA_1', publications};
+}
+
+function subscription(overrides: Partial<VoiceSubscriptionDebugSubscription>): VoiceSubscriptionDebugSubscription {
+	return {
+		key: 'user_1_connection-a:screen_share',
+		participantIdentity: 'user_1_connection-a',
+		source: 'screen_share',
+		desired: {enabled: true, quality: 'high', context: 'focused', isIntersecting: true, hasObservedElement: false},
+		actual: {subscribed: null, enabled: null, quality: null, lastCommandAt: null, lastError: null},
+		publication: {available: true, trackSid: 'TR_1', observedAt: 1000},
+		firstFrame: {renderedAt: null},
+		subscribed: true,
+		...overrides,
+	};
+}
+
+describe('summarizeVoiceSubscriptionDebug', () => {
+	it('reports zero issued commands when the client never asked for the stream', () => {
+		const summary = summarizeVoiceSubscriptionDebug([], [participant([publication({})])], [subscription({})], 0);
+		expect(summary.remoteScreenSharePublications).toBe(1);
+		expect(summary.screenShareSubscriptions).toBe(1);
+		expect(summary.screenShareSubscriptionsWithCommandIssued).toBe(0);
+		expect(summary.videoReceiveTransceivers).toBe(0);
+		expect(summary.negotiatedVideoReceiveTransceivers).toBe(0);
+		expect(summary.screenSharePublicationsWithTrack).toBe(0);
+	});
+
+	it('counts a negotiated receive transceiver once the answer applies it', () => {
+		const summary = summarizeVoiceSubscriptionDebug(
+			[transceiver({}), transceiver({mid: '1', currentDirection: null})],
+			[participant([publication({subscribed: true, track: null})])],
+			[subscription({actual: {subscribed: true, enabled: true, quality: 'high', lastCommandAt: 5, lastError: null}})],
+			2,
+		);
+		expect(summary.videoReceiveTransceivers).toBe(2);
+		expect(summary.negotiatedVideoReceiveTransceivers).toBe(1);
+		expect(summary.subscribedScreenSharePublications).toBe(1);
+		expect(summary.screenShareSubscriptionsWithCommandIssued).toBe(1);
+		expect(summary.pendingSubscriptionCommands).toBe(2);
+	});
+
+	it('ignores send-only video, audio receivers, and non-screen-share sources', () => {
+		const summary = summarizeVoiceSubscriptionDebug(
+			[
+				transceiver({direction: 'sendonly', currentDirection: 'sendonly'}),
+				transceiver({
+					mid: '1',
+					receiver: {id: 'audio', kind: 'audio', enabled: true, muted: false, readyState: 'live'},
+				}),
+			],
+			[
+				participant([
+					publication({source: 'camera'}),
+					publication({trackSid: 'TR_2', kind: 'audio', source: 'screen_share_audio'}),
+				]),
+			],
+			[subscription({source: 'camera'})],
+			0,
+		);
+		expect(summary.videoReceiveTransceivers).toBe(0);
+		expect(summary.negotiatedVideoReceiveTransceivers).toBe(0);
+		expect(summary.remoteScreenSharePublications).toBe(0);
+		expect(summary.screenShareSubscriptions).toBe(0);
+	});
+
+	it('counts a rendered first frame separately from an attached track', () => {
+		const summary = summarizeVoiceSubscriptionDebug(
+			[transceiver({})],
+			[
+				participant([
+					publication({
+						subscribed: true,
+						track: {id: 'track', kind: 'video', enabled: true, muted: false, readyState: 'live'},
+					}),
+				]),
+			],
+			[subscription({firstFrame: {renderedAt: 42}})],
+			0,
+		);
+		expect(summary.screenSharePublicationsWithTrack).toBe(1);
+		expect(summary.screenShareSubscriptionsWithFirstFrame).toBe(1);
+	});
+});

fluxer_app/src/features/voice/diagnostics/VoiceSubscriptionDebugSummary.ts

@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {VOICE_MEDIA_GRAPH_SCREEN_SHARE_SOURCE} from '@app/features/voice/engine/VoiceMediaGraph';
+
+export interface VoiceSubscriptionDebugMediaStreamTrack {
+	id: string;
+	kind: string;
+	enabled: boolean;
+	muted: boolean;
+	readyState: string;
+}
+
+export interface VoiceSubscriptionDebugTransceiver {
+	mid: string | null;
+	direction: string;
+	currentDirection: string | null;
+	receiver: VoiceSubscriptionDebugMediaStreamTrack | null;
+	sender: VoiceSubscriptionDebugMediaStreamTrack | null;
+}
+
+export interface VoiceSubscriptionDebugPublication {
+	trackSid: string;
+	trackName: string;
+	kind: string;
+	source: string;
+	mimeType: string | null;
+	dimensions: {width: number; height: number} | null;
+	simulcasted: boolean | null;
+	muted: boolean;
+	enabled: boolean;
+	subscribed: boolean;
+	desired: boolean;
+	subscriptionStatus: string;
+	permissionStatus: string;
+	videoQuality: number | null;
+	track: VoiceSubscriptionDebugMediaStreamTrack | null;
+}
+
+export interface VoiceSubscriptionDebugParticipant {
+	identity: string;
+	sid: string;
+	publications: Array<VoiceSubscriptionDebugPublication>;
+}
+
+export interface VoiceSubscriptionDebugSubscription {
+	key: string;
+	participantIdentity: string;
+	source: string;
+	desired: {
+		enabled: boolean;
+		quality: string;
+		context: string;
+		isIntersecting: boolean;
+		hasObservedElement: boolean;
+	};
+	actual: {
+		subscribed: boolean | null;
+		enabled: boolean | null;
+		quality: string | null;
+		lastCommandAt: number | null;
+		lastError: {code: number; reason: string; at: number} | null;
+	};
+	publication: {available: boolean; trackSid: string | null; observedAt: number | null};
+	firstFrame: {renderedAt: number | null};
+	subscribed: boolean;
+}
+
+export interface VoiceSubscriptionDebugSummary {
+	videoReceiveTransceivers: number;
+	negotiatedVideoReceiveTransceivers: number;
+	remoteScreenSharePublications: number;
+	subscribedScreenSharePublications: number;
+	screenSharePublicationsWithTrack: number;
+	screenShareSubscriptions: number;
+	screenShareSubscriptionsWithCommandIssued: number;
+	screenShareSubscriptionsWithFirstFrame: number;
+	pendingSubscriptionCommands: number;
+}
+
+function isReceiveDirection(direction: string | null): boolean {
+	return direction === 'recvonly' || direction === 'sendrecv';
+}
+
+export function summarizeVoiceSubscriptionDebug(
+	transceivers: ReadonlyArray<VoiceSubscriptionDebugTransceiver>,
+	participants: ReadonlyArray<VoiceSubscriptionDebugParticipant>,
+	subscriptions: ReadonlyArray<VoiceSubscriptionDebugSubscription>,
+	pendingSubscriptionCommands: number,
+): VoiceSubscriptionDebugSummary {
+	const videoTransceivers = transceivers.filter((transceiver) => transceiver.receiver?.kind === 'video');
+	const screenSharePublications = participants
+		.flatMap((participant) => participant.publications)
+		.filter(
+			(publication) => publication.source === VOICE_MEDIA_GRAPH_SCREEN_SHARE_SOURCE && publication.kind === 'video',
+		);
+	const screenShareSubscriptions = subscriptions.filter(
+		(subscription) => subscription.source === VOICE_MEDIA_GRAPH_SCREEN_SHARE_SOURCE,
+	);
+	return {
+		videoReceiveTransceivers: videoTransceivers.filter((transceiver) => isReceiveDirection(transceiver.direction))
+			.length,
+		negotiatedVideoReceiveTransceivers: videoTransceivers.filter((transceiver) =>
+			isReceiveDirection(transceiver.currentDirection),
+		).length,
+		remoteScreenSharePublications: screenSharePublications.length,
+		subscribedScreenSharePublications: screenSharePublications.filter((publication) => publication.subscribed).length,
+		screenSharePublicationsWithTrack: screenSharePublications.filter((publication) => publication.track != null).length,
+		screenShareSubscriptions: screenShareSubscriptions.length,
+		screenShareSubscriptionsWithCommandIssued: screenShareSubscriptions.filter(
+			(subscription) => subscription.actual.lastCommandAt != null,
+		).length,
+		screenShareSubscriptionsWithFirstFrame: screenShareSubscriptions.filter(
+			(subscription) => subscription.firstFrame.renderedAt != null,
+		).length,
+		pendingSubscriptionCommands,
+	};
+}

fluxer_app/src/index.tsx

@@ -30,6 +30,7 @@ import {
 } from '@app/features/platform/utils/ClientInfo';
 import {loadLazyModule} from '@app/features/platform/utils/LazyModuleLoader';
 import {scheduleNonLatinScriptFaces} from '@app/features/theme/fonts/ScriptFontLoader';
+import {installVoiceSubscriptionDebugApi} from '@app/features/voice/diagnostics/VoiceSubscriptionDebugApi';
 import {i18n} from '@lingui/core';
 import {I18nProvider} from '@lingui/react';
 import {configure} from 'mobx';
@@ -48,6 +49,7 @@ if (typeof window !== 'undefined' && window.history) {
 }
 
 installFluxerConfigDebugApi();
+installVoiceSubscriptionDebugApi();
 
 function createRoot(): ReactDOM.Root {
 	const container = document.getElementById('root');