feat(voice): record watch failures and decode counters (#2666)

fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts

@@ -515,6 +515,19 @@ class ScreenShareCodecNegotiation {
 		return this.selectedCodec;
 	}
 
+	getLocalCodecAdvertisements(): Array<FluxerCodecAdvertisement> {
+		if (this.localCodecs.length === 0) return buildLocalCodecAdvertisements();
+		return [...this.localCodecs];
+	}
+
+	getRemoteDecodeCodecsByIdentity(): Record<string, Array<VideoCodec>> {
+		const result: Record<string, Array<VideoCodec>> = {};
+		for (const [identity, codecs] of this.remoteCodecsByIdentity) {
+			result[identity] = [...getDecodeSet(codecs)];
+		}
+		return result;
+	}
+
 	setSelectionChangeListener(
 		listener: ((room: Room, codec: VideoCodec, reason: NegotiationReason) => void) | null,
 	): void {

fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.test.ts

@@ -2,6 +2,7 @@
 
 import {
 	type PerTrackStats,
+	type TransportInfo,
 	VoiceEngineV2AppStatsHostAdapter,
 	type VoiceEngineV2AppStatsHostAdapterScheduler,
 } from '@app/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter';
@@ -335,3 +336,124 @@ describe('VoiceEngineV2AppStatsHostAdapter firefox outbound rows', () => {
 		expect(rows[classification.localScreenShareTrackIndex!].mid).toBe('4');
 	});
 });
+
+function chromiumWatchedScreenShareReports(): Map<string, unknown> {
+	return new Map<string, unknown>([
+		[
+			'C1',
+			{
+				type: 'codec',
+				id: 'C1',
+				mimeType: 'video/H264',
+				payloadType: 108,
+				sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
+			},
+		],
+		[
+			'IT01V',
+			{
+				type: 'inbound-rtp',
+				id: 'IT01V',
+				kind: 'video',
+				ssrc: 2001,
+				mid: '5',
+				codecId: 'C1',
+				trackIdentifier: 'screen-share-track',
+				bytesReceived: 3_500_000,
+				packetsReceived: 4210,
+				framesReceived: 0,
+				framesDecoded: 0,
+				keyFramesDecoded: 0,
+				pliCount: 7,
+				frameWidth: 1920,
+				frameHeight: 1080,
+			},
+		],
+		[
+			'CP1',
+			{
+				type: 'candidate-pair',
+				id: 'CP1',
+				state: 'succeeded',
+				nominated: true,
+				selected: true,
+				localCandidateId: 'LC1',
+				remoteCandidateId: 'RC1',
+				currentRoundTripTime: 0.023,
+			},
+		],
+		[
+			'LC1',
+			{
+				type: 'local-candidate',
+				id: 'LC1',
+				candidateType: 'relay',
+				protocol: 'udp',
+				relayProtocol: 'tls',
+				url: 'turn:turn.fluxer.example:443?transport=tcp',
+				networkType: 'wifi',
+			},
+		],
+		['RC1', {type: 'remote-candidate', id: 'RC1', candidateType: 'host', protocol: 'udp'}],
+		[
+			'T1',
+			{type: 'transport', id: 'T1', dtlsState: 'connected', iceState: 'connected', selectedCandidatePairId: 'CP1'},
+		],
+	]);
+}
+
+async function collectWatchedScreenShareStats(): Promise<{
+	rows: Array<PerTrackStats>;
+	transport: TransportInfo | null;
+}> {
+	const scheduler = createScheduler();
+	const subscriber = {
+		getStats() {
+			return Promise.resolve(chromiumWatchedScreenShareReports());
+		},
+	};
+	const adapter = new VoiceEngineV2AppStatsHostAdapter({now: () => 1000, scheduler});
+	adapter.setRoom({
+		engine: {pcManager: {subscriber, mode: 'subscriber-primary'}},
+		numParticipants: 2,
+	} as unknown as Room);
+	adapter.startStatsTracking();
+	scheduler.runStatsTick();
+	await Promise.resolve();
+	await Promise.resolve();
+	await Promise.resolve();
+	const rows = adapter.perTrackStats;
+	const transport = adapter.subscriberTransport;
+	adapter.cleanup();
+	return {rows, transport};
+}
+
+describe('VoiceEngineV2AppStatsHostAdapter watched screen share rows', () => {
+	it('collects the inbound packet, byte and frame counters that tell a stalled decode apart from a dead feed', async () => {
+		const {rows} = await collectWatchedScreenShareStats();
+
+		expect(rows.map((row) => [row.packetsReceived, row.bytesReceived, row.framesReceived, row.framesDecoded])).toEqual([
+			[4210, 3_500_000, 0, 0],
+		]);
+	});
+
+	it('collects the codec sdp fmtp line of the receive row', async () => {
+		const {rows} = await collectWatchedScreenShareStats();
+
+		expect(rows[0].codecSdpFmtpLine).toBe('level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f');
+	});
+
+	it('collects the relay protocol of the selected local candidate, which the local protocol never shows', async () => {
+		const {transport} = await collectWatchedScreenShareStats();
+
+		expect(transport?.localProtocol).toBe('udp');
+		expect(transport?.localRelayProtocol).toBe('tls');
+		expect(transport?.localCandidateUrlHost).toBe('turn.fluxer.example');
+	});
+
+	it('collects the peer connection mode alongside the transport block', async () => {
+		const {transport} = await collectWatchedScreenShareStats();
+
+		expect(transport?.peerConnectionMode).toBe('subscriber-primary');
+	});
+});

fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.ts

@@ -59,11 +59,15 @@ export interface PerTrackStats {
 	trackIdentifier?: string;
 	mediaSourceId?: string;
 	codec?: string;
+	codecSdpFmtpLine?: string;
 	payloadType?: number;
 	bitrateKbps: number;
 	bitrateWindowMs?: number;
 	packetsLost?: number;
 	packetsLossPercent?: number;
+	packetsReceived?: number;
+	bytesReceived?: number;
+	framesReceived?: number;
 	jitterMs?: number;
 	framesPerSecond?: number;
 	sourceFramesPerSecond?: number;
@@ -121,6 +125,9 @@ export interface TransportInfo {
 	candidatePairState?: string;
 	localCandidateType?: string;
 	localProtocol?: string;
+	localRelayProtocol?: string;
+	localCandidateUrlHost?: string;
+	peerConnectionMode?: string;
 	localNetworkType?: string;
 	remoteCandidateType?: string;
 	remoteProtocol?: string;
@@ -156,6 +163,7 @@ type RoomWithEngine = Room & {
 		pcManager?: {
 			publisher?: StatsSource;
 			subscriber?: StatsSource;
+			mode?: string;
 		};
 	};
 };
@@ -191,6 +199,7 @@ interface RTCStatsEntry {
 	availableIncomingBitrate?: number;
 	codecId?: string;
 	mimeType?: string;
+	sdpFmtpLine?: string;
 	payloadType?: number;
 	ssrc?: number;
 	rid?: string;
@@ -205,6 +214,7 @@ interface RTCStatsEntry {
 	width?: number;
 	height?: number;
 	frames?: number;
+	framesReceived?: number;
 	framesEncoded?: number;
 	framesDecoded?: number;
 	framesDropped?: number;
@@ -223,6 +233,8 @@ interface RTCStatsEntry {
 	remoteCandidateId?: string;
 	candidateType?: string;
 	protocol?: string;
+	relayProtocol?: string;
+	url?: string;
 	networkType?: string;
 	nominated?: boolean;
 	selected?: boolean;
@@ -374,6 +386,9 @@ function buildInboundTrackExtras(
 	decoderAcceleration: VideoAccelerationStatus | undefined,
 ): Partial<PerTrackStats> {
 	return {
+		packetsReceived: report.packetsReceived,
+		bytesReceived: report.bytesReceived,
+		framesReceived: report.framesReceived,
 		keyFramesDecoded: report.keyFramesDecoded,
 		decoderImplementation: report.decoderImplementation,
 		powerEfficientDecoder: report.powerEfficientDecoder,
@@ -428,6 +443,7 @@ function buildPerTrackStat(args: {
 			(isOutbound && report.mid ? midToSenderTrackId.get(report.mid) : undefined),
 		mediaSourceId: report.mediaSourceId,
 		codec: codec?.mimeType,
+		codecSdpFmtpLine: codec?.sdpFmtpLine,
 		payloadType: codec?.payloadType,
 		bitrateKbps: Math.round(bitrate.bitrateKbps),
 		bitrateWindowMs: bitrate.windowMs,
@@ -466,10 +482,20 @@ function buildPerTrackStat(args: {
 	};
 }
 
+function getCandidateUrlHost(url: string | undefined): string | undefined {
+	if (!url) return undefined;
+	const schemeIndex = url.indexOf(':');
+	const authority = (schemeIndex === -1 ? url : url.slice(schemeIndex + 1)).split('?')[0];
+	const portIndex = authority.lastIndexOf(':');
+	const host = portIndex > 0 ? authority.slice(0, portIndex) : authority;
+	return host || undefined;
+}
+
 function buildTransportInfo(
 	activePair: RTCStatsEntry | null,
 	transportReport: RTCStatsEntry | null,
 	reportsById: Map<string, RTCStatsEntry>,
+	peerConnectionMode: string | undefined,
 ): TransportInfo | null {
 	if (!activePair && !transportReport) return null;
 	const local = activePair?.localCandidateId ? reportsById.get(activePair.localCandidateId) : undefined;
@@ -478,6 +504,9 @@ function buildTransportInfo(
 		candidatePairState: activePair?.state,
 		localCandidateType: local?.candidateType,
 		localProtocol: local?.protocol,
+		localRelayProtocol: local?.relayProtocol,
+		localCandidateUrlHost: getCandidateUrlHost(local?.url),
+		peerConnectionMode,
 		localNetworkType: local?.networkType,
 		remoteCandidateType: remote?.candidateType,
 		remoteProtocol: remote?.protocol,
@@ -503,6 +532,7 @@ async function collectFromStatsSource(
 	now: number,
 	rtpCounters: Map<string, VoiceStatsRtpCounter>,
 	activeCounterIds: Set<string>,
+	peerConnectionMode?: string,
 ): Promise<{
 	tracks: Array<PerTrackStats>;
 	rtt: number;
@@ -548,7 +578,7 @@ async function collectFromStatsSource(
 		const selectedPair = reportsById.get(transportReport.selectedCandidatePairId);
 		if (selectedPair?.type === 'candidate-pair') activePair = selectedPair;
 	}
-	const transport = buildTransportInfo(activePair, transportReport, reportsById);
+	const transport = buildTransportInfo(activePair, transportReport, reportsById, peerConnectionMode);
 	return {tracks, rtt, transport};
 }
 
@@ -866,6 +896,7 @@ export class VoiceEngineV2AppStatsHostAdapter extends Store {
 		if (!engine?.pcManager) return;
 		const publisher = engine.pcManager.publisher;
 		const subscriber = engine.pcManager.subscriber;
+		const peerConnectionMode = engine.pcManager.mode;
 		if (!publisher && !subscriber) return;
 		const now = this.now();
 		const tracks: Array<PerTrackStats> = [];
@@ -875,13 +906,27 @@ export class VoiceEngineV2AppStatsHostAdapter extends Store {
 		let publisherTransport: TransportInfo | null = null;
 		let subscriberTransport: TransportInfo | null = null;
 		if (publisher) {
-			const result = await collectFromStatsSource(publisher, 'publisher', now, rtpCounters, activeCounterIds);
+			const result = await collectFromStatsSource(
+				publisher,
+				'publisher',
+				now,
+				rtpCounters,
+				activeCounterIds,
+				peerConnectionMode,
+			);
 			tracks.push(...result.tracks);
 			if (result.rtt > rtt) rtt = result.rtt;
 			publisherTransport = result.transport;
 		}
 		if (subscriber) {
-			const result = await collectFromStatsSource(subscriber, 'subscriber', now, rtpCounters, activeCounterIds);
+			const result = await collectFromStatsSource(
+				subscriber,
+				'subscriber',
+				now,
+				rtpCounters,
+				activeCounterIds,
+				peerConnectionMode,
+			);
 			tracks.push(...result.tracks);
 			if (result.rtt > rtt) rtt = result.rtt;
 			subscriberTransport = result.transport;