diff --git a/fluxer_app/src/features/voice/components/VoiceParticipantTile.tsx b/fluxer_app/src/features/voice/components/VoiceParticipantTile.tsx index 87e0b9aa6..26886b019 100644 --- a/fluxer_app/src/features/voice/components/VoiceParticipantTile.tsx +++ b/fluxer_app/src/features/voice/components/VoiceParticipantTile.tsx @@ -552,6 +552,7 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({ isPublicationDesired, hasSubscribedVideo: hasSubscribedScreenShareVideo, operationKey: isScreenShareRepublishBuffering ? `republish:${screenSharePublicationMigrationVersion}` : null, + publication, videoRef, }); useStoreVersion(LastFrameSnapshotCache); diff --git a/fluxer_app/src/features/voice/components/useScreenShareWatchFailure.test.tsx b/fluxer_app/src/features/voice/components/useScreenShareWatchFailure.test.tsx new file mode 100644 index 000000000..fd19a681a --- /dev/null +++ b/fluxer_app/src/features/voice/components/useScreenShareWatchFailure.test.tsx @@ -0,0 +1,76 @@ +// @vitest-environment happy-dom +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {useScreenShareWatchFailure} from '@app/features/voice/components/useScreenShareWatchFailure'; +import {clearVideoFrameWatchersForTests} from '@app/features/voice/components/VideoElementFrameState'; +import {voiceMediaGraphStore} from '@app/features/voice/engine/VoiceMediaGraphStore'; +import {ScreenShareWatchFailures} from '@app/features/voice/state/ScreenShareWatchFailures'; +import type {RemoteTrackPublication} from 'livekit-client'; +import {act, useRef} from 'react'; +import {createRoot, type Root} from 'react-dom/client'; +import {afterEach, beforeEach, describe, expect, it} from 'vitest'; + +(globalThis as {IS_REACT_ACT_ENVIRONMENT?: boolean}).IS_REACT_ACT_ENVIRONMENT = true; + +const STREAM_KEY = 'guild-a:channel-a:connection-watch'; + +function WatchProbe({publication}: {publication: RemoteTrackPublication | null}) { + const videoRef = useRef(null); + useScreenShareWatchFailure({ + enabled: true, + streamKey: STREAM_KEY, + participantIdentity: 'user_1_connection-watch', + trackSid: 'TR_watch', + hasPublication: publication != null, + isPublicationDesired: publication != null, + hasSubscribedVideo: publication != null, + publication, + videoRef, + }); + return null; +} + +function watchDeadlineFor(streamKey: string) { + const snapshot = voiceMediaGraphStore.getGraphSnapshot(); + for (const deadline of snapshot.deadlinesByKey.values()) { + if (deadline.kind === 'watchAttempt' && deadline.streamKey === streamKey) return deadline; + } + return null; +} + +describe('useScreenShareWatchFailure', () => { + let root: Root | null = null; + let container: HTMLDivElement | null = null; + + beforeEach(() => { + voiceMediaGraphStore.reset(); + container = document.createElement('div'); + document.body.append(container); + root = createRoot(container); + }); + + afterEach(() => { + act(() => { + root?.unmount(); + }); + root = null; + container?.remove(); + container = null; + clearVideoFrameWatchersForTests(); + }); + + it('keeps the running watch deadline when the publication arrives after the watch started', () => { + ScreenShareWatchFailures.markWatchStarted(STREAM_KEY); + act(() => { + root?.render(); + }); + const deadline = watchDeadlineFor(STREAM_KEY); + expect(deadline).not.toBeNull(); + + act(() => { + root?.render(); + }); + + expect(watchDeadlineFor(STREAM_KEY)).toBe(deadline); + }); +}); diff --git a/fluxer_app/src/features/voice/components/useScreenShareWatchFailure.ts b/fluxer_app/src/features/voice/components/useScreenShareWatchFailure.ts index 9400ada40..58125f4b7 100644 --- a/fluxer_app/src/features/voice/components/useScreenShareWatchFailure.ts +++ b/fluxer_app/src/features/voice/components/useScreenShareWatchFailure.ts @@ -8,6 +8,7 @@ import { type ScreenShareWatchFailure, ScreenShareWatchFailures, } from '@app/features/voice/state/ScreenShareWatchFailures'; +import type {RemoteTrackPublication} from 'livekit-client'; import type React from 'react'; import {useEffect, useMemo} from 'react'; @@ -23,6 +24,7 @@ interface UseScreenShareWatchFailureOptions { isPublicationDesired: boolean; hasSubscribedVideo: boolean; operationKey?: string | number | null; + publication?: RemoteTrackPublication | null; videoRef: React.RefObject; } @@ -69,6 +71,7 @@ export function useScreenShareWatchFailure({ participantSid, trackSid, operationKey, + publication, videoRef, }: UseScreenShareWatchFailureOptions): ScreenShareWatchFailureState { const attemptEnabled = enabled && streamKey !== ''; @@ -101,6 +104,14 @@ export function useScreenShareWatchFailure({ }; }, [attemptEnabled, attemptKey, target]); + useEffect(() => { + if (!attemptEnabled) return; + ScreenShareWatchFailures.setWatchTarget(streamKey, {videoRef, publication}); + return () => { + ScreenShareWatchFailures.clearWatchTarget(streamKey); + }; + }, [attemptEnabled, publication, streamKey, videoRef]); + useEffect(() => { if (!attemptEnabled || !attemptKey) return; if (isOperationBuffering) return; diff --git a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts index 173acbb45..e157b8cdb 100644 --- a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts +++ b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts @@ -515,6 +515,19 @@ class ScreenShareCodecNegotiation { return this.selectedCodec; } + getLocalCodecAdvertisements(): Array { + if (this.localCodecs.length === 0) return buildLocalCodecAdvertisements(); + return [...this.localCodecs]; + } + + getRemoteDecodeCodecsByIdentity(): Record> { + const result: Record> = {}; + for (const [identity, codecs] of this.remoteCodecsByIdentity) { + result[identity] = [...getDecodeSet(codecs)]; + } + return result; + } + setSelectionChangeListener( listener: ((room: Room, codec: VideoCodec, reason: NegotiationReason) => void) | null, ): void { diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.test.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.test.ts index d8d7beb19..e5e7b64bb 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.test.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.test.ts @@ -2,6 +2,7 @@ import { type PerTrackStats, + type TransportInfo, VoiceEngineV2AppStatsHostAdapter, type VoiceEngineV2AppStatsHostAdapterScheduler, } from '@app/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter'; @@ -335,3 +336,124 @@ describe('VoiceEngineV2AppStatsHostAdapter firefox outbound rows', () => { expect(rows[classification.localScreenShareTrackIndex!].mid).toBe('4'); }); }); + +function chromiumWatchedScreenShareReports(): Map { + return new Map([ + [ + 'C1', + { + type: 'codec', + id: 'C1', + mimeType: 'video/H264', + payloadType: 108, + sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f', + }, + ], + [ + 'IT01V', + { + type: 'inbound-rtp', + id: 'IT01V', + kind: 'video', + ssrc: 2001, + mid: '5', + codecId: 'C1', + trackIdentifier: 'screen-share-track', + bytesReceived: 3_500_000, + packetsReceived: 4210, + framesReceived: 0, + framesDecoded: 0, + keyFramesDecoded: 0, + pliCount: 7, + frameWidth: 1920, + frameHeight: 1080, + }, + ], + [ + 'CP1', + { + type: 'candidate-pair', + id: 'CP1', + state: 'succeeded', + nominated: true, + selected: true, + localCandidateId: 'LC1', + remoteCandidateId: 'RC1', + currentRoundTripTime: 0.023, + }, + ], + [ + 'LC1', + { + type: 'local-candidate', + id: 'LC1', + candidateType: 'relay', + protocol: 'udp', + relayProtocol: 'tls', + url: 'turn:turn.fluxer.example:443?transport=tcp', + networkType: 'wifi', + }, + ], + ['RC1', {type: 'remote-candidate', id: 'RC1', candidateType: 'host', protocol: 'udp'}], + [ + 'T1', + {type: 'transport', id: 'T1', dtlsState: 'connected', iceState: 'connected', selectedCandidatePairId: 'CP1'}, + ], + ]); +} + +async function collectWatchedScreenShareStats(): Promise<{ + rows: Array; + transport: TransportInfo | null; +}> { + const scheduler = createScheduler(); + const subscriber = { + getStats() { + return Promise.resolve(chromiumWatchedScreenShareReports()); + }, + }; + const adapter = new VoiceEngineV2AppStatsHostAdapter({now: () => 1000, scheduler}); + adapter.setRoom({ + engine: {pcManager: {subscriber, mode: 'subscriber-primary'}}, + numParticipants: 2, + } as unknown as Room); + adapter.startStatsTracking(); + scheduler.runStatsTick(); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + const rows = adapter.perTrackStats; + const transport = adapter.subscriberTransport; + adapter.cleanup(); + return {rows, transport}; +} + +describe('VoiceEngineV2AppStatsHostAdapter watched screen share rows', () => { + it('collects the inbound packet, byte and frame counters that tell a stalled decode apart from a dead feed', async () => { + const {rows} = await collectWatchedScreenShareStats(); + + expect(rows.map((row) => [row.packetsReceived, row.bytesReceived, row.framesReceived, row.framesDecoded])).toEqual([ + [4210, 3_500_000, 0, 0], + ]); + }); + + it('collects the codec sdp fmtp line of the receive row', async () => { + const {rows} = await collectWatchedScreenShareStats(); + + expect(rows[0].codecSdpFmtpLine).toBe('level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f'); + }); + + it('collects the relay protocol of the selected local candidate, which the local protocol never shows', async () => { + const {transport} = await collectWatchedScreenShareStats(); + + expect(transport?.localProtocol).toBe('udp'); + expect(transport?.localRelayProtocol).toBe('tls'); + expect(transport?.localCandidateUrlHost).toBe('turn.fluxer.example'); + }); + + it('collects the peer connection mode alongside the transport block', async () => { + const {transport} = await collectWatchedScreenShareStats(); + + expect(transport?.peerConnectionMode).toBe('subscriber-primary'); + }); +}); diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.ts index 6d27f3adc..63cc3f3c0 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.ts @@ -59,11 +59,15 @@ export interface PerTrackStats { trackIdentifier?: string; mediaSourceId?: string; codec?: string; + codecSdpFmtpLine?: string; payloadType?: number; bitrateKbps: number; bitrateWindowMs?: number; packetsLost?: number; packetsLossPercent?: number; + packetsReceived?: number; + bytesReceived?: number; + framesReceived?: number; jitterMs?: number; framesPerSecond?: number; sourceFramesPerSecond?: number; @@ -121,6 +125,9 @@ export interface TransportInfo { candidatePairState?: string; localCandidateType?: string; localProtocol?: string; + localRelayProtocol?: string; + localCandidateUrlHost?: string; + peerConnectionMode?: string; localNetworkType?: string; remoteCandidateType?: string; remoteProtocol?: string; @@ -156,6 +163,7 @@ type RoomWithEngine = Room & { pcManager?: { publisher?: StatsSource; subscriber?: StatsSource; + mode?: string; }; }; }; @@ -191,6 +199,7 @@ interface RTCStatsEntry { availableIncomingBitrate?: number; codecId?: string; mimeType?: string; + sdpFmtpLine?: string; payloadType?: number; ssrc?: number; rid?: string; @@ -205,6 +214,7 @@ interface RTCStatsEntry { width?: number; height?: number; frames?: number; + framesReceived?: number; framesEncoded?: number; framesDecoded?: number; framesDropped?: number; @@ -223,6 +233,8 @@ interface RTCStatsEntry { remoteCandidateId?: string; candidateType?: string; protocol?: string; + relayProtocol?: string; + url?: string; networkType?: string; nominated?: boolean; selected?: boolean; @@ -374,6 +386,9 @@ function buildInboundTrackExtras( decoderAcceleration: VideoAccelerationStatus | undefined, ): Partial { return { + packetsReceived: report.packetsReceived, + bytesReceived: report.bytesReceived, + framesReceived: report.framesReceived, keyFramesDecoded: report.keyFramesDecoded, decoderImplementation: report.decoderImplementation, powerEfficientDecoder: report.powerEfficientDecoder, @@ -428,6 +443,7 @@ function buildPerTrackStat(args: { (isOutbound && report.mid ? midToSenderTrackId.get(report.mid) : undefined), mediaSourceId: report.mediaSourceId, codec: codec?.mimeType, + codecSdpFmtpLine: codec?.sdpFmtpLine, payloadType: codec?.payloadType, bitrateKbps: Math.round(bitrate.bitrateKbps), bitrateWindowMs: bitrate.windowMs, @@ -466,10 +482,20 @@ function buildPerTrackStat(args: { }; } +function getCandidateUrlHost(url: string | undefined): string | undefined { + if (!url) return undefined; + const schemeIndex = url.indexOf(':'); + const authority = (schemeIndex === -1 ? url : url.slice(schemeIndex + 1)).split('?')[0]; + const portIndex = authority.lastIndexOf(':'); + const host = portIndex > 0 ? authority.slice(0, portIndex) : authority; + return host || undefined; +} + function buildTransportInfo( activePair: RTCStatsEntry | null, transportReport: RTCStatsEntry | null, reportsById: Map, + peerConnectionMode: string | undefined, ): TransportInfo | null { if (!activePair && !transportReport) return null; const local = activePair?.localCandidateId ? reportsById.get(activePair.localCandidateId) : undefined; @@ -478,6 +504,9 @@ function buildTransportInfo( candidatePairState: activePair?.state, localCandidateType: local?.candidateType, localProtocol: local?.protocol, + localRelayProtocol: local?.relayProtocol, + localCandidateUrlHost: getCandidateUrlHost(local?.url), + peerConnectionMode, localNetworkType: local?.networkType, remoteCandidateType: remote?.candidateType, remoteProtocol: remote?.protocol, @@ -503,6 +532,7 @@ async function collectFromStatsSource( now: number, rtpCounters: Map, activeCounterIds: Set, + peerConnectionMode?: string, ): Promise<{ tracks: Array; rtt: number; @@ -548,7 +578,7 @@ async function collectFromStatsSource( const selectedPair = reportsById.get(transportReport.selectedCandidatePairId); if (selectedPair?.type === 'candidate-pair') activePair = selectedPair; } - const transport = buildTransportInfo(activePair, transportReport, reportsById); + const transport = buildTransportInfo(activePair, transportReport, reportsById, peerConnectionMode); return {tracks, rtt, transport}; } @@ -866,6 +896,7 @@ export class VoiceEngineV2AppStatsHostAdapter extends Store { if (!engine?.pcManager) return; const publisher = engine.pcManager.publisher; const subscriber = engine.pcManager.subscriber; + const peerConnectionMode = engine.pcManager.mode; if (!publisher && !subscriber) return; const now = this.now(); const tracks: Array = []; @@ -875,13 +906,27 @@ export class VoiceEngineV2AppStatsHostAdapter extends Store { let publisherTransport: TransportInfo | null = null; let subscriberTransport: TransportInfo | null = null; if (publisher) { - const result = await collectFromStatsSource(publisher, 'publisher', now, rtpCounters, activeCounterIds); + const result = await collectFromStatsSource( + publisher, + 'publisher', + now, + rtpCounters, + activeCounterIds, + peerConnectionMode, + ); tracks.push(...result.tracks); if (result.rtt > rtt) rtt = result.rtt; publisherTransport = result.transport; } if (subscriber) { - const result = await collectFromStatsSource(subscriber, 'subscriber', now, rtpCounters, activeCounterIds); + const result = await collectFromStatsSource( + subscriber, + 'subscriber', + now, + rtpCounters, + activeCounterIds, + peerConnectionMode, + ); tracks.push(...result.tracks); if (result.rtt > rtt) rtt = result.rtt; subscriberTransport = result.transport; diff --git a/fluxer_app/src/features/voice/state/ScreenShareWatchFailures.test.ts b/fluxer_app/src/features/voice/state/ScreenShareWatchFailures.test.ts new file mode 100644 index 000000000..f0753f343 --- /dev/null +++ b/fluxer_app/src/features/voice/state/ScreenShareWatchFailures.test.ts @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {voiceMediaGraphStore} from '@app/features/voice/engine/VoiceMediaGraphStore'; +import {ScreenShareWatchErrorCode, ScreenShareWatchFailures} from '@app/features/voice/state/ScreenShareWatchFailures'; +import type {RemoteTrackPublication} from 'livekit-client'; +import {beforeEach, describe, expect, it} from 'vitest'; + +function inboundStatsReport(): RTCStatsReport { + return new Map([ + [ + 'IT01V', + { + type: 'inbound-rtp', + id: 'IT01V', + kind: 'video', + codecId: 'C1', + packetsReceived: 4210, + bytesReceived: 3_500_000, + framesReceived: 0, + framesDecoded: 0, + keyFramesDecoded: 0, + framesDropped: 0, + pliCount: 7, + firCount: 0, + nackCount: 2, + freezeCount: 0, + decoderImplementation: 'ExternalDecoder (D3D11VideoDecoder)', + powerEfficientDecoder: true, + frameWidth: 1920, + frameHeight: 1080, + }, + ], + [ + 'C1', + { + type: 'codec', + id: 'C1', + mimeType: 'video/H264', + sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f', + }, + ], + ]) as unknown as RTCStatsReport; +} + +function publicationWithStats(report: RTCStatsReport): RemoteTrackPublication { + return { + videoTrack: { + getRTCStatsReport: () => Promise.resolve(report), + }, + } as unknown as RemoteTrackPublication; +} + +async function settleStatsSnapshot(): Promise { + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); +} + +function fireWatchDeadlines(streamKey: string): void { + const snapshot = voiceMediaGraphStore.getGraphSnapshot(); + const dueAt = voiceMediaGraphStore.nowMs() + 60_000; + for (const [key, deadline] of snapshot.deadlinesByKey) { + if (deadline.streamKey !== streamKey) continue; + voiceMediaGraphStore.transition({type: 'time.deadlineFired', key, at: dueAt}); + } +} + +function historyFor(streamKey: string) { + return ScreenShareWatchFailures.getFailureHistory().filter((record) => record.streamKey === streamKey); +} + +function watchDeadlineFor(streamKey: string) { + const snapshot = voiceMediaGraphStore.getGraphSnapshot(); + for (const deadline of snapshot.deadlinesByKey.values()) { + if (deadline.kind === 'watchAttempt' && deadline.streamKey === streamKey) return deadline; + } + return null; +} + +describe('ScreenShareWatchFailures failure history', () => { + beforeEach(() => { + voiceMediaGraphStore.reset(); + }); + + it('keeps a reported failure after the viewer stops watching', () => { + const streamKey = 'guild-a:channel-a:connection-stop'; + ScreenShareWatchFailures.markWatchStarted(streamKey); + ScreenShareWatchFailures.reportFailure({ + streamKey, + participantIdentity: 'user_1_connection-stop', + trackSid: 'TR_stop', + code: ScreenShareWatchErrorCode.RemoteTrackSubscriptionFailed, + reason: 'remote-track-subscription-failed', + }); + expect(ScreenShareWatchFailures.getFailure({streamKey})).not.toBeNull(); + + ScreenShareWatchFailures.markWatchStopped(streamKey); + + expect(ScreenShareWatchFailures.getFailure({streamKey})).toBeNull(); + expect(historyFor(streamKey).map((record) => [record.code, record.trackSid, record.participantIdentity])).toEqual([ + [ScreenShareWatchErrorCode.RemoteTrackSubscriptionFailed, 'TR_stop', 'user_1_connection-stop'], + ]); + }); + + it('keeps a failure raised by a watch deadline after the viewer stops watching', () => { + const streamKey = 'guild-a:channel-a:connection-deadline'; + const attemptKey = `${streamKey}:1:watch`; + ScreenShareWatchFailures.markWatchStarted(streamKey); + ScreenShareWatchFailures.ensureAttempt({streamKey}, attemptKey); + ScreenShareWatchFailures.setWatchTarget(streamKey, {videoRef: {current: null}}); + + fireWatchDeadlines(streamKey); + + const graphFailure = ScreenShareWatchFailures.getFailure({streamKey}); + expect(graphFailure).not.toBeNull(); + + ScreenShareWatchFailures.markWatchStopped(streamKey); + + expect(ScreenShareWatchFailures.getFailure({streamKey})).toBeNull(); + expect(historyFor(streamKey).map((record) => record.code)).toEqual([graphFailure!.code]); + }); + + it('records the inbound counters of the watched publication alongside the failure', async () => { + const streamKey = 'guild-a:channel-a:connection-counters'; + const attemptKey = `${streamKey}:1:watch`; + ScreenShareWatchFailures.markWatchStarted(streamKey); + ScreenShareWatchFailures.ensureAttempt({streamKey}, attemptKey); + ScreenShareWatchFailures.setWatchTarget(streamKey, { + videoRef: {current: null}, + publication: publicationWithStats(inboundStatsReport()), + }); + + ScreenShareWatchFailures.reportFailure({ + streamKey, + trackSid: 'TR_counters', + code: ScreenShareWatchErrorCode.FirstFrameTimeout, + reason: 'first-frame-timeout', + }); + await settleStatsSnapshot(); + + expect(historyFor(streamKey)[0].inbound).toEqual({ + packetsReceived: 4210, + bytesReceived: 3_500_000, + framesReceived: 0, + framesDecoded: 0, + keyFramesDecoded: 0, + framesDropped: 0, + pliCount: 7, + firCount: 0, + nackCount: 2, + freezeCount: 0, + decoderImplementation: 'ExternalDecoder (D3D11VideoDecoder)', + powerEfficientDecoder: true, + mimeType: 'video/H264', + sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f', + frameWidth: 1920, + frameHeight: 1080, + }); + }); + + it('records the size and readiness of the attached video element', () => { + const streamKey = 'guild-a:channel-a:connection-tile'; + const attemptKey = `${streamKey}:1:watch`; + const video = {readyState: 0, videoWidth: 0, videoHeight: 0, clientWidth: 960, clientHeight: 540}; + ScreenShareWatchFailures.markWatchStarted(streamKey); + ScreenShareWatchFailures.ensureAttempt({streamKey}, attemptKey); + ScreenShareWatchFailures.setWatchTarget(streamKey, { + videoRef: {current: video as unknown as HTMLVideoElement}, + }); + + ScreenShareWatchFailures.reportFailure({ + streamKey, + code: ScreenShareWatchErrorCode.FirstFrameTimeout, + reason: 'first-frame-timeout', + }); + + expect(historyFor(streamKey)[0].tile).toMatchObject({ + hasVideoElement: true, + readyState: 0, + videoWidth: 0, + videoHeight: 0, + clientWidth: 960, + clientHeight: 540, + }); + }); + + it('leaves the running watch deadline alone when the watch target is registered', () => { + const streamKey = 'guild-a:channel-a:connection-target'; + const attemptKey = `${streamKey}:1:watch`; + ScreenShareWatchFailures.markWatchStarted(streamKey); + ScreenShareWatchFailures.ensureAttempt({streamKey}, attemptKey); + const deadline = watchDeadlineFor(streamKey); + expect(deadline).not.toBeNull(); + + ScreenShareWatchFailures.setWatchTarget(streamKey, {videoRef: {current: null}}); + ScreenShareWatchFailures.setWatchTarget(streamKey, { + videoRef: {current: null}, + publication: publicationWithStats(inboundStatsReport()), + }); + + expect(watchDeadlineFor(streamKey)).toBe(deadline); + }); + + it('keeps only the most recent failures once the ring is full', () => { + const streamKey = 'guild-a:channel-a:connection-ring'; + ScreenShareWatchFailures.markWatchStarted(streamKey); + for (let index = 0; index < 20; index += 1) { + ScreenShareWatchFailures.reportFailure({ + streamKey, + trackSid: `TR_ring_${index}`, + code: ScreenShareWatchErrorCode.FirstFrameTimeout, + reason: 'first-frame-timeout', + }); + } + + const history = historyFor(streamKey); + expect(history.length).toBe(16); + expect([history[0].trackSid, history[history.length - 1].trackSid]).toEqual(['TR_ring_4', 'TR_ring_19']); + }); +}); diff --git a/fluxer_app/src/features/voice/state/ScreenShareWatchFailures.ts b/fluxer_app/src/features/voice/state/ScreenShareWatchFailures.ts index ff61a2efb..fc1e2b42a 100644 --- a/fluxer_app/src/features/voice/state/ScreenShareWatchFailures.ts +++ b/fluxer_app/src/features/voice/state/ScreenShareWatchFailures.ts @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +import type {RemoteTrackPublication} from 'livekit-client'; import {runInAction} from 'mobx'; import { selectVoiceMediaGraphAttempt, @@ -15,6 +16,9 @@ import type {ScreenSharePublicationOperation} from '../utils/ScreenShareSubscrip export type {ScreenSharePublicationOperation}; +const FAILURE_HISTORY_LIMIT = 16; +const HARVESTED_FAILURE_ID_LIMIT = 64; + export const ScreenShareWatchErrorCode = { SubscriptionSetSubscribedFailed: -2101, SubscriptionSetEnabledFailed: -2102, @@ -76,6 +80,158 @@ export interface ScreenShareWatchAttempt { generation: number; } +export interface ScreenShareWatchFailureInboundSnapshot { + packetsReceived: number | null; + bytesReceived: number | null; + framesReceived: number | null; + framesDecoded: number | null; + keyFramesDecoded: number | null; + framesDropped: number | null; + pliCount: number | null; + firCount: number | null; + nackCount: number | null; + freezeCount: number | null; + decoderImplementation: string | null; + powerEfficientDecoder: boolean | null; + mimeType: string | null; + sdpFmtpLine: string | null; + frameWidth: number | null; + frameHeight: number | null; +} + +export interface ScreenShareWatchFailureTileSnapshot { + hasVideoElement: boolean; + readyState: number | null; + videoWidth: number | null; + videoHeight: number | null; + clientWidth: number | null; + clientHeight: number | null; + visibilityState: string | null; +} + +export interface ScreenShareWatchFailureRecord { + code: ScreenShareWatchErrorCode; + reason: ScreenShareWatchFailureReason; + reportedAt: number; + streamKey: string | null; + trackSid: string | null; + participantIdentity: string | null; + inbound: ScreenShareWatchFailureInboundSnapshot | null; + tile: ScreenShareWatchFailureTileSnapshot | null; +} + +export interface ScreenShareWatchVideoElementRef { + current: HTMLVideoElement | null; +} + +export interface ScreenShareWatchTarget { + videoRef?: ScreenShareWatchVideoElementRef | null; + publication?: RemoteTrackPublication | null; +} + +interface InboundStatsEntry { + type?: string; + kind?: string; + mediaType?: string; + codecId?: string; + packetsReceived?: number; + bytesReceived?: number; + framesReceived?: number; + framesDecoded?: number; + keyFramesDecoded?: number; + framesDropped?: number; + pliCount?: number; + firCount?: number; + nackCount?: number; + freezeCount?: number; + decoderImplementation?: string; + powerEfficientDecoder?: boolean; + frameWidth?: number; + frameHeight?: number; + mimeType?: string; + sdpFmtpLine?: string; +} + +function numberOrNull(value: number | undefined): number | null { + return typeof value === 'number' && Number.isFinite(value) ? value : null; +} + +function stringOrNull(value: string | undefined): string | null { + return typeof value === 'string' && value !== '' ? value : null; +} + +function findInboundVideoEntry(report: RTCStatsReport): InboundStatsEntry | null { + for (const raw of report.values()) { + const entry = raw as InboundStatsEntry; + if (entry.type !== 'inbound-rtp') continue; + const kind = entry.kind ?? entry.mediaType; + if (kind !== 'video') continue; + return entry; + } + return null; +} + +function buildInboundSnapshot(report: RTCStatsReport): ScreenShareWatchFailureInboundSnapshot | null { + const inbound = findInboundVideoEntry(report); + if (!inbound) return null; + const codec = inbound.codecId ? (report.get(inbound.codecId) as InboundStatsEntry | undefined) : undefined; + return { + packetsReceived: numberOrNull(inbound.packetsReceived), + bytesReceived: numberOrNull(inbound.bytesReceived), + framesReceived: numberOrNull(inbound.framesReceived), + framesDecoded: numberOrNull(inbound.framesDecoded), + keyFramesDecoded: numberOrNull(inbound.keyFramesDecoded), + framesDropped: numberOrNull(inbound.framesDropped), + pliCount: numberOrNull(inbound.pliCount), + firCount: numberOrNull(inbound.firCount), + nackCount: numberOrNull(inbound.nackCount), + freezeCount: numberOrNull(inbound.freezeCount), + decoderImplementation: stringOrNull(inbound.decoderImplementation), + powerEfficientDecoder: typeof inbound.powerEfficientDecoder === 'boolean' ? inbound.powerEfficientDecoder : null, + mimeType: stringOrNull(codec?.mimeType), + sdpFmtpLine: stringOrNull(codec?.sdpFmtpLine), + frameWidth: numberOrNull(inbound.frameWidth), + frameHeight: numberOrNull(inbound.frameHeight), + }; +} + +function buildTileSnapshot(videoRef: ScreenShareWatchVideoElementRef | null | undefined) { + const video = videoRef?.current ?? null; + const visibilityState = typeof document === 'undefined' ? null : document.visibilityState; + if (!video) { + return { + hasVideoElement: false, + readyState: null, + videoWidth: null, + videoHeight: null, + clientWidth: null, + clientHeight: null, + visibilityState, + }; + } + return { + hasVideoElement: true, + readyState: numberOrNull(video.readyState), + videoWidth: numberOrNull(video.videoWidth), + videoHeight: numberOrNull(video.videoHeight), + clientWidth: numberOrNull(video.clientWidth), + clientHeight: numberOrNull(video.clientHeight), + visibilityState, + }; +} + +function readInboundStatsReport(publication: RemoteTrackPublication | null | undefined) { + try { + return publication?.videoTrack?.getRTCStatsReport?.(); + } catch { + return undefined; + } +} + +function failureHistoryId(failure: ScreenShareWatchFailure): string { + return `${failure.code}:${failure.streamKey ?? ''}:${failure.trackSid ?? ''}:${failure.reportedAt}`; +} + export function getScreenShareWatchFailureForPublicationOperation( operation: ScreenSharePublicationOperation, ): Pick { @@ -144,6 +300,11 @@ export function selectScreenShareWatchTimeoutFailureCode({ } class ScreenShareWatchFailuresStore { + private readonly failureHistory: Array = []; + private readonly harvestedFailureIds = new Set(); + private readonly watchTargetsByStreamKey = new Map(); + private graphUnsubscribe: (() => void) | null = null; + private get graph(): VoiceMediaGraphSnapshot { return voiceMediaGraphStore.graph as VoiceMediaGraphSnapshot; } @@ -152,8 +313,71 @@ class ScreenShareWatchFailuresStore { voiceMediaGraphStore.transitionTypedFailure(event); } + private rememberFailure(failure: ScreenShareWatchFailure): void { + const id = failureHistoryId(failure); + if (this.harvestedFailureIds.has(id)) return; + if (this.harvestedFailureIds.size >= HARVESTED_FAILURE_ID_LIMIT) { + const oldest = this.harvestedFailureIds.values().next().value; + if (oldest !== undefined) this.harvestedFailureIds.delete(oldest); + } + this.harvestedFailureIds.add(id); + const target = failure.streamKey ? this.watchTargetsByStreamKey.get(failure.streamKey) : undefined; + const record: ScreenShareWatchFailureRecord = { + code: failure.code, + reason: failure.reason, + reportedAt: failure.reportedAt, + streamKey: failure.streamKey ?? null, + trackSid: failure.trackSid ?? null, + participantIdentity: failure.participantIdentity ?? null, + inbound: null, + tile: buildTileSnapshot(target?.videoRef), + }; + if (this.failureHistory.length >= FAILURE_HISTORY_LIMIT) this.failureHistory.shift(); + this.failureHistory.push(record); + void this.attachInboundSnapshot(record, target?.publication); + } + + private async attachInboundSnapshot( + record: ScreenShareWatchFailureRecord, + publication: RemoteTrackPublication | null | undefined, + ): Promise { + try { + const report = await readInboundStatsReport(publication); + if (!report) return; + record.inbound = buildInboundSnapshot(report); + } catch { + record.inbound = null; + } + } + + private harvestGraphFailures(): void { + for (const failure of this.graph.failuresByKey.values()) { + this.rememberFailure(failure); + } + } + + private ensureGraphSubscription(): void { + if (this.graphUnsubscribe !== null) return; + this.graphUnsubscribe = voiceMediaGraphStore.subscribe(() => { + this.harvestGraphFailures(); + }); + } + + private releaseGraphSubscriptionIfIdle(): void { + if (this.watchTargetsByStreamKey.size > 0) return; + if (this.graphUnsubscribe === null) return; + this.graphUnsubscribe(); + this.graphUnsubscribe = null; + } + + getFailureHistory(): Array { + this.harvestGraphFailures(); + return this.failureHistory.map((record) => ({...record})); + } + markWatchStarted(streamKey: string): number { if (!streamKey) return 0; + this.harvestGraphFailures(); let nextGeneration = 0; runInAction(() => { this.transition({type: 'watch.started', streamKey, at: voiceMediaGraphStore.nowMs()}); @@ -164,6 +388,7 @@ class ScreenShareWatchFailuresStore { markWatchStopped(streamKey: string): void { if (!streamKey) return; + this.harvestGraphFailures(); runInAction(() => { this.transition({type: 'watch.stopped', streamKey}); }); @@ -177,6 +402,19 @@ class ScreenShareWatchFailuresStore { return selectVoiceMediaGraphAttempt(this.graph, streamKey); } + setWatchTarget(streamKey: string, watchTarget: ScreenShareWatchTarget): void { + if (!streamKey) return; + this.watchTargetsByStreamKey.set(streamKey, watchTarget); + this.ensureGraphSubscription(); + } + + clearWatchTarget(streamKey: string): void { + if (!streamKey) return; + this.harvestGraphFailures(); + this.watchTargetsByStreamKey.delete(streamKey); + this.releaseGraphSubscriptionIfIdle(); + } + ensureAttempt( target: ScreenShareWatchFailureTarget & {streamKey: string}, attemptKey: string, @@ -196,12 +434,14 @@ class ScreenShareWatchFailuresStore { releaseAttempt(target: ScreenShareWatchFailureTarget & {streamKey: string}, attemptKey: string): void { if (!target.streamKey || !attemptKey) return; + this.harvestGraphFailures(); runInAction(() => { this.transition({type: 'watch.attemptReleased', streamKey: target.streamKey, attemptKey}); }); } markRenderedVideoFrame(target: ScreenShareWatchFailureTarget & {streamKey: string}, attemptKey: string): void { + this.harvestGraphFailures(); const existingAttempt = selectVoiceMediaGraphAttempt(this.graph, target.streamKey); if (existingAttempt && existingAttempt.attemptKey !== attemptKey) return; const generation = @@ -226,6 +466,7 @@ class ScreenShareWatchFailuresStore { runInAction(() => { this.transition({type: 'failure.reported', failure: normalizedFailure, generation: normalizedFailure.generation}); }); + this.rememberFailure(normalizedFailure); return normalizedFailure; } @@ -238,12 +479,14 @@ class ScreenShareWatchFailuresStore { } clearFailure(target: ScreenShareWatchFailureTarget): void { + this.harvestGraphFailures(); runInAction(() => { this.transition({type: 'failure.cleared', target}); }); } clearAll(): void { + this.harvestGraphFailures(); runInAction(() => { this.transition({type: 'failureWatch.clearAll'}); }); diff --git a/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.test.ts b/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.test.ts new file mode 100644 index 000000000..efd9f6c4d --- /dev/null +++ b/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.test.ts @@ -0,0 +1,164 @@ +// @vitest-environment happy-dom +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {StatsForNerdsData} from '@app/features/voice/utils/VoiceStatsForNerdsPresenter'; +import {describe, expect, it, vi} from 'vitest'; + +const failureRecord = { + code: -2303, + reason: 'first-frame-timeout', + reportedAt: 1000, + streamKey: 'guild-a:channel-a:connection-a', + trackSid: 'TR_screen_share', + participantIdentity: 'user_1_connection-a', + inbound: { + packetsReceived: 4210, + bytesReceived: 3_500_000, + framesReceived: 0, + framesDecoded: 0, + keyFramesDecoded: 0, + framesDropped: 0, + pliCount: 7, + firCount: 0, + nackCount: 2, + freezeCount: 0, + decoderImplementation: 'ExternalDecoder (D3D11VideoDecoder)', + powerEfficientDecoder: true, + mimeType: 'video/H264', + sdpFmtpLine: 'profile-level-id=42e01f', + frameWidth: 1920, + frameHeight: 1080, + }, + tile: { + hasVideoElement: true, + readyState: 0, + videoWidth: 0, + videoHeight: 0, + clientWidth: 960, + clientHeight: 540, + visibilityState: 'visible', + }, +}; + +const localCodecs = [{name: 'h264', type: 'video', payload_type: 108, priority: 900, encode: true, decode: true}]; + +const subscriptionDebugReport = {capturedAt: '2026-09-11T00:00:00.000Z', failures: []}; + +vi.mock('@app/features/app/config/Config', () => ({default: {PUBLIC_BUILD_VERSION: 'test'}})); + +vi.mock('@app/features/devtools/utils/DesktopTroubleshootingUtils', () => ({ + getCachedDesktopTroubleshootingSettings: () => null, + getDesktopTroubleshootingSettings: async () => ({}), +})); + +vi.mock('@app/features/ui/utils/NativeUtils', () => ({ + getElectronAPI: () => null, + getNativePlatform: async () => 'web', + isDesktop: () => false, + supportsDesktopScreenShareAudioCapture: () => false, +})); + +vi.mock('@app/features/voice/diagnostics/VoiceSubscriptionDebugReport', () => ({ + collectVoiceSubscriptionDebugReport: () => subscriptionDebugReport, +})); + +vi.mock('@app/features/voice/engine/MediaEngineFacade', () => ({ + default: { + room: null, + connected: false, + connecting: false, + guildId: null, + channelId: null, + connectionId: null, + voiceServerEndpoint: null, + voiceStats: null, + perTrackStats: [], + statsTimeSeries: [], + publisherTransport: null, + subscriberTransport: null, + }, +})); + +vi.mock('@app/features/voice/engine/ScreenShareCodecNegotiation', () => ({ + default: { + getSelectedCodec: () => 'h264', + getLocalCodecAdvertisements: () => localCodecs, + getRemoteDecodeCodecsByIdentity: () => ({'user_1_connection-a': ['h264', 'vp8']}), + }, + getScreenShareCodecPreferenceOrder: () => ['h264', 'vp8'], +})); + +vi.mock('@app/features/voice/engine/voice_screen_share_manager/NativeEngineAudioTrackPump', () => ({ + getNativeEngineAudioTrackPumpStats: () => null, +})); + +vi.mock('@app/features/voice/engine/voice_screen_share_manager/shared', () => ({ + getPublishedScreenShareMaxBitrateBps: () => null, +})); + +vi.mock('@app/features/voice/state/ScreenShareWatchFailures', () => ({ + ScreenShareWatchFailures: {getFailureHistory: () => [failureRecord]}, +})); + +vi.mock('@app/features/voice/state/VoiceSettings', () => ({ + default: new Proxy({}, {get: () => () => null}), +})); + +vi.mock('@app/features/voice/utils/CodecCapabilityDetector', () => ({ + getCameraPublishCodecPolicy: () => ({primary: 'h264'}), + getCodecCapabilityReport: () => ({}), + getLiveKitSupportedCodecs: () => ['h264', 'vp8'], + selectOptimalScreenShareCodec: () => 'h264', +})); + +vi.mock('@app/features/voice/utils/GpuEncoderCapabilities', () => ({loadGpuEncoderReport: async () => null})); + +vi.mock('@app/features/voice/utils/NativeAudioCaptureBridge', () => ({ + getNativeAudioBridgeStats: () => null, + getNativeAudioCaptureDiagnosticState: () => null, +})); + +vi.mock('@app/features/voice/utils/ScreenShareEnvironment', () => ({getDisplayShareEnvironment: async () => ({})})); + +vi.mock('@app/features/voice/utils/ScreenShareOptions', () => ({ + getScreenShareBitrateBps: () => 2_500_000, + resolveStreamingModeSettings: () => ({resolution: 1080, frameRate: 60}), +})); + +vi.mock('@app/features/voice/utils/VideoDecoderCapabilities', () => ({ + getScreenShareDecodeFailures: () => new Set(['vp9']), +})); + +vi.mock('@app/features/voice/utils/VideoQualityEntitlement', () => ({hasHigherVideoQuality: () => false})); + +vi.mock('@app/features/voice/utils/VoiceStatsForNerdsPresenter', () => ({ + buildVoiceStatsForNerdsPresentation: () => ({}), + collectScreenShareAudioPublicationDiagnostics: () => [], +})); + +const {buildStatsForNerdsCopyPayload} = await import('@app/features/voice/utils/StatsForNerdsCopy'); + +describe('buildStatsForNerdsCopyPayload watch failure diagnostics', () => { + it('copies the screen share failure ring with its codes, track sids and inbound counters', async () => { + const payload = await buildStatsForNerdsCopyPayload({} as StatsForNerdsData); + + expect(payload.screenShareWatchFailures).toEqual([failureRecord]); + }); + + it('copies the screen share negotiation block with the selected codec and decode sets', async () => { + const payload = await buildStatsForNerdsCopyPayload({} as StatsForNerdsData); + + expect(payload.screenShareNegotiation).toEqual({ + selectedCodec: 'h264', + localCodecs, + remoteDecodeCodecsByIdentity: {'user_1_connection-a': ['h264', 'vp8']}, + decodeFailures: ['vp9'], + }); + }); + + it('copies the voice subscription debug report that was only reachable from the debug console', async () => { + const payload = await buildStatsForNerdsCopyPayload({} as StatsForNerdsData); + + expect(payload.voiceSubscriptionDebug).toEqual(subscriptionDebugReport); + }); +}); diff --git a/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts b/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts index 7a2273c1e..fe6cb1a10 100644 --- a/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts +++ b/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts @@ -11,12 +11,14 @@ import { isDesktop, supportsDesktopScreenShareAudioCapture, } from '@app/features/ui/utils/NativeUtils'; +import {collectVoiceSubscriptionDebugReport} from '@app/features/voice/diagnostics/VoiceSubscriptionDebugReport'; import MediaEngine from '@app/features/voice/engine/MediaEngineFacade'; import ScreenShareCodecNegotiation, { getScreenShareCodecPreferenceOrder, } from '@app/features/voice/engine/ScreenShareCodecNegotiation'; import {getNativeEngineAudioTrackPumpStats} from '@app/features/voice/engine/voice_screen_share_manager/NativeEngineAudioTrackPump'; import {getPublishedScreenShareMaxBitrateBps} from '@app/features/voice/engine/voice_screen_share_manager/shared'; +import {ScreenShareWatchFailures} from '@app/features/voice/state/ScreenShareWatchFailures'; import VoiceSettings from '@app/features/voice/state/VoiceSettings'; import { type CodecCapabilityReport, @@ -33,6 +35,7 @@ import { import {getDisplayShareEnvironment} from '@app/features/voice/utils/ScreenShareEnvironment'; import {getRecentScreenShares} from '@app/features/voice/utils/ScreenShareLifecycleLog'; import {getScreenShareBitrateBps, resolveStreamingModeSettings} from '@app/features/voice/utils/ScreenShareOptions'; +import {getScreenShareDecodeFailures} from '@app/features/voice/utils/VideoDecoderCapabilities'; import {hasHigherVideoQuality} from '@app/features/voice/utils/VideoQualityEntitlement'; import { buildVoiceStatsForNerdsPresentation, @@ -55,6 +58,14 @@ function safeError(error: unknown): Record { return {message: String(error)}; } +function safeCollect(collect: () => T): T | {error: Record} { + try { + return collect(); + } catch (error) { + return {error: safeError(error)}; + } +} + async function withTimeout( label: string, task: Promise, @@ -586,6 +597,14 @@ export async function buildStatsForNerdsCopyPayload(data: StatsForNerdsData): Pr voiceSettings, voiceSession: summarizeRoom(), recentScreenShares: getRecentScreenShares(), + screenShareWatchFailures: safeCollect(() => ScreenShareWatchFailures.getFailureHistory()), + screenShareNegotiation: safeCollect(() => ({ + selectedCodec: ScreenShareCodecNegotiation.getSelectedCodec(), + localCodecs: ScreenShareCodecNegotiation.getLocalCodecAdvertisements(), + remoteDecodeCodecsByIdentity: ScreenShareCodecNegotiation.getRemoteDecodeCodecsByIdentity(), + decodeFailures: [...getScreenShareDecodeFailures()], + })), + voiceSubscriptionDebug: safeCollect(() => collectVoiceSubscriptionDebugReport()), desktop, }; }