feat(voice): record watch failures and decode counters (#2666)

This commit is contained in:
Hampus authored and GitHub committed 2026-09-11 13:05:34 +02:00
1 parent 320949a79d
commit 88a5ff9c45
10 files changed
+917 -3

No files matched your search

@@ -552,6 +552,7 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({
isPublicationDesired,
hasSubscribedVideo: hasSubscribedScreenShareVideo,
operationKey: isScreenShareRepublishBuffering ? `republish:${screenSharePublicationMigrationVersion}` : null,
publication,
videoRef,
});
useStoreVersion(LastFrameSnapshotCache);
@@ -0,0 +1,76 @@
// @vitest-environment happy-dom
// SPDX-License-Identifier: AGPL-3.0-or-later
import {useScreenShareWatchFailure} from '@app/features/voice/components/useScreenShareWatchFailure';
import {clearVideoFrameWatchersForTests} from '@app/features/voice/components/VideoElementFrameState';
import {voiceMediaGraphStore} from '@app/features/voice/engine/VoiceMediaGraphStore';
import {ScreenShareWatchFailures} from '@app/features/voice/state/ScreenShareWatchFailures';
import type {RemoteTrackPublication} from 'livekit-client';
import {act, useRef} from 'react';
import {createRoot, type Root} from 'react-dom/client';
import {afterEach, beforeEach, describe, expect, it} from 'vitest';
(globalThis as {IS_REACT_ACT_ENVIRONMENT?: boolean}).IS_REACT_ACT_ENVIRONMENT = true;
const STREAM_KEY = 'guild-a:channel-a:connection-watch';
function WatchProbe({publication}: {publication: RemoteTrackPublication | null}) {
const videoRef = useRef<HTMLVideoElement | null>(null);
useScreenShareWatchFailure({
enabled: true,
streamKey: STREAM_KEY,
participantIdentity: 'user_1_connection-watch',
trackSid: 'TR_watch',
hasPublication: publication != null,
isPublicationDesired: publication != null,
hasSubscribedVideo: publication != null,
publication,
videoRef,
});
return null;
}
function watchDeadlineFor(streamKey: string) {
const snapshot = voiceMediaGraphStore.getGraphSnapshot();
for (const deadline of snapshot.deadlinesByKey.values()) {
if (deadline.kind === 'watchAttempt' && deadline.streamKey === streamKey) return deadline;
}
return null;
}
describe('useScreenShareWatchFailure', () => {
let root: Root | null = null;
let container: HTMLDivElement | null = null;
beforeEach(() => {
voiceMediaGraphStore.reset();
container = document.createElement('div');
document.body.append(container);
root = createRoot(container);
});
afterEach(() => {
act(() => {
root?.unmount();
});
root = null;
container?.remove();
container = null;
clearVideoFrameWatchersForTests();
});
it('keeps the running watch deadline when the publication arrives after the watch started', () => {
ScreenShareWatchFailures.markWatchStarted(STREAM_KEY);
act(() => {
root?.render(<WatchProbe publication={null} />);
});
const deadline = watchDeadlineFor(STREAM_KEY);
expect(deadline).not.toBeNull();
act(() => {
root?.render(<WatchProbe publication={{trackSid: 'TR_watch'} as unknown as RemoteTrackPublication} />);
});
expect(watchDeadlineFor(STREAM_KEY)).toBe(deadline);
});
});
@@ -8,6 +8,7 @@ import {
type ScreenShareWatchFailure,
ScreenShareWatchFailures,
} from '@app/features/voice/state/ScreenShareWatchFailures';
import type {RemoteTrackPublication} from 'livekit-client';
import type React from 'react';
import {useEffect, useMemo} from 'react';
@@ -23,6 +24,7 @@ interface UseScreenShareWatchFailureOptions {
isPublicationDesired: boolean;
hasSubscribedVideo: boolean;
operationKey?: string | number | null;
publication?: RemoteTrackPublication | null;
videoRef: React.RefObject<HTMLVideoElement | null>;
}
@@ -69,6 +71,7 @@ export function useScreenShareWatchFailure({
participantSid,
trackSid,
operationKey,
publication,
videoRef,
}: UseScreenShareWatchFailureOptions): ScreenShareWatchFailureState {
const attemptEnabled = enabled && streamKey !== '';
@@ -101,6 +104,14 @@ export function useScreenShareWatchFailure({
};
}, [attemptEnabled, attemptKey, target]);
useEffect(() => {
if (!attemptEnabled) return;
ScreenShareWatchFailures.setWatchTarget(streamKey, {videoRef, publication});
return () => {
ScreenShareWatchFailures.clearWatchTarget(streamKey);
};
}, [attemptEnabled, publication, streamKey, videoRef]);
useEffect(() => {
if (!attemptEnabled || !attemptKey) return;
if (isOperationBuffering) return;
@@ -515,6 +515,19 @@ class ScreenShareCodecNegotiation {
return this.selectedCodec;
}
getLocalCodecAdvertisements(): Array<FluxerCodecAdvertisement> {
if (this.localCodecs.length === 0) return buildLocalCodecAdvertisements();
return [...this.localCodecs];
}
getRemoteDecodeCodecsByIdentity(): Record<string, Array<VideoCodec>> {
const result: Record<string, Array<VideoCodec>> = {};
for (const [identity, codecs] of this.remoteCodecsByIdentity) {
result[identity] = [...getDecodeSet(codecs)];
}
return result;
}
setSelectionChangeListener(
listener: ((room: Room, codec: VideoCodec, reason: NegotiationReason) => void) | null,
): void {
@@ -2,6 +2,7 @@
import {
type PerTrackStats,
type TransportInfo,
VoiceEngineV2AppStatsHostAdapter,
type VoiceEngineV2AppStatsHostAdapterScheduler,
} from '@app/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter';
@@ -335,3 +336,124 @@ describe('VoiceEngineV2AppStatsHostAdapter firefox outbound rows', () => {
expect(rows[classification.localScreenShareTrackIndex!].mid).toBe('4');
});
});
function chromiumWatchedScreenShareReports(): Map<string, unknown> {
return new Map<string, unknown>([
[
'C1',
{
type: 'codec',
id: 'C1',
mimeType: 'video/H264',
payloadType: 108,
sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
},
],
[
'IT01V',
{
type: 'inbound-rtp',
id: 'IT01V',
kind: 'video',
ssrc: 2001,
mid: '5',
codecId: 'C1',
trackIdentifier: 'screen-share-track',
bytesReceived: 3_500_000,
packetsReceived: 4210,
framesReceived: 0,
framesDecoded: 0,
keyFramesDecoded: 0,
pliCount: 7,
frameWidth: 1920,
frameHeight: 1080,
},
],
[
'CP1',
{
type: 'candidate-pair',
id: 'CP1',
state: 'succeeded',
nominated: true,
selected: true,
localCandidateId: 'LC1',
remoteCandidateId: 'RC1',
currentRoundTripTime: 0.023,
},
],
[
'LC1',
{
type: 'local-candidate',
id: 'LC1',
candidateType: 'relay',
protocol: 'udp',
relayProtocol: 'tls',
url: 'turn:turn.fluxer.example:443?transport=tcp',
networkType: 'wifi',
},
],
['RC1', {type: 'remote-candidate', id: 'RC1', candidateType: 'host', protocol: 'udp'}],
[
'T1',
{type: 'transport', id: 'T1', dtlsState: 'connected', iceState: 'connected', selectedCandidatePairId: 'CP1'},
],
]);
}
async function collectWatchedScreenShareStats(): Promise<{
rows: Array<PerTrackStats>;
transport: TransportInfo | null;
}> {
const scheduler = createScheduler();
const subscriber = {
getStats() {
return Promise.resolve(chromiumWatchedScreenShareReports());
},
};
const adapter = new VoiceEngineV2AppStatsHostAdapter({now: () => 1000, scheduler});
adapter.setRoom({
engine: {pcManager: {subscriber, mode: 'subscriber-primary'}},
numParticipants: 2,
} as unknown as Room);
adapter.startStatsTracking();
scheduler.runStatsTick();
await Promise.resolve();
await Promise.resolve();
await Promise.resolve();
const rows = adapter.perTrackStats;
const transport = adapter.subscriberTransport;
adapter.cleanup();
return {rows, transport};
}
describe('VoiceEngineV2AppStatsHostAdapter watched screen share rows', () => {
it('collects the inbound packet, byte and frame counters that tell a stalled decode apart from a dead feed', async () => {
const {rows} = await collectWatchedScreenShareStats();
expect(rows.map((row) => [row.packetsReceived, row.bytesReceived, row.framesReceived, row.framesDecoded])).toEqual([
[4210, 3_500_000, 0, 0],
]);
});
it('collects the codec sdp fmtp line of the receive row', async () => {
const {rows} = await collectWatchedScreenShareStats();
expect(rows[0].codecSdpFmtpLine).toBe('level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f');
});
it('collects the relay protocol of the selected local candidate, which the local protocol never shows', async () => {
const {transport} = await collectWatchedScreenShareStats();
expect(transport?.localProtocol).toBe('udp');
expect(transport?.localRelayProtocol).toBe('tls');
expect(transport?.localCandidateUrlHost).toBe('turn.fluxer.example');
});
it('collects the peer connection mode alongside the transport block', async () => {
const {transport} = await collectWatchedScreenShareStats();
expect(transport?.peerConnectionMode).toBe('subscriber-primary');
});
});
@@ -59,11 +59,15 @@ export interface PerTrackStats {
trackIdentifier?: string;
mediaSourceId?: string;
codec?: string;
codecSdpFmtpLine?: string;
payloadType?: number;
bitrateKbps: number;
bitrateWindowMs?: number;
packetsLost?: number;
packetsLossPercent?: number;
packetsReceived?: number;
bytesReceived?: number;
framesReceived?: number;
jitterMs?: number;
framesPerSecond?: number;
sourceFramesPerSecond?: number;
@@ -121,6 +125,9 @@ export interface TransportInfo {
candidatePairState?: string;
localCandidateType?: string;
localProtocol?: string;
localRelayProtocol?: string;
localCandidateUrlHost?: string;
peerConnectionMode?: string;
localNetworkType?: string;
remoteCandidateType?: string;
remoteProtocol?: string;
@@ -156,6 +163,7 @@ type RoomWithEngine = Room & {
pcManager?: {
publisher?: StatsSource;
subscriber?: StatsSource;
mode?: string;
};
};
};
@@ -191,6 +199,7 @@ interface RTCStatsEntry {
availableIncomingBitrate?: number;
codecId?: string;
mimeType?: string;
sdpFmtpLine?: string;
payloadType?: number;
ssrc?: number;
rid?: string;
@@ -205,6 +214,7 @@ interface RTCStatsEntry {
width?: number;
height?: number;
frames?: number;
framesReceived?: number;
framesEncoded?: number;
framesDecoded?: number;
framesDropped?: number;
@@ -223,6 +233,8 @@ interface RTCStatsEntry {
remoteCandidateId?: string;
candidateType?: string;
protocol?: string;
relayProtocol?: string;
url?: string;
networkType?: string;
nominated?: boolean;
selected?: boolean;
@@ -374,6 +386,9 @@ function buildInboundTrackExtras(
decoderAcceleration: VideoAccelerationStatus | undefined,
): Partial<PerTrackStats> {
return {
packetsReceived: report.packetsReceived,
bytesReceived: report.bytesReceived,
framesReceived: report.framesReceived,
keyFramesDecoded: report.keyFramesDecoded,
decoderImplementation: report.decoderImplementation,
powerEfficientDecoder: report.powerEfficientDecoder,
@@ -428,6 +443,7 @@ function buildPerTrackStat(args: {
(isOutbound && report.mid ? midToSenderTrackId.get(report.mid) : undefined),
mediaSourceId: report.mediaSourceId,
codec: codec?.mimeType,
codecSdpFmtpLine: codec?.sdpFmtpLine,
payloadType: codec?.payloadType,
bitrateKbps: Math.round(bitrate.bitrateKbps),
bitrateWindowMs: bitrate.windowMs,
@@ -466,10 +482,20 @@ function buildPerTrackStat(args: {
};
}
function getCandidateUrlHost(url: string | undefined): string | undefined {
if (!url) return undefined;
const schemeIndex = url.indexOf(':');
const authority = (schemeIndex === -1 ? url : url.slice(schemeIndex + 1)).split('?')[0];
const portIndex = authority.lastIndexOf(':');
const host = portIndex > 0 ? authority.slice(0, portIndex) : authority;
return host || undefined;
}
function buildTransportInfo(
activePair: RTCStatsEntry | null,
transportReport: RTCStatsEntry | null,
reportsById: Map<string, RTCStatsEntry>,
peerConnectionMode: string | undefined,
): TransportInfo | null {
if (!activePair && !transportReport) return null;
const local = activePair?.localCandidateId ? reportsById.get(activePair.localCandidateId) : undefined;
@@ -478,6 +504,9 @@ function buildTransportInfo(
candidatePairState: activePair?.state,
localCandidateType: local?.candidateType,
localProtocol: local?.protocol,
localRelayProtocol: local?.relayProtocol,
localCandidateUrlHost: getCandidateUrlHost(local?.url),
peerConnectionMode,
localNetworkType: local?.networkType,
remoteCandidateType: remote?.candidateType,
remoteProtocol: remote?.protocol,
@@ -503,6 +532,7 @@ async function collectFromStatsSource(
now: number,
rtpCounters: Map<string, VoiceStatsRtpCounter>,
activeCounterIds: Set<string>,
peerConnectionMode?: string,
): Promise<{
tracks: Array<PerTrackStats>;
rtt: number;
@@ -548,7 +578,7 @@ async function collectFromStatsSource(
const selectedPair = reportsById.get(transportReport.selectedCandidatePairId);
if (selectedPair?.type === 'candidate-pair') activePair = selectedPair;
}
const transport = buildTransportInfo(activePair, transportReport, reportsById);
const transport = buildTransportInfo(activePair, transportReport, reportsById, peerConnectionMode);
return {tracks, rtt, transport};
}
@@ -866,6 +896,7 @@ export class VoiceEngineV2AppStatsHostAdapter extends Store {
if (!engine?.pcManager) return;
const publisher = engine.pcManager.publisher;
const subscriber = engine.pcManager.subscriber;
const peerConnectionMode = engine.pcManager.mode;
if (!publisher && !subscriber) return;
const now = this.now();
const tracks: Array<PerTrackStats> = [];
@@ -875,13 +906,27 @@ export class VoiceEngineV2AppStatsHostAdapter extends Store {
let publisherTransport: TransportInfo | null = null;
let subscriberTransport: TransportInfo | null = null;
if (publisher) {
const result = await collectFromStatsSource(publisher, 'publisher', now, rtpCounters, activeCounterIds);
const result = await collectFromStatsSource(
publisher,
'publisher',
now,
rtpCounters,
activeCounterIds,
peerConnectionMode,
);
tracks.push(...result.tracks);
if (result.rtt > rtt) rtt = result.rtt;
publisherTransport = result.transport;
}
if (subscriber) {
const result = await collectFromStatsSource(subscriber, 'subscriber', now, rtpCounters, activeCounterIds);
const result = await collectFromStatsSource(
subscriber,
'subscriber',
now,
rtpCounters,
activeCounterIds,
peerConnectionMode,
);
tracks.push(...result.tracks);
if (result.rtt > rtt) rtt = result.rtt;
subscriberTransport = result.transport;
@@ -0,0 +1,220 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {voiceMediaGraphStore} from '@app/features/voice/engine/VoiceMediaGraphStore';
import {ScreenShareWatchErrorCode, ScreenShareWatchFailures} from '@app/features/voice/state/ScreenShareWatchFailures';
import type {RemoteTrackPublication} from 'livekit-client';
import {beforeEach, describe, expect, it} from 'vitest';
function inboundStatsReport(): RTCStatsReport {
return new Map<string, unknown>([
[
'IT01V',
{
type: 'inbound-rtp',
id: 'IT01V',
kind: 'video',
codecId: 'C1',
packetsReceived: 4210,
bytesReceived: 3_500_000,
framesReceived: 0,
framesDecoded: 0,
keyFramesDecoded: 0,
framesDropped: 0,
pliCount: 7,
firCount: 0,
nackCount: 2,
freezeCount: 0,
decoderImplementation: 'ExternalDecoder (D3D11VideoDecoder)',
powerEfficientDecoder: true,
frameWidth: 1920,
frameHeight: 1080,
},
],
[
'C1',
{
type: 'codec',
id: 'C1',
mimeType: 'video/H264',
sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
},
],
]) as unknown as RTCStatsReport;
}
function publicationWithStats(report: RTCStatsReport): RemoteTrackPublication {
return {
videoTrack: {
getRTCStatsReport: () => Promise.resolve(report),
},
} as unknown as RemoteTrackPublication;
}
async function settleStatsSnapshot(): Promise<void> {
await Promise.resolve();
await Promise.resolve();
await Promise.resolve();
}
function fireWatchDeadlines(streamKey: string): void {
const snapshot = voiceMediaGraphStore.getGraphSnapshot();
const dueAt = voiceMediaGraphStore.nowMs() + 60_000;
for (const [key, deadline] of snapshot.deadlinesByKey) {
if (deadline.streamKey !== streamKey) continue;
voiceMediaGraphStore.transition({type: 'time.deadlineFired', key, at: dueAt});
}
}
function historyFor(streamKey: string) {
return ScreenShareWatchFailures.getFailureHistory().filter((record) => record.streamKey === streamKey);
}
function watchDeadlineFor(streamKey: string) {
const snapshot = voiceMediaGraphStore.getGraphSnapshot();
for (const deadline of snapshot.deadlinesByKey.values()) {
if (deadline.kind === 'watchAttempt' && deadline.streamKey === streamKey) return deadline;
}
return null;
}
describe('ScreenShareWatchFailures failure history', () => {
beforeEach(() => {
voiceMediaGraphStore.reset();
});
it('keeps a reported failure after the viewer stops watching', () => {
const streamKey = 'guild-a:channel-a:connection-stop';
ScreenShareWatchFailures.markWatchStarted(streamKey);
ScreenShareWatchFailures.reportFailure({
streamKey,
participantIdentity: 'user_1_connection-stop',
trackSid: 'TR_stop',
code: ScreenShareWatchErrorCode.RemoteTrackSubscriptionFailed,
reason: 'remote-track-subscription-failed',
});
expect(ScreenShareWatchFailures.getFailure({streamKey})).not.toBeNull();
ScreenShareWatchFailures.markWatchStopped(streamKey);
expect(ScreenShareWatchFailures.getFailure({streamKey})).toBeNull();
expect(historyFor(streamKey).map((record) => [record.code, record.trackSid, record.participantIdentity])).toEqual([
[ScreenShareWatchErrorCode.RemoteTrackSubscriptionFailed, 'TR_stop', 'user_1_connection-stop'],
]);
});
it('keeps a failure raised by a watch deadline after the viewer stops watching', () => {
const streamKey = 'guild-a:channel-a:connection-deadline';
const attemptKey = `${streamKey}:1:watch`;
ScreenShareWatchFailures.markWatchStarted(streamKey);
ScreenShareWatchFailures.ensureAttempt({streamKey}, attemptKey);
ScreenShareWatchFailures.setWatchTarget(streamKey, {videoRef: {current: null}});
fireWatchDeadlines(streamKey);
const graphFailure = ScreenShareWatchFailures.getFailure({streamKey});
expect(graphFailure).not.toBeNull();
ScreenShareWatchFailures.markWatchStopped(streamKey);
expect(ScreenShareWatchFailures.getFailure({streamKey})).toBeNull();
expect(historyFor(streamKey).map((record) => record.code)).toEqual([graphFailure!.code]);
});
it('records the inbound counters of the watched publication alongside the failure', async () => {
const streamKey = 'guild-a:channel-a:connection-counters';
const attemptKey = `${streamKey}:1:watch`;
ScreenShareWatchFailures.markWatchStarted(streamKey);
ScreenShareWatchFailures.ensureAttempt({streamKey}, attemptKey);
ScreenShareWatchFailures.setWatchTarget(streamKey, {
videoRef: {current: null},
publication: publicationWithStats(inboundStatsReport()),
});
ScreenShareWatchFailures.reportFailure({
streamKey,
trackSid: 'TR_counters',
code: ScreenShareWatchErrorCode.FirstFrameTimeout,
reason: 'first-frame-timeout',
});
await settleStatsSnapshot();
expect(historyFor(streamKey)[0].inbound).toEqual({
packetsReceived: 4210,
bytesReceived: 3_500_000,
framesReceived: 0,
framesDecoded: 0,
keyFramesDecoded: 0,
framesDropped: 0,
pliCount: 7,
firCount: 0,
nackCount: 2,
freezeCount: 0,
decoderImplementation: 'ExternalDecoder (D3D11VideoDecoder)',
powerEfficientDecoder: true,
mimeType: 'video/H264',
sdpFmtpLine: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
frameWidth: 1920,
frameHeight: 1080,
});
});
it('records the size and readiness of the attached video element', () => {
const streamKey = 'guild-a:channel-a:connection-tile';
const attemptKey = `${streamKey}:1:watch`;
const video = {readyState: 0, videoWidth: 0, videoHeight: 0, clientWidth: 960, clientHeight: 540};
ScreenShareWatchFailures.markWatchStarted(streamKey);
ScreenShareWatchFailures.ensureAttempt({streamKey}, attemptKey);
ScreenShareWatchFailures.setWatchTarget(streamKey, {
videoRef: {current: video as unknown as HTMLVideoElement},
});
ScreenShareWatchFailures.reportFailure({
streamKey,
code: ScreenShareWatchErrorCode.FirstFrameTimeout,
reason: 'first-frame-timeout',
});
expect(historyFor(streamKey)[0].tile).toMatchObject({
hasVideoElement: true,
readyState: 0,
videoWidth: 0,
videoHeight: 0,
clientWidth: 960,
clientHeight: 540,
});
});
it('leaves the running watch deadline alone when the watch target is registered', () => {
const streamKey = 'guild-a:channel-a:connection-target';
const attemptKey = `${streamKey}:1:watch`;
ScreenShareWatchFailures.markWatchStarted(streamKey);
ScreenShareWatchFailures.ensureAttempt({streamKey}, attemptKey);
const deadline = watchDeadlineFor(streamKey);
expect(deadline).not.toBeNull();
ScreenShareWatchFailures.setWatchTarget(streamKey, {videoRef: {current: null}});
ScreenShareWatchFailures.setWatchTarget(streamKey, {
videoRef: {current: null},
publication: publicationWithStats(inboundStatsReport()),
});
expect(watchDeadlineFor(streamKey)).toBe(deadline);
});
it('keeps only the most recent failures once the ring is full', () => {
const streamKey = 'guild-a:channel-a:connection-ring';
ScreenShareWatchFailures.markWatchStarted(streamKey);
for (let index = 0; index < 20; index += 1) {
ScreenShareWatchFailures.reportFailure({
streamKey,
trackSid: `TR_ring_${index}`,
code: ScreenShareWatchErrorCode.FirstFrameTimeout,
reason: 'first-frame-timeout',
});
}
const history = historyFor(streamKey);
expect(history.length).toBe(16);
expect([history[0].trackSid, history[history.length - 1].trackSid]).toEqual(['TR_ring_4', 'TR_ring_19']);
});
});
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {RemoteTrackPublication} from 'livekit-client';
import {runInAction} from 'mobx';
import {
selectVoiceMediaGraphAttempt,
@@ -15,6 +16,9 @@ import type {ScreenSharePublicationOperation} from '../utils/ScreenShareSubscrip
export type {ScreenSharePublicationOperation};
const FAILURE_HISTORY_LIMIT = 16;
const HARVESTED_FAILURE_ID_LIMIT = 64;
export const ScreenShareWatchErrorCode = {
SubscriptionSetSubscribedFailed: -2101,
SubscriptionSetEnabledFailed: -2102,
@@ -76,6 +80,158 @@ export interface ScreenShareWatchAttempt {
generation: number;
}
export interface ScreenShareWatchFailureInboundSnapshot {
packetsReceived: number | null;
bytesReceived: number | null;
framesReceived: number | null;
framesDecoded: number | null;
keyFramesDecoded: number | null;
framesDropped: number | null;
pliCount: number | null;
firCount: number | null;
nackCount: number | null;
freezeCount: number | null;
decoderImplementation: string | null;
powerEfficientDecoder: boolean | null;
mimeType: string | null;
sdpFmtpLine: string | null;
frameWidth: number | null;
frameHeight: number | null;
}
export interface ScreenShareWatchFailureTileSnapshot {
hasVideoElement: boolean;
readyState: number | null;
videoWidth: number | null;
videoHeight: number | null;
clientWidth: number | null;
clientHeight: number | null;
visibilityState: string | null;
}
export interface ScreenShareWatchFailureRecord {
code: ScreenShareWatchErrorCode;
reason: ScreenShareWatchFailureReason;
reportedAt: number;
streamKey: string | null;
trackSid: string | null;
participantIdentity: string | null;
inbound: ScreenShareWatchFailureInboundSnapshot | null;
tile: ScreenShareWatchFailureTileSnapshot | null;
}
export interface ScreenShareWatchVideoElementRef {
current: HTMLVideoElement | null;
}
export interface ScreenShareWatchTarget {
videoRef?: ScreenShareWatchVideoElementRef | null;
publication?: RemoteTrackPublication | null;
}
interface InboundStatsEntry {
type?: string;
kind?: string;
mediaType?: string;
codecId?: string;
packetsReceived?: number;
bytesReceived?: number;
framesReceived?: number;
framesDecoded?: number;
keyFramesDecoded?: number;
framesDropped?: number;
pliCount?: number;
firCount?: number;
nackCount?: number;
freezeCount?: number;
decoderImplementation?: string;
powerEfficientDecoder?: boolean;
frameWidth?: number;
frameHeight?: number;
mimeType?: string;
sdpFmtpLine?: string;
}
function numberOrNull(value: number | undefined): number | null {
return typeof value === 'number' && Number.isFinite(value) ? value : null;
}
function stringOrNull(value: string | undefined): string | null {
return typeof value === 'string' && value !== '' ? value : null;
}
function findInboundVideoEntry(report: RTCStatsReport): InboundStatsEntry | null {
for (const raw of report.values()) {
const entry = raw as InboundStatsEntry;
if (entry.type !== 'inbound-rtp') continue;
const kind = entry.kind ?? entry.mediaType;
if (kind !== 'video') continue;
return entry;
}
return null;
}
function buildInboundSnapshot(report: RTCStatsReport): ScreenShareWatchFailureInboundSnapshot | null {
const inbound = findInboundVideoEntry(report);
if (!inbound) return null;
const codec = inbound.codecId ? (report.get(inbound.codecId) as InboundStatsEntry | undefined) : undefined;
return {
packetsReceived: numberOrNull(inbound.packetsReceived),
bytesReceived: numberOrNull(inbound.bytesReceived),
framesReceived: numberOrNull(inbound.framesReceived),
framesDecoded: numberOrNull(inbound.framesDecoded),
keyFramesDecoded: numberOrNull(inbound.keyFramesDecoded),
framesDropped: numberOrNull(inbound.framesDropped),
pliCount: numberOrNull(inbound.pliCount),
firCount: numberOrNull(inbound.firCount),
nackCount: numberOrNull(inbound.nackCount),
freezeCount: numberOrNull(inbound.freezeCount),
decoderImplementation: stringOrNull(inbound.decoderImplementation),
powerEfficientDecoder: typeof inbound.powerEfficientDecoder === 'boolean' ? inbound.powerEfficientDecoder : null,
mimeType: stringOrNull(codec?.mimeType),
sdpFmtpLine: stringOrNull(codec?.sdpFmtpLine),
frameWidth: numberOrNull(inbound.frameWidth),
frameHeight: numberOrNull(inbound.frameHeight),
};
}
function buildTileSnapshot(videoRef: ScreenShareWatchVideoElementRef | null | undefined) {
const video = videoRef?.current ?? null;
const visibilityState = typeof document === 'undefined' ? null : document.visibilityState;
if (!video) {
return {
hasVideoElement: false,
readyState: null,
videoWidth: null,
videoHeight: null,
clientWidth: null,
clientHeight: null,
visibilityState,
};
}
return {
hasVideoElement: true,
readyState: numberOrNull(video.readyState),
videoWidth: numberOrNull(video.videoWidth),
videoHeight: numberOrNull(video.videoHeight),
clientWidth: numberOrNull(video.clientWidth),
clientHeight: numberOrNull(video.clientHeight),
visibilityState,
};
}
function readInboundStatsReport(publication: RemoteTrackPublication | null | undefined) {
try {
return publication?.videoTrack?.getRTCStatsReport?.();
} catch {
return undefined;
}
}
function failureHistoryId(failure: ScreenShareWatchFailure): string {
return `${failure.code}:${failure.streamKey ?? ''}:${failure.trackSid ?? ''}:${failure.reportedAt}`;
}
export function getScreenShareWatchFailureForPublicationOperation(
operation: ScreenSharePublicationOperation,
): Pick<ScreenShareWatchFailure, 'code' | 'reason'> {
@@ -144,6 +300,11 @@ export function selectScreenShareWatchTimeoutFailureCode({
}
class ScreenShareWatchFailuresStore {
private readonly failureHistory: Array<ScreenShareWatchFailureRecord> = [];
private readonly harvestedFailureIds = new Set<string>();
private readonly watchTargetsByStreamKey = new Map<string, ScreenShareWatchTarget>();
private graphUnsubscribe: (() => void) | null = null;
private get graph(): VoiceMediaGraphSnapshot<ScreenShareWatchFailure> {
return voiceMediaGraphStore.graph as VoiceMediaGraphSnapshot<ScreenShareWatchFailure>;
}
@@ -152,8 +313,71 @@ class ScreenShareWatchFailuresStore {
voiceMediaGraphStore.transitionTypedFailure(event);
}
private rememberFailure(failure: ScreenShareWatchFailure): void {
const id = failureHistoryId(failure);
if (this.harvestedFailureIds.has(id)) return;
if (this.harvestedFailureIds.size >= HARVESTED_FAILURE_ID_LIMIT) {
const oldest = this.harvestedFailureIds.values().next().value;
if (oldest !== undefined) this.harvestedFailureIds.delete(oldest);
}
this.harvestedFailureIds.add(id);
const target = failure.streamKey ? this.watchTargetsByStreamKey.get(failure.streamKey) : undefined;
const record: ScreenShareWatchFailureRecord = {
code: failure.code,
reason: failure.reason,
reportedAt: failure.reportedAt,
streamKey: failure.streamKey ?? null,
trackSid: failure.trackSid ?? null,
participantIdentity: failure.participantIdentity ?? null,
inbound: null,
tile: buildTileSnapshot(target?.videoRef),
};
if (this.failureHistory.length >= FAILURE_HISTORY_LIMIT) this.failureHistory.shift();
this.failureHistory.push(record);
void this.attachInboundSnapshot(record, target?.publication);
}
private async attachInboundSnapshot(
record: ScreenShareWatchFailureRecord,
publication: RemoteTrackPublication | null | undefined,
): Promise<void> {
try {
const report = await readInboundStatsReport(publication);
if (!report) return;
record.inbound = buildInboundSnapshot(report);
} catch {
record.inbound = null;
}
}
private harvestGraphFailures(): void {
for (const failure of this.graph.failuresByKey.values()) {
this.rememberFailure(failure);
}
}
private ensureGraphSubscription(): void {
if (this.graphUnsubscribe !== null) return;
this.graphUnsubscribe = voiceMediaGraphStore.subscribe(() => {
this.harvestGraphFailures();
});
}
private releaseGraphSubscriptionIfIdle(): void {
if (this.watchTargetsByStreamKey.size > 0) return;
if (this.graphUnsubscribe === null) return;
this.graphUnsubscribe();
this.graphUnsubscribe = null;
}
getFailureHistory(): Array<ScreenShareWatchFailureRecord> {
this.harvestGraphFailures();
return this.failureHistory.map((record) => ({...record}));
}
markWatchStarted(streamKey: string): number {
if (!streamKey) return 0;
this.harvestGraphFailures();
let nextGeneration = 0;
runInAction(() => {
this.transition({type: 'watch.started', streamKey, at: voiceMediaGraphStore.nowMs()});
@@ -164,6 +388,7 @@ class ScreenShareWatchFailuresStore {
markWatchStopped(streamKey: string): void {
if (!streamKey) return;
this.harvestGraphFailures();
runInAction(() => {
this.transition({type: 'watch.stopped', streamKey});
});
@@ -177,6 +402,19 @@ class ScreenShareWatchFailuresStore {
return selectVoiceMediaGraphAttempt(this.graph, streamKey);
}
setWatchTarget(streamKey: string, watchTarget: ScreenShareWatchTarget): void {
if (!streamKey) return;
this.watchTargetsByStreamKey.set(streamKey, watchTarget);
this.ensureGraphSubscription();
}
clearWatchTarget(streamKey: string): void {
if (!streamKey) return;
this.harvestGraphFailures();
this.watchTargetsByStreamKey.delete(streamKey);
this.releaseGraphSubscriptionIfIdle();
}
ensureAttempt(
target: ScreenShareWatchFailureTarget & {streamKey: string},
attemptKey: string,
@@ -196,12 +434,14 @@ class ScreenShareWatchFailuresStore {
releaseAttempt(target: ScreenShareWatchFailureTarget & {streamKey: string}, attemptKey: string): void {
if (!target.streamKey || !attemptKey) return;
this.harvestGraphFailures();
runInAction(() => {
this.transition({type: 'watch.attemptReleased', streamKey: target.streamKey, attemptKey});
});
}
markRenderedVideoFrame(target: ScreenShareWatchFailureTarget & {streamKey: string}, attemptKey: string): void {
this.harvestGraphFailures();
const existingAttempt = selectVoiceMediaGraphAttempt(this.graph, target.streamKey);
if (existingAttempt && existingAttempt.attemptKey !== attemptKey) return;
const generation =
@@ -226,6 +466,7 @@ class ScreenShareWatchFailuresStore {
runInAction(() => {
this.transition({type: 'failure.reported', failure: normalizedFailure, generation: normalizedFailure.generation});
});
this.rememberFailure(normalizedFailure);
return normalizedFailure;
}
@@ -238,12 +479,14 @@ class ScreenShareWatchFailuresStore {
}
clearFailure(target: ScreenShareWatchFailureTarget): void {
this.harvestGraphFailures();
runInAction(() => {
this.transition({type: 'failure.cleared', target});
});
}
clearAll(): void {
this.harvestGraphFailures();
runInAction(() => {
this.transition({type: 'failureWatch.clearAll'});
});
@@ -0,0 +1,164 @@
// @vitest-environment happy-dom
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {StatsForNerdsData} from '@app/features/voice/utils/VoiceStatsForNerdsPresenter';
import {describe, expect, it, vi} from 'vitest';
const failureRecord = {
code: -2303,
reason: 'first-frame-timeout',
reportedAt: 1000,
streamKey: 'guild-a:channel-a:connection-a',
trackSid: 'TR_screen_share',
participantIdentity: 'user_1_connection-a',
inbound: {
packetsReceived: 4210,
bytesReceived: 3_500_000,
framesReceived: 0,
framesDecoded: 0,
keyFramesDecoded: 0,
framesDropped: 0,
pliCount: 7,
firCount: 0,
nackCount: 2,
freezeCount: 0,
decoderImplementation: 'ExternalDecoder (D3D11VideoDecoder)',
powerEfficientDecoder: true,
mimeType: 'video/H264',
sdpFmtpLine: 'profile-level-id=42e01f',
frameWidth: 1920,
frameHeight: 1080,
},
tile: {
hasVideoElement: true,
readyState: 0,
videoWidth: 0,
videoHeight: 0,
clientWidth: 960,
clientHeight: 540,
visibilityState: 'visible',
},
};
const localCodecs = [{name: 'h264', type: 'video', payload_type: 108, priority: 900, encode: true, decode: true}];
const subscriptionDebugReport = {capturedAt: '2026-09-11T00:00:00.000Z', failures: []};
vi.mock('@app/features/app/config/Config', () => ({default: {PUBLIC_BUILD_VERSION: 'test'}}));
vi.mock('@app/features/devtools/utils/DesktopTroubleshootingUtils', () => ({
getCachedDesktopTroubleshootingSettings: () => null,
getDesktopTroubleshootingSettings: async () => ({}),
}));
vi.mock('@app/features/ui/utils/NativeUtils', () => ({
getElectronAPI: () => null,
getNativePlatform: async () => 'web',
isDesktop: () => false,
supportsDesktopScreenShareAudioCapture: () => false,
}));
vi.mock('@app/features/voice/diagnostics/VoiceSubscriptionDebugReport', () => ({
collectVoiceSubscriptionDebugReport: () => subscriptionDebugReport,
}));
vi.mock('@app/features/voice/engine/MediaEngineFacade', () => ({
default: {
room: null,
connected: false,
connecting: false,
guildId: null,
channelId: null,
connectionId: null,
voiceServerEndpoint: null,
voiceStats: null,
perTrackStats: [],
statsTimeSeries: [],
publisherTransport: null,
subscriberTransport: null,
},
}));
vi.mock('@app/features/voice/engine/ScreenShareCodecNegotiation', () => ({
default: {
getSelectedCodec: () => 'h264',
getLocalCodecAdvertisements: () => localCodecs,
getRemoteDecodeCodecsByIdentity: () => ({'user_1_connection-a': ['h264', 'vp8']}),
},
getScreenShareCodecPreferenceOrder: () => ['h264', 'vp8'],
}));
vi.mock('@app/features/voice/engine/voice_screen_share_manager/NativeEngineAudioTrackPump', () => ({
getNativeEngineAudioTrackPumpStats: () => null,
}));
vi.mock('@app/features/voice/engine/voice_screen_share_manager/shared', () => ({
getPublishedScreenShareMaxBitrateBps: () => null,
}));
vi.mock('@app/features/voice/state/ScreenShareWatchFailures', () => ({
ScreenShareWatchFailures: {getFailureHistory: () => [failureRecord]},
}));
vi.mock('@app/features/voice/state/VoiceSettings', () => ({
default: new Proxy({}, {get: () => () => null}),
}));
vi.mock('@app/features/voice/utils/CodecCapabilityDetector', () => ({
getCameraPublishCodecPolicy: () => ({primary: 'h264'}),
getCodecCapabilityReport: () => ({}),
getLiveKitSupportedCodecs: () => ['h264', 'vp8'],
selectOptimalScreenShareCodec: () => 'h264',
}));
vi.mock('@app/features/voice/utils/GpuEncoderCapabilities', () => ({loadGpuEncoderReport: async () => null}));
vi.mock('@app/features/voice/utils/NativeAudioCaptureBridge', () => ({
getNativeAudioBridgeStats: () => null,
getNativeAudioCaptureDiagnosticState: () => null,
}));
vi.mock('@app/features/voice/utils/ScreenShareEnvironment', () => ({getDisplayShareEnvironment: async () => ({})}));
vi.mock('@app/features/voice/utils/ScreenShareOptions', () => ({
getScreenShareBitrateBps: () => 2_500_000,
resolveStreamingModeSettings: () => ({resolution: 1080, frameRate: 60}),
}));
vi.mock('@app/features/voice/utils/VideoDecoderCapabilities', () => ({
getScreenShareDecodeFailures: () => new Set(['vp9']),
}));
vi.mock('@app/features/voice/utils/VideoQualityEntitlement', () => ({hasHigherVideoQuality: () => false}));
vi.mock('@app/features/voice/utils/VoiceStatsForNerdsPresenter', () => ({
buildVoiceStatsForNerdsPresentation: () => ({}),
collectScreenShareAudioPublicationDiagnostics: () => [],
}));
const {buildStatsForNerdsCopyPayload} = await import('@app/features/voice/utils/StatsForNerdsCopy');
describe('buildStatsForNerdsCopyPayload watch failure diagnostics', () => {
it('copies the screen share failure ring with its codes, track sids and inbound counters', async () => {
const payload = await buildStatsForNerdsCopyPayload({} as StatsForNerdsData);
expect(payload.screenShareWatchFailures).toEqual([failureRecord]);
});
it('copies the screen share negotiation block with the selected codec and decode sets', async () => {
const payload = await buildStatsForNerdsCopyPayload({} as StatsForNerdsData);
expect(payload.screenShareNegotiation).toEqual({
selectedCodec: 'h264',
localCodecs,
remoteDecodeCodecsByIdentity: {'user_1_connection-a': ['h264', 'vp8']},
decodeFailures: ['vp9'],
});
});
it('copies the voice subscription debug report that was only reachable from the debug console', async () => {
const payload = await buildStatsForNerdsCopyPayload({} as StatsForNerdsData);
expect(payload.voiceSubscriptionDebug).toEqual(subscriptionDebugReport);
});
});
@@ -11,12 +11,14 @@ import {
isDesktop,
supportsDesktopScreenShareAudioCapture,
} from '@app/features/ui/utils/NativeUtils';
import {collectVoiceSubscriptionDebugReport} from '@app/features/voice/diagnostics/VoiceSubscriptionDebugReport';
import MediaEngine from '@app/features/voice/engine/MediaEngineFacade';
import ScreenShareCodecNegotiation, {
getScreenShareCodecPreferenceOrder,
} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import {getNativeEngineAudioTrackPumpStats} from '@app/features/voice/engine/voice_screen_share_manager/NativeEngineAudioTrackPump';
import {getPublishedScreenShareMaxBitrateBps} from '@app/features/voice/engine/voice_screen_share_manager/shared';
import {ScreenShareWatchFailures} from '@app/features/voice/state/ScreenShareWatchFailures';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {
type CodecCapabilityReport,
@@ -33,6 +35,7 @@ import {
import {getDisplayShareEnvironment} from '@app/features/voice/utils/ScreenShareEnvironment';
import {getRecentScreenShares} from '@app/features/voice/utils/ScreenShareLifecycleLog';
import {getScreenShareBitrateBps, resolveStreamingModeSettings} from '@app/features/voice/utils/ScreenShareOptions';
import {getScreenShareDecodeFailures} from '@app/features/voice/utils/VideoDecoderCapabilities';
import {hasHigherVideoQuality} from '@app/features/voice/utils/VideoQualityEntitlement';
import {
buildVoiceStatsForNerdsPresentation,
@@ -55,6 +58,14 @@ function safeError(error: unknown): Record<string, unknown> {
return {message: String(error)};
}
function safeCollect<T>(collect: () => T): T | {error: Record<string, unknown>} {
try {
return collect();
} catch (error) {
return {error: safeError(error)};
}
}
async function withTimeout<T>(
label: string,
task: Promise<T>,
@@ -586,6 +597,14 @@ export async function buildStatsForNerdsCopyPayload(data: StatsForNerdsData): Pr
voiceSettings,
voiceSession: summarizeRoom(),
recentScreenShares: getRecentScreenShares(),
screenShareWatchFailures: safeCollect(() => ScreenShareWatchFailures.getFailureHistory()),
screenShareNegotiation: safeCollect(() => ({
selectedCodec: ScreenShareCodecNegotiation.getSelectedCodec(),
localCodecs: ScreenShareCodecNegotiation.getLocalCodecAdvertisements(),
remoteDecodeCodecsByIdentity: ScreenShareCodecNegotiation.getRemoteDecodeCodecsByIdentity(),
decodeFailures: [...getScreenShareDecodeFailures()],
})),
voiceSubscriptionDebug: safeCollect(() => collectVoiceSubscriptionDebugReport()),
desktop,
};
}