fix(voice): report real state in voice diagnostics (#2600)

This commit is contained in:
Hampus
2026-09-08 19:16:22 +02:00
committed by GitHub
parent fc2b6b5299
commit ef067f36c6
27 changed files with 1935 additions and 106 deletions
@@ -0,0 +1,101 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {beforeEach, describe, expect, it, vi} from 'vitest';
const refreshMicrophoneFromSettings = vi.fn();
const refreshCameraBackgroundFromSettings = vi.fn();
const refreshCameraCaptureFromSettings = vi.fn();
const refreshScreenShareCodecNegotiationFromSettings = vi.fn();
const settings = {
cameraResolution: 720,
videoDeviceId: 'camera',
screenshareResolution: 1080,
videoFrameRate: 60,
streamingMode: 'smooth',
screenShareContentHint: 'auto',
preferredScreenShareCodec: 'auto',
screenShareAv1OptIn: true,
screenShareHevcOptIn: false,
screenShareEncoderMode: 'auto',
screenShareSoftwareQuality: 'auto',
screenShareScalabilityMode: 'auto',
screenShareBackupCodecMode: 'auto',
openH264Enabled: false,
};
vi.mock('@app/features/voice/engine/MediaEngineFacade', () => ({
default: {
room: null,
refreshMicrophoneFromSettings,
refreshCameraBackgroundFromSettings,
refreshCameraCaptureFromSettings,
refreshScreenShareCodecNegotiationFromSettings,
setScreenShareAudioMuted: vi.fn(),
applyAllLocalAudioPreferences: vi.fn(),
applyLocalInputVolume: vi.fn(),
},
}));
vi.mock('@app/features/voice/engine/ScreenShareUnderperformance', () => ({
noteDeliberateScreenShareQualityChange: vi.fn(),
}));
vi.mock('@app/features/voice/utils/VoiceProcessingProfile', () => ({
getActiveInputDeviceLabel: () => null,
}));
vi.mock('@app/features/voice/state/VoiceSettings', () => ({
default: {
updateSettings: (patch: Record<string, unknown>) => Object.assign(settings, patch),
get cameraResolution() {
return settings.cameraResolution;
},
get videoDeviceId() {
return settings.videoDeviceId;
},
getScreenshareResolution: () => settings.screenshareResolution,
getVideoFrameRate: () => settings.videoFrameRate,
getStreamingMode: () => settings.streamingMode,
getScreenShareContentHint: () => settings.screenShareContentHint,
getPreferredScreenShareCodec: () => settings.preferredScreenShareCodec,
getScreenShareAv1OptIn: () => settings.screenShareAv1OptIn,
getScreenShareHevcOptIn: () => settings.screenShareHevcOptIn,
getScreenShareEncoderMode: () => settings.screenShareEncoderMode,
getScreenShareSoftwareQuality: () => settings.screenShareSoftwareQuality,
getScreenShareScalabilityMode: () => settings.screenShareScalabilityMode,
getScreenShareBackupCodecMode: () => settings.screenShareBackupCodecMode,
getOpenH264Enabled: () => settings.openH264Enabled,
},
}));
const VoiceSettingsCommands = await import('./VoiceSettingsCommands');
describe('screen-share codec settings reactions', () => {
beforeEach(() => {
settings.screenShareAv1OptIn = true;
settings.screenShareHevcOptIn = false;
settings.preferredScreenShareCodec = 'auto';
refreshScreenShareCodecNegotiationFromSettings.mockClear();
});
it('renegotiates the live screen share when the AV1 opt-in is turned off', () => {
VoiceSettingsCommands.update({screenShareAv1OptIn: false});
expect(refreshScreenShareCodecNegotiationFromSettings).toHaveBeenCalledTimes(1);
});
it('renegotiates the live screen share when the HEVC opt-in changes', () => {
VoiceSettingsCommands.update({screenShareHevcOptIn: true});
expect(refreshScreenShareCodecNegotiationFromSettings).toHaveBeenCalledTimes(1);
});
it('renegotiates the live screen share when the preferred codec changes', () => {
VoiceSettingsCommands.update({preferredScreenShareCodec: 'vp9'});
expect(refreshScreenShareCodecNegotiationFromSettings).toHaveBeenCalledTimes(1);
});
it('leaves the negotiation alone for unrelated screen share settings', () => {
VoiceSettingsCommands.update({videoFrameRate: 30});
expect(refreshScreenShareCodecNegotiationFromSettings).not.toHaveBeenCalled();
});
});
@@ -103,6 +103,11 @@ const CAMERA_BACKGROUND_REFRESH_KEYS: Array<keyof VoiceSettingsPatch> = [
'mirrorCamera',
];
const CAMERA_CAPTURE_REFRESH_KEYS: Array<keyof VoiceSettingsPatch> = ['cameraResolution', 'videoDeviceId'];
const SCREEN_SHARE_CODEC_NEGOTIATION_REFRESH_KEYS: Array<keyof VoiceSettingsPatch> = [
'preferredScreenShareCodec',
'screenShareAv1OptIn',
'screenShareHevcOptIn',
];
function refreshMicrophone(): void {
MediaEngine.refreshMicrophoneFromSettings();
}
@@ -115,6 +120,10 @@ function refreshCameraCapture(): void {
MediaEngine.refreshCameraCaptureFromSettings();
}
function refreshScreenShareCodecNegotiation(): void {
MediaEngine.refreshScreenShareCodecNegotiationFromSettings();
}
function shouldRefreshMicrophone(settings: VoiceSettingsPatch): boolean {
return MICROPHONE_REFRESH_KEYS.some((key) => settings[key] !== undefined);
}
@@ -123,6 +132,10 @@ function shouldRefreshCameraBackground(settings: VoiceSettingsPatch): boolean {
return CAMERA_BACKGROUND_REFRESH_KEYS.some((key) => settings[key] !== undefined);
}
function shouldRefreshScreenShareCodecNegotiation(settings: VoiceSettingsPatch): boolean {
return SCREEN_SHARE_CODEC_NEGOTIATION_REFRESH_KEYS.some((key) => settings[key] !== undefined);
}
function readCameraCaptureRefreshValues(): VoiceSettingsPatch {
return {
cameraResolution: VoiceSettings.cameraResolution,
@@ -196,6 +209,9 @@ function applyUpdatedVoiceSettings(
if (shouldRefreshCameraCapture(settings, cameraCaptureBefore, readCameraCaptureRefreshValues())) {
refreshCameraCapture();
}
if (shouldRefreshScreenShareCodecNegotiation(settings)) {
refreshScreenShareCodecNegotiation();
}
}
export function setVoiceProcessingModeForDeviceLabel(label: string, mode: VoiceProcessingMode): void {
@@ -9,7 +9,6 @@ import MediaEngine, {useMediaEngineVersion} from '@app/features/voice/engine/Med
import ScreenShareCodecNegotiation, {
getScreenShareCodecPreferenceOrder,
} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import {getScreenShareAudioPumpDiagnostics} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump';
import {getPublishedScreenShareMaxBitrateBps} from '@app/features/voice/engine/voice_screen_share_manager/shared';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {getNativeAudioCaptureDiagnosticState} from '@app/features/voice/utils/NativeAudioCaptureBridge';
@@ -17,6 +16,7 @@ import {getScreenShareBitrateBps, resolveStreamingModeSettings} from '@app/featu
import {hasHigherVideoQuality} from '@app/features/voice/utils/VideoQualityEntitlement';
import {
buildVoiceStatsForNerdsPresentation,
collectScreenShareAudioPublicationDiagnostics,
type StatsForNerdsData,
} from '@app/features/voice/utils/VoiceStatsForNerdsPresenter';
import type {VoiceEngineV2PerTrackStats, VoiceEngineV2TransportInfo} from '@fluxer/voice_engine_v2';
@@ -216,8 +216,8 @@ export function useStatsForNerds({enabled = true}: UseStatsForNerdsOptions = {})
openH264Enabled: VoiceSettings.getOpenH264Enabled(),
},
screenShareAudioCapture: {
pump: getScreenShareAudioPumpDiagnostics(),
nativeCapture: getNativeAudioCaptureDiagnosticState(),
publications: collectScreenShareAudioPublicationDiagnostics(localParticipant),
},
appInfo: {
appVersion: Config.PUBLIC_BUILD_VERSION ?? 'dev',
@@ -27,6 +27,7 @@ function createStatsData(): StatsForNerdsData {
subscriberTransport: null,
},
localVideo: null,
localVideoLayers: [],
localAudio: null,
localScreenShare: null,
localScreenShareAudio: null,
@@ -68,20 +69,8 @@ function createStatsData(): StatsForNerdsData {
openH264Enabled: true,
},
screenShareAudioCapture: {
pump: {
active: false,
captureId: null,
sampleRate: null,
channels: null,
usesNativeSink: false,
publishStrategy: 'none',
publishedFormatKey: null,
eagerPublish: null,
eagerPublishError: null,
droppedPushFrames: 0,
pendingPushFrames: 0,
},
nativeCapture: {},
publications: [],
},
appInfo: {
appVersion: 'dev',
@@ -111,6 +100,50 @@ function createStatsData(): StatsForNerdsData {
};
}
function createNativeCaptureRecord(): Record<string, unknown> {
return {
armedCapture: null,
activeBridge: {captureId: 'native-audio:live'},
supersededBridge: null,
lastStartedCapture: null,
lastArmFailure: null,
bridgeStats: {
active: true,
bridgeMode: 'generator',
captureId: 'native-audio:live',
startedAt: 1772000000000,
lastFrameAt: 1772000011240,
lastFrameTimestampUs: 11240000,
framesReceived: 1124,
framesDropped: 0,
lateFrameCount: 0,
rebufferCount: 0,
maxFrameArrivalGapMs: 32,
maxFrameTimestampGapMs: 10,
maxPendingFrames: 3,
maxBufferedDurationMs: 30,
lastFramePeak: 0.42,
lastFrameRms: 0.11,
maxFramePeak: 0.71,
maxFrameRms: 0.19,
nonSilentFrameCount: 1118,
prebufferTargetMs: 60,
frameDurationMs: 10,
endReason: null,
endDetail: null,
endedAt: null,
},
endedBridgeCaptures: [],
lifecycleFaults: [],
};
}
function screenShareAudioCaptureSection(html: string): string {
const match = /<section[^>]*><h3[^>]*>screenShareAudioCapture \(native\)<\/h3>[\s\S]*?<\/section>/.exec(html);
expect(match).not.toBeNull();
return match?.[0] ?? '';
}
describe('renderVoiceDebugStatsHtml', () => {
it('renders pure SVG sparklines for key popout metrics', () => {
const html = renderVoiceDebugStatsHtml(createStatsData(), '2026-06-10T18:00:00.000Z');
@@ -124,4 +157,57 @@ describe('renderVoiceDebugStatsHtml', () => {
expect(html).toContain('heapUsedMB sparkline');
expect(html).toContain('mainProcessCpuPercent sparkline');
});
it('does not render a screen-share audio pump section while the native capture is live', () => {
const data = createStatsData();
data.screenShareAudioCapture = {nativeCapture: createNativeCaptureRecord(), publications: []};
const html = renderVoiceDebugStatsHtml(data, '2026-06-10T18:00:00.000Z');
expect(html).toContain('screenShareAudioCapture (native)');
expect(html).toContain('framesReceived');
expect(html).not.toContain('screenShareAudioCapture (pump)');
expect(html).not.toContain('publishStrategy');
expect(html).not.toContain('droppedPushFrames');
expect(html).not.toContain('usesNativeSink');
});
it('renders the screen-share audio capture section identically for Chromium and Firefox dumps', () => {
const chromium = createStatsData();
chromium.screenShareAudioCapture = {nativeCapture: createNativeCaptureRecord(), publications: []};
chromium.system.platform = 'Win32';
chromium.system.userAgent =
'(Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36';
const firefox = createStatsData();
firefox.screenShareAudioCapture = {nativeCapture: createNativeCaptureRecord(), publications: []};
firefox.system.platform = 'Win32';
firefox.system.userAgent = '(Windows NT 10.0; Win64; x64; rv:155.0) Gecko/20100101 Firefox/155.0';
firefox.appInfo.electronVersion = null;
firefox.appInfo.chromiumVersion = null;
firefox.appInfo.hardwareAccelerationEnabled = null;
const chromiumHtml = renderVoiceDebugStatsHtml(chromium, '2026-06-10T18:00:00.000Z');
const firefoxHtml = renderVoiceDebugStatsHtml(firefox, '2026-06-10T18:00:00.000Z');
expect(screenShareAudioCaptureSection(chromiumHtml)).toBe(screenShareAudioCaptureSection(firefoxHtml));
expect(chromiumHtml).not.toContain('screenShareAudioCapture (pump)');
expect(firefoxHtml).not.toContain('screenShareAudioCapture (pump)');
});
it('renders every camera simulcast layer instead of only the projected localVideo row', () => {
const data = createStatsData();
data.localVideoLayers = [
{direction: 'send', kind: 'video', ssrc: 1001, rid: 'q', active: false, bitrateKbps: 0, bitrateWindowMs: 2001},
{direction: 'send', kind: 'video', ssrc: 1002, rid: 'h', active: false, bitrateKbps: 0, bitrateWindowMs: 2001},
{direction: 'send', kind: 'video', ssrc: 1003, rid: 'f', active: true, bitrateKbps: 1133, bitrateWindowMs: 2001},
];
data.localVideo = data.localVideoLayers[0];
const html = renderVoiceDebugStatsHtml(data, '2026-06-10T18:00:00.000Z');
expect(html).toContain('track:localVideoLayers[0]');
expect(html).toContain('track:localVideoLayers[1]');
expect(html).toContain('track:localVideoLayers[2]');
expect(html).toContain('bitrateWindowMs');
});
});
@@ -233,6 +233,10 @@ function TrackTables({data}: TrackTablesProps) {
const tracks: Array<[string, VoiceEngineV2PerTrackStats | null]> = [
['track:localAudio', data.localAudio],
['track:localVideo', data.localVideo],
...data.localVideoLayers.map((layer, index): [string, VoiceEngineV2PerTrackStats | null] => [
`track:localVideoLayers[${index}]`,
layer,
]),
['track:localScreenShare', data.localScreenShare],
['track:localScreenShareAudio', data.localScreenShareAudio],
['track:remoteAudio', data.remoteAudio],
@@ -372,11 +376,6 @@ function VoiceDebugStats({data, generatedAtIso}: VoiceDebugStatsProps) {
record={asStatsRecord(data.screenShareSettings)}
data-flx="voice.diagnostics.voice-debug-stats-html.voice-debug-stats.stats-table--7"
/>
<StatsTable
title="screenShareAudioCapture (pump)"
record={asStatsRecord(data.screenShareAudioCapture.pump)}
data-flx="voice.diagnostics.voice-debug-stats-html.voice-debug-stats.stats-table--8"
/>
<StatsTable
title="screenShareAudioCapture (native)"
record={asStatsRecord(data.screenShareAudioCapture.nativeCapture)}
@@ -749,6 +749,16 @@ class MediaEngineFacade extends Store {
await voiceEngineV2AppMediaExecutionAdapter.refreshCameraCapture();
}
refreshScreenShareCodecNegotiationFromSettings(): void {
void this.refreshScreenShareCodecNegotiationFromCurrentEngine().catch((error) => {
logger.warn('Failed to refresh screen share codec negotiation from settings', {error});
});
}
private async refreshScreenShareCodecNegotiationFromCurrentEngine(): Promise<void> {
await ScreenShareCodecNegotiation.publishLocalCapabilities(this.room, 'manual');
}
private reconcileLocalAudioStateInBackground(reason: string): void {
const previous = this.localAudioReconcileCoalescer;
const next = transitionVoiceLocalAudioReconcileCoalescerSnapshot(previous, {type: 'run.requested', reason});
@@ -1,13 +1,17 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {FluxerCodecAdvertisement} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import type {
FluxerCodecAdvertisement,
FluxerSelectProtocolMessage,
} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import type {HardwareEncodeReport} from '@app/features/voice/utils/GpuEncoderCapabilities';
import {beforeEach, describe, expect, it, vi} from 'vitest';
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
let av1OptIn = false;
let hevcOptIn = false;
let preferredScreenShareCodec = 'auto';
let gpuReport: HardwareEncodeReport | null = null;
let firefoxBrowser = false;
vi.mock('@app/features/voice/state/VoiceSettings', () => ({
default: {
@@ -26,7 +30,7 @@ vi.mock('@app/features/ui/utils/NativeUtils', () => ({
guessPlatform: () => 'windows',
isChromiumBrowser: () => true,
isDesktop: () => true,
isFirefoxBrowser: () => false,
isFirefoxBrowser: () => firefoxBrowser,
}));
vi.mock('@app/features/voice/utils/GpuEncoderCapabilities', () => ({
@@ -48,6 +52,8 @@ vi.mock('@app/features/voice/utils/OpenH264Status', () => ({
}));
vi.mock('@app/features/voice/utils/VideoDecoderCapabilities', () => ({
clearScreenShareDecodeFailures: () => undefined,
getScreenShareDecodeFailures: () => new Set(),
getVideoDecoderExclusionsSync: () => [],
loadVideoDecoderExclusions: async () => [],
}));
@@ -216,6 +222,92 @@ describe('screen-share codec negotiation with the HEVC opt-in off', () => {
});
});
describe('screen-share codec negotiation on Gecko', () => {
beforeEach(() => {
av1OptIn = false;
hevcOptIn = false;
preferredScreenShareCodec = 'auto';
gpuReport = null;
firefoxBrowser = true;
resetCachedCodecCapabilities();
});
afterEach(() => {
firefoxBrowser = false;
resetCachedCodecCapabilities();
});
it('ranks VP8 ahead of H.264 because Gecko always software-encodes H.264', () => {
expect(getScreenShareCodecPreferenceOrder()).toEqual(['vp8', 'h264']);
});
it('negotiates VP8 when both ends can encode and decode VP8 and H.264', () => {
const local = buildLocalCodecAdvertisements();
const remote = [buildLocalCodecAdvertisements()];
expect(computeNegotiatedVideoCodec(local, remote, 0, getScreenShareCodecPreferenceOrder()).codec).toBe('vp8');
});
it('still lets an explicit H.264 pin win over the Gecko order', () => {
expect(getScreenShareCodecPreferenceOrder('h264')[0]).toBe('h264');
});
});
describe('advertising local capabilities over the negotiation topic', () => {
let room: FakeRoom;
beforeEach(() => {
av1OptIn = false;
hevcOptIn = false;
preferredScreenShareCodec = 'auto';
gpuReport = {av1: 'hardware', h265: 'hardware', h264: 'hardware', vp9: 'software', vp8: 'software'};
resetCachedCodecCapabilities();
room = new FakeRoom();
ScreenShareCodecNegotiation.setSelectionChangeListener(null);
});
async function publishAndCaptureSelectProtocol(): Promise<{
payload: Uint8Array;
message: FluxerSelectProtocolMessage;
}> {
Object.defineProperty(globalThis, 'window', {configurable: true, writable: true, value: {}});
try {
ScreenShareCodecNegotiation.bind(room as never);
await ScreenShareCodecNegotiation.publishLocalCapabilities(room as never, 'manual');
} finally {
Reflect.deleteProperty(globalThis, 'window');
}
const payloads = (room.localParticipant.publishData.mock.calls as Array<Array<unknown>>).map(
(call) => call[0] as Uint8Array,
);
const payload = payloads.findLast(
(candidate) => (JSON.parse(new TextDecoder().decode(candidate)) as {op: number}).op === 1,
);
if (!payload) throw new Error('no select-protocol message was published');
return {payload, message: JSON.parse(new TextDecoder().decode(payload)) as FluxerSelectProtocolMessage};
}
it('advertises no experiments because nothing on either side reads them', async () => {
const {message} = await publishAndCaptureSelectProtocol();
expect(message.op).toBe(1);
expect(message.d.experiments).toEqual([]);
ScreenShareCodecNegotiation.dispose();
});
it('keeps the empty experiments list acceptable to the receive-side validator', async () => {
const {payload} = await publishAndCaptureSelectProtocol();
ScreenShareCodecNegotiation.dispose();
const peerRoom = new FakeRoom();
peerRoom.remoteParticipants.set('peer', {identity: 'peer'});
ScreenShareCodecNegotiation.bind(peerRoom as never);
const listener = vi.fn();
ScreenShareCodecNegotiation.setSelectionChangeListener(listener);
peerRoom.emit(RoomEvent.DataReceived, payload, {identity: 'peer'}, undefined, SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC);
await new Promise((resolve) => setTimeout(resolve, 0));
expect(listener).toHaveBeenCalled();
ScreenShareCodecNegotiation.dispose();
});
});
describe('reacting to a mid-session codec selection change', () => {
let room: FakeRoom;
@@ -267,6 +359,47 @@ describe('reacting to a mid-session codec selection change', () => {
ScreenShareCodecNegotiation.dispose();
});
async function withWindowDefined(run: () => Promise<void>): Promise<void> {
Object.defineProperty(globalThis, 'window', {configurable: true, writable: true, value: {}});
try {
await run();
} finally {
Reflect.deleteProperty(globalThis, 'window');
}
}
it('stops reusing a live AV1 selection once the AV1 opt-in is turned off mid-share', async () => {
av1OptIn = true;
resetCachedCodecCapabilities();
await establishBaseline();
expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('av1');
av1OptIn = false;
resetCachedCodecCapabilities();
expect(ScreenShareCodecNegotiation.selectScreenShareCodec('auto')).not.toBe('av1');
expect(ScreenShareCodecNegotiation.selectNativeScreenShareCodec('auto')).not.toBe('av1');
expect(getScreenShareCodecPreferenceOrder()).toContain(ScreenShareCodecNegotiation.selectScreenShareCodec('auto'));
ScreenShareCodecNegotiation.dispose();
});
it('renegotiates the live share off AV1 when the opt-in toggle refreshes the local capabilities', async () => {
av1OptIn = true;
resetCachedCodecCapabilities();
await establishBaseline();
expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('av1');
const listener = vi.fn();
ScreenShareCodecNegotiation.setSelectionChangeListener(listener);
av1OptIn = false;
resetCachedCodecCapabilities();
await withWindowDefined(async () => {
await ScreenShareCodecNegotiation.publishLocalCapabilities(room as never, 'manual');
});
const negotiated = ScreenShareCodecNegotiation.getSelectedCodec();
expect(negotiated).not.toBe('av1');
expect(listener).toHaveBeenCalledTimes(1);
expect(listener).toHaveBeenCalledWith(room, negotiated, 'manual');
ScreenShareCodecNegotiation.dispose();
});
it('does not notify the listener when the recomputed selection is unchanged', async () => {
await establishBaseline();
room.remoteParticipants.set('peer-with-hevc', {identity: 'peer-with-hevc'});
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Logger} from '@app/features/platform/utils/AppLogger';
import {isFirefoxBrowser} from '@app/features/ui/utils/NativeUtils';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {
type CodecCapabilityReport,
@@ -17,6 +18,8 @@ import {loadGpuEncoderReport} from '@app/features/voice/utils/GpuEncoderCapabili
import {loadNativeHardwareEncoderCapabilities} from '@app/features/voice/utils/NativeHardwareEncoderCapabilities';
import {loadOpenH264Status} from '@app/features/voice/utils/OpenH264Status';
import {
clearScreenShareDecodeFailures,
getScreenShareDecodeFailures,
getVideoDecoderExclusionsSync,
loadVideoDecoderExclusions,
} from '@app/features/voice/utils/VideoDecoderCapabilities';
@@ -39,6 +42,7 @@ const RTP_PAYLOAD_TYPE_MAX = 255;
const CODEC_PRIORITY_MAX = 65_535;
const CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['av1', 'h265', 'h264', 'vp9', 'vp8'];
const SOFTWARE_CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['av1', 'vp9', 'h264', 'vp8', 'h265'];
const GECKO_SOFTWARE_CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['vp8', 'h264'];
const COMPATIBILITY_FALLBACK_CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['h264', 'vp9', 'vp8'];
const BASELINE_VIDEO_CODEC: VideoCodec = 'vp8';
const VIDEO_CODEC_NAMES: Record<VideoCodec, FluxerVideoCodecName> = {
@@ -163,6 +167,9 @@ function hasReceiverCapability(codec: VideoCodec): boolean | null {
function getLocalDecodeCapabilities(): Record<VideoCodec, boolean> {
const exclusions = new Set(getVideoDecoderExclusionsSync() ?? []);
for (const codec of getScreenShareDecodeFailures()) {
exclusions.add(codec);
}
const result: Record<VideoCodec, boolean> = {
av1: false,
h265: false,
@@ -172,7 +179,7 @@ function getLocalDecodeCapabilities(): Record<VideoCodec, boolean> {
};
for (const codec of CODEC_PREFERENCE) {
const advertised = hasReceiverCapability(codec);
result[codec] = advertised === null ? result[codec] : advertised && !exclusions.has(codec);
result[codec] = (advertised === null ? result[codec] : advertised) && !exclusions.has(codec);
}
return result;
}
@@ -181,8 +188,11 @@ export function getScreenShareCodecPreferenceOrder(
preference: CodecPreference = VoiceSettings.getPreferredScreenShareCodec(),
): ReadonlyArray<VideoCodec> {
const encoderMode = resolveEffectiveScreenShareEncoderMode(VoiceSettings.getScreenShareEncoderMode());
const automaticOrder =
encoderMode === 'software' ? SOFTWARE_CODEC_PREFERENCE : getHardwareFirstScreenShareCodecPreferenceOrder();
const automaticOrder = isFirefoxBrowser()
? GECKO_SOFTWARE_CODEC_PREFERENCE
: encoderMode === 'software'
? SOFTWARE_CODEC_PREFERENCE
: getHardwareFirstScreenShareCodecPreferenceOrder();
const order =
preference !== 'auto' ? [preference, ...automaticOrder.filter((codec) => codec !== preference)] : automaticOrder;
return order.filter((codec) => isVideoCodecAllowedForPublish(codec));
@@ -537,6 +547,7 @@ class ScreenShareCodecNegotiation {
}
private canUseSelectedCodecForCurrentParticipants(codec: VideoCodec): boolean {
if (!isVideoCodecAllowedForPublish(codec)) return false;
if (!this.canLocalEncode(codec)) return false;
const {knownRemoteCodecs, unknownParticipants} = this.getRemoteCodecInputs();
if (unknownParticipants > 0) return false;
@@ -631,6 +642,7 @@ class ScreenShareCodecNegotiation {
this.remoteCodecsByIdentity.clear();
this.mediaSessionId = createId('media');
this.negotiationSnapshot = createScreenShareCodecNegotiationSnapshot();
clearScreenShareDecodeFailures();
}
async publishLocalCapabilities(
@@ -665,7 +677,7 @@ class ScreenShareCodecNegotiation {
},
codecs: this.localCodecs,
rtc_connection_id: this.rtcConnectionId,
experiments: ['fixed_keyframe_interval', 'maintain_framerate', 'opus_red', 'transport_cc', 'loss_based_bwe_v2'],
experiments: [],
},
};
await this.publishMessage(room, message);
@@ -0,0 +1,166 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {FluxerCodecAdvertisement} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import type {HardwareEncodeReport} from '@app/features/voice/utils/GpuEncoderCapabilities';
import {beforeEach, describe, expect, it, vi} from 'vitest';
const gpuReport: HardwareEncodeReport = {
av1: 'hardware',
h265: 'hardware',
h264: 'hardware',
vp9: 'software',
vp8: 'software',
};
vi.mock('@app/features/voice/state/VoiceSettings', () => ({
default: {
getScreenShareAv1OptIn: () => false,
getScreenShareHevcOptIn: () => false,
getPreferredScreenShareCodec: () => 'auto',
getScreenShareEncoderMode: () => 'auto',
},
}));
vi.mock('@app/features/devtools/utils/DesktopTroubleshootingUtils', () => ({
getCachedDesktopTroubleshootingSettings: () => null,
}));
vi.mock('@app/features/ui/utils/NativeUtils', () => ({
guessPlatform: () => 'windows',
isChromiumBrowser: () => true,
isDesktop: () => true,
isFirefoxBrowser: () => false,
}));
vi.mock('@app/features/voice/utils/GpuEncoderCapabilities', () => ({
getGpuEncoderReportSync: () => gpuReport,
loadGpuEncoderReport: async () => gpuReport,
}));
vi.mock('@app/features/voice/utils/NativeHardwareEncoderCapabilities', () => ({
getNativeHardwareEncoderCapabilitiesSync: () => null,
hasNativeHardwareEncoder: () => false,
resetNativeHardwareEncoderCapabilities: () => undefined,
loadNativeHardwareEncoderCapabilities: async () => null,
}));
vi.mock('@app/features/voice/utils/OpenH264Status', () => ({
getOpenH264StatusSync: () => null,
resetOpenH264Status: () => undefined,
loadOpenH264Status: async () => null,
}));
const VIDEO_CAPABILITIES = {
codecs: [
{mimeType: 'video/VP8'},
{mimeType: 'video/VP9'},
{mimeType: 'video/H264'},
{mimeType: 'video/H265'},
{mimeType: 'video/AV1'},
],
};
Object.defineProperty(globalThis, 'RTCRtpSender', {
configurable: true,
writable: true,
value: {getCapabilities: () => VIDEO_CAPABILITIES},
});
Object.defineProperty(globalThis, 'RTCRtpReceiver', {
configurable: true,
writable: true,
value: {getCapabilities: () => VIDEO_CAPABILITIES},
});
const {
default: ScreenShareCodecNegotiation,
buildLocalCodecAdvertisements,
computeNegotiatedVideoCodec,
} = await import('./ScreenShareCodecNegotiation');
const {findStalledVideoDecoder} = await import('@app/features/voice/utils/ScreenShareCodecDiagnostics');
const {getVideoDecoderExclusionsSync, markScreenShareDecodeFailure, resetVideoDecoderExclusions} = await import(
'@app/features/voice/utils/VideoDecoderCapabilities'
);
const {resetCachedCodecCapabilities} = await import('@app/features/voice/utils/CodecCapabilityDetector');
function stalledScreenShareStats(mimeType: string): RTCStatsReport {
const entries: Array<Record<string, unknown>> = [
{id: 'codec-1', type: 'codec', mimeType},
{
id: 'inbound-1',
type: 'inbound-rtp',
kind: 'video',
codecId: 'codec-1',
packetsReceived: 4200,
bytesReceived: 3_500_000,
framesReceived: 180,
framesDecoded: 0,
},
];
return new Map(entries.map((entry) => [entry.id as string, entry])) as unknown as RTCStatsReport;
}
function decodeAdvertisedFor(name: 'H264' | 'VP8' | 'VP9'): boolean | undefined {
return buildLocalCodecAdvertisements().find((codec) => codec.name === name)?.decode;
}
function publisherThatEncodes(): Array<FluxerCodecAdvertisement> {
return [
{name: 'H264', type: 'video', payload_type: 102, priority: 1, encode: true, decode: true},
{name: 'VP9', type: 'video', payload_type: 109, priority: 2, encode: true, decode: true},
];
}
describe('a stalled H.264 screen share decode', () => {
beforeEach(() => {
resetVideoDecoderExclusions();
resetCachedCodecCapabilities();
ScreenShareCodecNegotiation.dispose();
});
it('is reported as an h264 decode stall by the stats diagnostic', () => {
expect(findStalledVideoDecoder(stalledScreenShareStats('video/H264'))?.codec).toBe('h264');
});
it('withdraws the local H.264 decode advertisement so the publisher renegotiates away from it', () => {
const stall = findStalledVideoDecoder(stalledScreenShareStats('video/H264'));
if (!stall) throw new Error('expected the stats diagnostic to report a stalled decoder');
expect(stall.codec).toBe('h264');
expect(decodeAdvertisedFor('H264')).toBe(true);
expect(
computeNegotiatedVideoCodec(publisherThatEncodes(), [buildLocalCodecAdvertisements()], 0, ['h264', 'vp9']).codec,
).toBe('h264');
expect(markScreenShareDecodeFailure(stall.codec, 'screen-share-decode-stalled')).toBe(true);
expect(decodeAdvertisedFor('H264')).toBe(false);
expect(
computeNegotiatedVideoCodec(publisherThatEncodes(), [buildLocalCodecAdvertisements()], 0, ['h264', 'vp9']).codec,
).toBe('vp9');
});
it('never removes h264 or vp8 from the SDP-level subscriber exclusions', () => {
markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled');
expect(getVideoDecoderExclusionsSync() ?? []).not.toContain('h264');
expect(getVideoDecoderExclusionsSync() ?? []).not.toContain('vp8');
});
it('keeps a non-baseline runtime stall out of the SDP-level subscriber exclusions too', () => {
expect(markScreenShareDecodeFailure('av1', 'screen-share-decode-stalled')).toBe(true);
expect(getVideoDecoderExclusionsSync() ?? []).not.toContain('av1');
});
it('restores the H.264 decode advertisement when the room is torn down', () => {
expect(markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled')).toBe(true);
expect(decodeAdvertisedFor('H264')).toBe(false);
ScreenShareCodecNegotiation.dispose();
expect(decodeAdvertisedFor('H264')).toBe(true);
});
it('stops after one codec change instead of cycling through the remaining codecs', () => {
expect(markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled')).toBe(true);
expect(markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled')).toBe(false);
expect(markScreenShareDecodeFailure('vp8', 'screen-share-decode-stalled')).toBe(false);
expect(decodeAdvertisedFor('VP8')).toBe(true);
});
});
@@ -18,7 +18,7 @@ import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
import ParticipantVolume from '@app/features/voice/state/ParticipantVolume';
import {ScreenShareWatchErrorCode, ScreenShareWatchFailures} from '@app/features/voice/state/ScreenShareWatchFailures';
import {scheduleScreenShareDecoderVerification} from '@app/features/voice/utils/ScreenShareCodecDiagnostics';
import {markVideoDecoderRuntimeFailure} from '@app/features/voice/utils/VideoDecoderCapabilities';
import {markScreenShareDecodeFailure} from '@app/features/voice/utils/VideoDecoderCapabilities';
import {parseVoiceParticipantIdentity} from '@app/features/voice/utils/VoiceParticipantIdentity';
import type {
LocalParticipant,
@@ -266,7 +266,7 @@ export function bindRoomEvents(
screenShareDecoderVerificationTimers.delete(trackSid);
},
(failure) => {
if (!markVideoDecoderRuntimeFailure(failure.codec, 'screen-share-decode-stalled')) return;
if (!markScreenShareDecodeFailure(failure.codec, 'screen-share-decode-stalled')) return;
void ScreenShareCodecNegotiation.publishLocalCapabilities(room, 'manual').catch((error) => {
logger.warn('Failed to publish updated codec capabilities after decode stall', {
error,
@@ -1,31 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
export interface ScreenShareAudioPumpDiagnostics {
active: boolean;
captureId: string | null;
sampleRate: number | null;
channels: number | null;
usesNativeSink: boolean;
publishStrategy: 'eager' | 'lazy' | 'none';
publishedFormatKey: string | null;
eagerPublish: 'succeeded' | 'failed' | 'skipped' | null;
eagerPublishError: string | null;
droppedPushFrames: number;
pendingPushFrames: number;
}
export function getScreenShareAudioPumpDiagnostics(): ScreenShareAudioPumpDiagnostics {
return {
active: false,
captureId: null,
sampleRate: null,
channels: null,
usesNativeSink: false,
publishStrategy: 'none',
publishedFormatKey: null,
eagerPublish: null,
eagerPublishError: null,
droppedPushFrames: 0,
pendingPushFrames: 0,
};
}
@@ -0,0 +1,337 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
type PerTrackStats,
VoiceEngineV2AppStatsHostAdapter,
type VoiceEngineV2AppStatsHostAdapterScheduler,
} from '@app/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter';
import {classifyVoiceEngineV2TrackStats} from '@fluxer/voice_engine_v2';
import type {Room} from 'livekit-client';
import {describe, expect, it} from 'vitest';
interface FakeScheduler extends VoiceEngineV2AppStatsHostAdapterScheduler {
runStatsTick(): void;
}
function createScheduler(): FakeScheduler {
const handlers = new Map<number, () => void>();
return {
setInterval(handler, intervalMs) {
handlers.set(intervalMs, handler);
return intervalMs;
},
clearInterval(handle) {
handlers.delete(handle as number);
},
runStatsTick() {
handlers.get(2000)?.();
},
};
}
function chromiumCameraSimulcastReports(bytesSentByRid: Record<string, number>): Map<string, unknown> {
return new Map<string, unknown>([
[
'SV1',
{
type: 'media-source',
id: 'SV1',
kind: 'video',
trackIdentifier: 'cam-track',
width: 1280,
height: 720,
framesPerSecond: 30,
frames: 1801,
},
],
['C1', {type: 'codec', id: 'C1', mimeType: 'video/H264', payloadType: 108}],
[
'OT01V1',
{
type: 'outbound-rtp',
id: 'OT01V1',
kind: 'video',
ssrc: 1001,
rid: 'q',
mid: '2',
active: false,
mediaSourceId: 'SV1',
codecId: 'C1',
bytesSent: bytesSentByRid.q,
framesEncoded: 1,
targetBitrate: 0,
qualityLimitationReason: 'none',
},
],
[
'OT01V2',
{
type: 'outbound-rtp',
id: 'OT01V2',
kind: 'video',
ssrc: 1002,
rid: 'h',
mid: '2',
active: false,
mediaSourceId: 'SV1',
codecId: 'C1',
bytesSent: bytesSentByRid.h,
framesEncoded: 1,
targetBitrate: 0,
qualityLimitationReason: 'none',
},
],
[
'OT01V3',
{
type: 'outbound-rtp',
id: 'OT01V3',
kind: 'video',
ssrc: 1003,
rid: 'f',
mid: '2',
active: true,
mediaSourceId: 'SV1',
codecId: 'C1',
bytesSent: bytesSentByRid.f,
framesEncoded: 1,
frameWidth: 1280,
frameHeight: 720,
targetBitrate: 1133000,
qualityLimitationReason: 'none',
},
],
]);
}
async function collectChromiumCameraRows(
samples: ReadonlyArray<{now: number; bytesSentByRid: Record<string, number>}>,
): Promise<Array<PerTrackStats>> {
const scheduler = createScheduler();
let sampleIndex = 0;
let now = samples[0].now;
const publisher = {
getStats() {
return Promise.resolve(chromiumCameraSimulcastReports(samples[sampleIndex].bytesSentByRid));
},
};
const adapter = new VoiceEngineV2AppStatsHostAdapter({now: () => now, scheduler});
adapter.setRoom({engine: {pcManager: {publisher}}, numParticipants: 2} as unknown as Room);
adapter.startStatsTracking();
for (sampleIndex = 0; sampleIndex < samples.length; sampleIndex += 1) {
now = samples[sampleIndex].now;
scheduler.runStatsTick();
await Promise.resolve();
await Promise.resolve();
await Promise.resolve();
}
const rows = adapter.perTrackStats;
adapter.cleanup();
return rows;
}
describe('VoiceEngineV2AppStatsHostAdapter chromium outbound rows', () => {
it('collects the outbound-rtp active flag for every simulcast layer', async () => {
const rows = await collectChromiumCameraRows([{now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}}]);
expect(rows.map((row) => row.rid)).toEqual(['q', 'h', 'f']);
expect(rows.map((row) => row.active)).toEqual([false, false, true]);
});
it('omits the bitrate window on a first observation so a zero bitrate is not read as measured', async () => {
const rows = await collectChromiumCameraRows([{now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}}]);
expect(rows.map((row) => row.bitrateKbps)).toEqual([0, 0, 0]);
expect(rows.map((row) => row.bitrateWindowMs)).toEqual([undefined, undefined, undefined]);
});
it('reports the bitrate window once a delta was measured so a zero bitrate is falsifiable', async () => {
const rows = await collectChromiumCameraRows([
{now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}},
{now: 3001, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}},
]);
expect(rows.map((row) => row.bitrateKbps)).toEqual([0, 0, 0]);
expect(rows.map((row) => row.bitrateWindowMs)).toEqual([2001, 2001, 2001]);
});
it('keeps every pre-existing chromium row field byte-identical', async () => {
const rows = await collectChromiumCameraRows([
{now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}},
{now: 3000, bytesSentByRid: {q: 0, h: 0, f: 5_500_000}},
]);
const {active, bitrateWindowMs, ...legacy} = rows[2];
expect(legacy).toEqual({
direction: 'send',
kind: 'video',
ssrc: 1003,
rid: 'f',
mid: '2',
trackIdentifier: 'cam-track',
mediaSourceId: 'SV1',
codec: 'video/H264',
payloadType: 108,
bitrateKbps: 2000,
framesEncoded: 1,
frameWidth: 1280,
frameHeight: 720,
sourceFrameWidth: 1280,
sourceFrameHeight: 720,
sourceFramesPerSecond: 30,
sourceFrames: 1801,
targetBitrateKbps: 1133,
qualityLimitationReason: 'none',
encoderAcceleration: 'unknown',
});
});
});
function firefoxCameraAndScreenShareReports(): Map<string, unknown> {
return new Map<string, unknown>([
['C1', {type: 'codec', id: 'C1', mimeType: 'video/H264', payloadType: 108}],
['C2', {type: 'codec', id: 'C2', mimeType: 'audio/opus', payloadType: 109}],
[
'outbound_rtp_audio_1',
{
type: 'outbound-rtp',
id: 'outbound_rtp_audio_1',
kind: 'audio',
ssrc: 724915693,
mid: '1',
codecId: 'C2',
bytesSent: 100_000,
},
],
[
'outbound_rtp_video_3',
{
type: 'outbound-rtp',
id: 'outbound_rtp_video_3',
kind: 'video',
ssrc: 4089824766,
mid: '3',
codecId: 'C1',
bytesSent: 5_000_000,
framesEncoded: 900,
},
],
[
'outbound_rtp_video_4',
{
type: 'outbound-rtp',
id: 'outbound_rtp_video_4',
kind: 'video',
ssrc: 1477030229,
mid: '4',
codecId: 'C1',
bytesSent: 2_000_000,
framesEncoded: 600,
},
],
[
'inbound_rtp_audio_2',
{
type: 'inbound-rtp',
id: 'inbound_rtp_audio_2',
kind: 'audio',
ssrc: 1661789229,
mid: '2',
codecId: 'C2',
trackIdentifier: '{8a106956-d202-4916-8e19-0a1672c64b73}',
bytesReceived: 50_000,
},
],
]);
}
async function collectFirefoxRows(
transceivers: ReadonlyArray<{mid: string | null; sender?: {track?: {id: string} | null} | null}> | null,
): Promise<Array<PerTrackStats>> {
const scheduler = createScheduler();
const publisher = {
getStats() {
return Promise.resolve(firefoxCameraAndScreenShareReports());
},
...(transceivers === null ? {} : {getTransceivers: () => transceivers}),
};
const adapter = new VoiceEngineV2AppStatsHostAdapter({now: () => 1000, scheduler});
adapter.setRoom({engine: {pcManager: {publisher}}, numParticipants: 2} as unknown as Room);
adapter.startStatsTracking();
scheduler.runStatsTick();
await Promise.resolve();
await Promise.resolve();
await Promise.resolve();
const rows = adapter.perTrackStats;
adapter.cleanup();
return rows;
}
describe('VoiceEngineV2AppStatsHostAdapter firefox outbound rows', () => {
it('backfills the outbound trackIdentifier from the transceiver mid when the report carries neither identity member', async () => {
const rows = await collectFirefoxRows([
{mid: '1', sender: {track: {id: '{mic-track}'}}},
{mid: '2', sender: {track: null}},
{mid: '3', sender: {track: {id: '{camera-track}'}}},
{mid: '4', sender: {track: {id: '{screen-track}'}}},
]);
expect(rows.map((row) => [row.direction, row.kind, row.mid, row.trackIdentifier])).toEqual([
['send', 'audio', '1', '{mic-track}'],
['send', 'video', '3', '{camera-track}'],
['send', 'video', '4', '{screen-track}'],
['recv', 'audio', '2', '{8a106956-d202-4916-8e19-0a1672c64b73}'],
]);
});
it('leaves an outbound row unidentified when no transceiver carries its mid', async () => {
const rows = await collectFirefoxRows([{mid: '3', sender: {track: {id: '{camera-track}'}}}]);
expect(rows.map((row) => row.trackIdentifier)).toEqual([
undefined,
'{camera-track}',
undefined,
'{8a106956-d202-4916-8e19-0a1672c64b73}',
]);
});
it('keeps the inbound trackIdentifier when the source exposes no transceivers at all', async () => {
const rows = await collectFirefoxRows(null);
expect(rows.map((row) => row.trackIdentifier)).toEqual([
undefined,
undefined,
undefined,
'{8a106956-d202-4916-8e19-0a1672c64b73}',
]);
});
it('lets the classifier tell a firefox camera apart from a simultaneous screen share', async () => {
const rows = await collectFirefoxRows([
{mid: '1', sender: {track: {id: '{mic-track}'}}},
{mid: '3', sender: {track: {id: '{camera-track}'}}},
{mid: '4', sender: {track: {id: '{screen-track}'}}},
]);
const classification = classifyVoiceEngineV2TrackStats({
tracks: rows.map((row) => ({
direction: row.direction,
kind: row.kind,
rid: row.rid,
trackIdentifier: row.trackIdentifier,
bitrateKbps: row.bitrateKbps,
})),
publications: {
localCameraTrackId: '{camera-track}',
localMicrophoneTrackId: '{mic-track}',
localScreenShareTrackId: '{screen-track}',
localScreenShareAudioTrackId: null,
remoteMicrophoneTrackIds: [],
remoteScreenShareTrackIds: [],
remoteScreenShareAudioTrackIds: [],
},
});
expect(rows[classification.localVideoTrackIndex!].mid).toBe('3');
expect(rows[classification.localScreenShareTrackIndex!].mid).toBe('4');
});
});
@@ -54,12 +54,14 @@ export interface PerTrackStats {
kind: 'audio' | 'video' | 'unknown';
ssrc?: number;
rid?: string;
active?: boolean;
mid?: string;
trackIdentifier?: string;
mediaSourceId?: string;
codec?: string;
payloadType?: number;
bitrateKbps: number;
bitrateWindowMs?: number;
packetsLost?: number;
packetsLossPercent?: number;
jitterMs?: number;
@@ -160,6 +162,12 @@ type RoomWithEngine = Room & {
interface StatsSource {
getStats(): Promise<StatsReportMap>;
getTransceivers?(): ReadonlyArray<StatsTransceiver>;
}
interface StatsTransceiver {
mid: string | null;
sender?: {track?: {id: string} | null} | null;
}
interface StatsReportMap {
@@ -186,6 +194,7 @@ interface RTCStatsEntry {
payloadType?: number;
ssrc?: number;
rid?: string;
active?: boolean;
mid?: string;
trackId?: string;
trackIdentifier?: string;
@@ -248,6 +257,11 @@ const LATENCY_UPDATE_INTERVAL_MS = 2000;
const STATS_CLOCK_INTERVAL_MS = 1000;
const STATS_UPDATE_INTERVAL_MS = 2000;
interface BitrateSample {
bitrateKbps: number;
windowMs?: number;
}
function normalizeTrackKind(kind: string | undefined): PerTrackStats['kind'] {
if (kind === 'audio' || kind === 'video') return kind;
return 'unknown';
@@ -264,28 +278,32 @@ function getReportKind(report: RTCStatsEntry, reportsById: Map<string, RTCStatsE
return normalizeTrackKind(track?.kind ?? track?.mediaType ?? mediaSource?.kind ?? mediaSource?.mediaType);
}
function getBitrateKbps(
function getBitrateSample(
reportId: string,
currentBytes: number | undefined,
now: number,
rtpCounters: Map<string, VoiceStatsRtpCounter>,
): number {
): BitrateSample {
if (typeof currentBytes !== 'number' || !Number.isFinite(currentBytes)) {
return 0;
return {bitrateKbps: 0};
}
const previous = rtpCounters.get(reportId);
let bitrateKbps = 0;
let windowMs: number | undefined;
if (previous?.bytes !== undefined) {
const dt = (now - previous.timestamp) / 1000;
const db = currentBytes - previous.bytes;
if (dt > 0 && db >= 0) bitrateKbps = (db * 8) / 1000 / dt;
if (dt > 0 && db >= 0) {
bitrateKbps = (db * 8) / 1000 / dt;
windowMs = Math.round(now - previous.timestamp);
}
}
rtpCounters.set(reportId, {
...previous,
bytes: currentBytes,
timestamp: now,
});
return bitrateKbps;
return {bitrateKbps, windowMs};
}
function hasUsableLossDeltas(lostDelta: number, receivedDelta: number): boolean {
@@ -344,6 +362,7 @@ function hasUsableJitterBuffer(report: RTCStatsEntry): boolean {
function buildOutboundTrackExtras(report: RTCStatsEntry): Partial<PerTrackStats> {
return {
active: report.active,
retransmittedPacketsSent: report.retransmittedPacketsSent,
retransmittedBytesSent: report.retransmittedBytesSent,
keyFramesEncoded: report.keyFramesEncoded,
@@ -379,11 +398,12 @@ function buildPerTrackStat(args: {
now: number;
rtpCounters: Map<string, VoiceStatsRtpCounter>;
reportsById: Map<string, RTCStatsEntry>;
midToSenderTrackId: Map<string, string>;
}): PerTrackStats {
const {report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById} = args;
const {report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById, midToSenderTrackId} = args;
const id = `${sourceId}:${report.type}:${report.id}`;
const currentBytes = isOutbound ? report.bytesSent : report.bytesReceived;
const bitrateKbps = getBitrateKbps(id, currentBytes, now, rtpCounters);
const bitrate = getBitrateSample(id, currentBytes, now, rtpCounters);
const codec = report.codecId ? reportsById.get(report.codecId) : undefined;
const mediaSource = report.mediaSourceId ? reportsById.get(report.mediaSourceId) : undefined;
const packetsLossPercent = isInbound ? getPacketLossPercent(id, report, now, rtpCounters) : undefined;
@@ -402,11 +422,15 @@ function buildPerTrackStat(args: {
ssrc: report.ssrc,
rid: report.rid,
mid: report.mid,
trackIdentifier: report.trackIdentifier ?? mediaSource?.trackIdentifier,
trackIdentifier:
report.trackIdentifier ??
mediaSource?.trackIdentifier ??
(isOutbound && report.mid ? midToSenderTrackId.get(report.mid) : undefined),
mediaSourceId: report.mediaSourceId,
codec: codec?.mimeType,
payloadType: codec?.payloadType,
bitrateKbps: Math.round(bitrateKbps),
bitrateKbps: Math.round(bitrate.bitrateKbps),
bitrateWindowMs: bitrate.windowMs,
packetsLost: report.packetsLost,
packetsLossPercent: packetsLossPercent !== undefined ? Math.round(packetsLossPercent * 10) / 10 : undefined,
jitterMs: typeof report.jitter === 'number' ? Math.round(report.jitter * 1000 * 10) / 10 : undefined,
@@ -485,6 +509,11 @@ async function collectFromStatsSource(
transport: TransportInfo | null;
}> {
const reports = await source.getStats();
const midToSenderTrackId = new Map<string, string>();
for (const transceiver of source.getTransceivers?.() ?? []) {
const senderTrackId = transceiver.sender?.track?.id;
if (transceiver.mid && senderTrackId) midToSenderTrackId.set(transceiver.mid, senderTrackId);
}
const reportsById = new Map<string, RTCStatsEntry>();
for (const report of reports.values()) {
const statsEntry = report as RTCStatsEntry;
@@ -511,7 +540,9 @@ async function collectFromStatsSource(
if (!isOutbound && !isInbound) continue;
const id = `${sourceId}:${report.type}:${report.id}`;
activeCounterIds.add(id);
tracks.push(buildPerTrackStat({report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById}));
tracks.push(
buildPerTrackStat({report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById, midToSenderTrackId}),
);
}
if (!activePair && transportReport?.selectedCandidatePairId) {
const selectedPair = reportsById.get(transportReport.selectedCandidatePairId);
@@ -0,0 +1,61 @@
// @vitest-environment happy-dom
// SPDX-License-Identifier: AGPL-3.0-or-later
import {findStalledVideoEncoder} from '@app/features/voice/engine/voice_screen_share_manager/shared';
import {describe, expect, test, vi} from 'vitest';
vi.mock('@app/features/voice/utils/VideoQualityEntitlement', () => ({hasHigherVideoQuality: () => false}));
function makeStats(entries: ReadonlyArray<Record<string, unknown>>): RTCStatsReport {
return new Map(entries.map((entry) => [entry.id as string, entry])) as unknown as RTCStatsReport;
}
const CODEC_REPORT = {type: 'codec', id: 'codec-h264', mimeType: 'video/H264'};
const SOURCE_REPORT = {type: 'media-source', id: 'source-1', kind: 'video', frames: 240, framesPerSecond: 30};
function outboundReport(overrides: Record<string, unknown>): Record<string, unknown> {
return {
type: 'outbound-rtp',
id: 'outbound-1',
kind: 'video',
codecId: 'codec-h264',
mediaSourceId: 'source-1',
framesEncoded: 0,
framesSent: 0,
...overrides,
};
}
describe('findStalledVideoEncoder', () => {
test('reports a stall when the encoding is active and the source is producing frames', () => {
const stats = makeStats([CODEC_REPORT, SOURCE_REPORT, outboundReport({active: true})]);
expect(findStalledVideoEncoder(stats, 'h264')).toEqual({
codec: 'h264',
framesEncoded: 0,
framesSent: 0,
sourceFrames: 240,
sourceFramesPerSecond: 30,
});
});
test('ignores an encoding the SFU deactivated even though the source is producing frames', () => {
const stats = makeStats([CODEC_REPORT, SOURCE_REPORT, outboundReport({active: false})]);
expect(findStalledVideoEncoder(stats, 'h264')).toBeNull();
});
test('still reports a stall when active is absent, as on Firefox', () => {
const stats = makeStats([CODEC_REPORT, SOURCE_REPORT, outboundReport({})]);
expect(findStalledVideoEncoder(stats, 'h264')?.codec).toBe('h264');
});
test('skips a deactivated layer and reports the still-active sibling instead', () => {
const stats = makeStats([
CODEC_REPORT,
SOURCE_REPORT,
{type: 'media-source', id: 'source-low', kind: 'video', frames: 90, framesPerSecond: 15},
outboundReport({id: 'outbound-low', mediaSourceId: 'source-low', active: false}),
outboundReport({id: 'outbound-high', active: true}),
]);
expect(findStalledVideoEncoder(stats, 'h264')?.sourceFrames).toBe(240);
});
});
@@ -469,6 +469,7 @@ interface OutboundVideoStatsEntry {
mediaType?: string;
codecId?: string;
mediaSourceId?: string;
active?: boolean;
framesEncoded?: number;
framesSent?: number;
encoderImplementation?: string;
@@ -592,6 +593,7 @@ export function findStalledVideoEncoder(stats: RTCStatsReport, codec?: VideoCode
}
for (const report of reports) {
if (getStatsKind(report, reportsById) !== 'video') continue;
if (report.active === false) continue;
const mimeType = report.codecId ? reportsById.get(report.codecId)?.mimeType : undefined;
if (report.codecId && !codecMatchesTarget(mimeType, codec)) continue;
const resolvedCodec = codec ?? getVideoCodecFromMimeType(mimeType);
@@ -0,0 +1,43 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {getNativeAudioCaptureDiagnosticState} from '@app/features/voice/utils/NativeAudioCaptureBridge';
import {recordBridgeFrame, startBridgeStats} from '@app/features/voice/utils/native_audio_capture_bridge/bridgeStats';
import type {
NativeAudioBridgeEndedCapture,
NativeAudioBridgeStats,
} from '@app/features/voice/utils/native_audio_capture_bridge/shared';
import {describe, expect, it} from 'vitest';
describe('getNativeAudioCaptureDiagnosticState', () => {
it('keeps the six fields real dumps already carry and appends the retained histories', () => {
expect(Object.keys(getNativeAudioCaptureDiagnosticState())).toEqual([
'armedCapture',
'activeBridge',
'supersededBridge',
'lastStartedCapture',
'lastArmFailure',
'bridgeStats',
'endedBridgeCaptures',
'lifecycleFaults',
]);
});
it('emits the superseded capture next to the live one instead of only the replacement', () => {
startBridgeStats('generator', 'native-audio:diag-failed', {prebufferTargetUs: 60_000, frameDurationUs: 10_000});
recordBridgeFrame('native-audio:diag-failed', {timestampUs: 0, durationUs: 10_000, peak: 0.31, rms: 0.08});
startBridgeStats('generator', 'native-audio:diag-live', {prebufferTargetUs: 60_000, frameDurationUs: 10_000});
const state = getNativeAudioCaptureDiagnosticState();
expect((state.bridgeStats as NativeAudioBridgeStats).captureId).toBe('native-audio:diag-live');
const ended = state.endedBridgeCaptures as Array<NativeAudioBridgeEndedCapture>;
const retained = ended.find((capture) => capture.captureId === 'native-audio:diag-failed');
expect(retained?.framesReceived).toBe(1);
expect(retained?.nonSilentFrameCount).toBe(1);
expect(retained?.endReason).toBe('superseded');
});
it('exposes a native-audio fault history that outlives the lifecycle unbind', () => {
expect(Array.isArray(getNativeAudioCaptureDiagnosticState().lifecycleFaults)).toBe(true);
});
});
@@ -4,7 +4,10 @@ import {Logger} from '@app/features/platform/utils/AppLogger';
import {getElectronAPI} from '@app/features/ui/utils/NativeUtils';
import type {VoiceEngineV2AppSourceLifecycleBridge} from '@app/features/voice/engine/v2/VoiceEngineV2AppSourceLifecycleBridge';
import {getNativeAudioErrorDetail} from '@app/features/voice/utils/NativeAudioFailureUtils';
import {getBridgeStats as getNativeAudioBridgeStats} from '@app/features/voice/utils/native_audio_capture_bridge/bridgeStats';
import {
getEndedBridgeCaptures,
getBridgeStats as getNativeAudioBridgeStats,
} from '@app/features/voice/utils/native_audio_capture_bridge/bridgeStats';
import {createGeneratorBridge} from '@app/features/voice/utils/native_audio_capture_bridge/createGeneratorBridge';
import {createScriptProcessorBridge} from '@app/features/voice/utils/native_audio_capture_bridge/createScriptProcessorBridge';
import {
@@ -67,8 +70,18 @@ interface NativeAudioStartedCaptureDiagnostic {
includeSelfWindowAudio?: boolean;
}
interface NativeAudioLifecycleFaultDiagnostic {
captureId: string;
sourceId: string;
message: string;
atMs: number;
}
const MAX_RETAINED_LIFECYCLE_FAULTS = 8;
let lastStartedCapture: NativeAudioStartedCaptureDiagnostic | null = null;
let sourceLifecycleBridge: VoiceEngineV2AppSourceLifecycleBridge | null = null;
let lifecycleFaults: Array<NativeAudioLifecycleFaultDiagnostic> = [];
const lifecycleBoundCaptureIds = new Set<string>();
export {getNativeAudioBridgeStats};
@@ -93,9 +106,19 @@ function bindNativeAudioCaptureLifecycle(captureId: string): void {
}
}
function recordLifecycleFault(captureId: string, message: string): void {
lifecycleFaults.push({captureId, sourceId: `native-audio-tap:${captureId}`, message, atMs: Date.now()});
if (lifecycleFaults.length > MAX_RETAINED_LIFECYCLE_FAULTS) {
lifecycleFaults = lifecycleFaults.slice(-MAX_RETAINED_LIFECYCLE_FAULTS);
}
}
function unbindNativeAudioCaptureLifecycle(captureId: string, faulted: boolean): void {
if (!lifecycleBoundCaptureIds.has(captureId)) return;
const bridge = sourceLifecycleBridge;
if (faulted) {
recordLifecycleFault(captureId, 'native-audio-tap-track-ended');
}
if (bridge) {
if (faulted) {
bridge.reportLifecycle({captureId, kind: 'error', message: 'native-audio-tap-track-ended'});
@@ -167,6 +190,8 @@ export function getNativeAudioCaptureDiagnosticState(): Record<string, unknown>
lastStartedCapture: started,
lastArmFailure: getLastNativeAudioArmFailure(),
bridgeStats: getNativeAudioBridgeStats(),
endedBridgeCaptures: getEndedBridgeCaptures(),
lifecycleFaults: lifecycleFaults.map((fault) => ({...fault})),
};
}
@@ -1499,5 +1524,6 @@ export function resetNativeAudioCaptureBridgeForTests(): void {
activeBridge = null;
supersededBridge = null;
sourceLifecycleBridge = null;
lifecycleFaults = [];
lifecycleBoundCaptureIds.clear();
}
@@ -15,7 +15,6 @@ import MediaEngine from '@app/features/voice/engine/MediaEngineFacade';
import ScreenShareCodecNegotiation, {
getScreenShareCodecPreferenceOrder,
} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import {getScreenShareAudioPumpDiagnostics} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump';
import {getNativeEngineAudioTrackPumpStats} from '@app/features/voice/engine/voice_screen_share_manager/NativeEngineAudioTrackPump';
import {getPublishedScreenShareMaxBitrateBps} from '@app/features/voice/engine/voice_screen_share_manager/shared';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
@@ -36,6 +35,7 @@ import {getScreenShareBitrateBps, resolveStreamingModeSettings} from '@app/featu
import {hasHigherVideoQuality} from '@app/features/voice/utils/VideoQualityEntitlement';
import {
buildVoiceStatsForNerdsPresentation,
collectScreenShareAudioPublicationDiagnostics,
type StatsForNerdsData,
} from '@app/features/voice/utils/VoiceStatsForNerdsPresenter';
import type {NativeAudioApplication, VirtmicNode} from '@app/types/electron.d';
@@ -519,8 +519,8 @@ export function collectStatsForNerdsSnapshot(): StatsForNerdsData {
openH264Enabled: VoiceSettings.getOpenH264Enabled(),
},
screenShareAudioCapture: {
pump: getScreenShareAudioPumpDiagnostics(),
nativeCapture: getNativeAudioCaptureDiagnosticState(),
publications: collectScreenShareAudioPublicationDiagnostics(localParticipant),
},
appInfo: {
appVersion: Config.PUBLIC_BUILD_VERSION ?? 'dev',
@@ -45,9 +45,11 @@ const CODEC_PROBES: ReadonlyArray<CodecProbe> = [
},
];
const SCREEN_SHARE_DECODE_FAILURES_MAX = 1;
let cachedExclusions: Array<VideoCodec> | null = null;
let pendingProbe: Promise<Array<VideoCodec>> | null = null;
const runtimeExcludedCodecs = new Set<VideoCodec>();
const screenShareDecodeFailures = new Set<VideoCodec>();
interface VideoDecoderLike {
isConfigSupported(config: WebCodecsConfig): Promise<{supported: boolean}>;
@@ -80,15 +82,6 @@ function isBaselineWebRtcDecodeCodec(codec: VideoCodec): boolean {
return codec === 'h264' || codec === 'vp8';
}
function mergeRuntimeExclusions(exclusions: Array<VideoCodec> | null): Array<VideoCodec> | null {
if (runtimeExcludedCodecs.size === 0) return exclusions;
const merged = new Set(exclusions ?? []);
for (const codec of runtimeExcludedCodecs) {
merged.add(codec);
}
return [...merged];
}
function getPlatformPolicyExclusions(): Array<VideoCodec> | null {
if ((globalThis as Record<string, unknown>).RTCRtpReceiver === undefined) return null;
const excluded: Array<VideoCodec> = [];
@@ -145,17 +138,17 @@ async function probeAllCodecs(): Promise<Array<VideoCodec>> {
}
export function loadVideoDecoderExclusions(): Promise<Array<VideoCodec>> {
if (cachedExclusions) return Promise.resolve(mergeRuntimeExclusions(cachedExclusions) ?? []);
if (cachedExclusions) return Promise.resolve(cachedExclusions);
if (pendingProbe) return pendingProbe;
pendingProbe = probeAllCodecs()
.then((result) => {
cachedExclusions = result;
return mergeRuntimeExclusions(result) ?? [];
return result;
})
.catch((error) => {
logger.warn('Video decoder probe failed entirely, not excluding any codecs', {error});
cachedExclusions = [];
return mergeRuntimeExclusions([]) ?? [];
return [];
})
.finally(() => {
pendingProbe = null;
@@ -164,21 +157,29 @@ export function loadVideoDecoderExclusions(): Promise<Array<VideoCodec>> {
}
export function getVideoDecoderExclusionsSync(): Array<VideoCodec> | null {
return mergeRuntimeExclusions(cachedExclusions ?? getPlatformPolicyExclusions());
return cachedExclusions ?? getPlatformPolicyExclusions();
}
export function markVideoDecoderRuntimeFailure(codec: VideoCodec, reason: string): boolean {
if (isBaselineWebRtcDecodeCodec(codec)) return false;
if (runtimeExcludedCodecs.has(codec)) return false;
runtimeExcludedCodecs.add(codec);
logger.warn('Excluding codec from subscriber decode after runtime failure', {codec, reason});
export function markScreenShareDecodeFailure(codec: VideoCodec, reason: string): boolean {
if (screenShareDecodeFailures.has(codec)) return false;
if (screenShareDecodeFailures.size >= SCREEN_SHARE_DECODE_FAILURES_MAX) return false;
screenShareDecodeFailures.add(codec);
logger.warn('Withdrawing decode advertisement for a codec after a screen share decode stall', {codec, reason});
return true;
}
export function getScreenShareDecodeFailures(): ReadonlySet<VideoCodec> {
return screenShareDecodeFailures;
}
export function clearScreenShareDecodeFailures(): void {
screenShareDecodeFailures.clear();
}
export function resetVideoDecoderExclusions(): void {
cachedExclusions = null;
pendingProbe = null;
runtimeExcludedCodecs.clear();
screenShareDecodeFailures.clear();
}
if (typeof window !== 'undefined') {
@@ -2,6 +2,7 @@
import {
buildVoiceStatsForNerdsPresentation,
collectScreenShareAudioPublicationDiagnostics,
type ParticipantPublicationLookup,
} from '@app/features/voice/utils/VoiceStatsForNerdsPresenter';
import type {
@@ -86,6 +87,64 @@ function nativeStats(overrides: Partial<VoiceEngineV2Stats>): VoiceEngineV2Stats
return {rttMs: null, outbound: [], inbound: [], ...overrides};
}
function chromiumCameraSimulcastTracks(): Array<VoiceEngineV2PerTrackStats> {
return [
{
direction: 'send',
kind: 'video',
ssrc: 1001,
rid: 'q',
active: false,
mid: '2',
trackIdentifier: 'cam-track',
mediaSourceId: 'SV1',
bitrateKbps: 0,
bitrateWindowMs: 2001,
framesEncoded: 1,
targetBitrateKbps: 0,
},
{
direction: 'send',
kind: 'video',
ssrc: 1002,
rid: 'h',
active: false,
mid: '2',
trackIdentifier: 'cam-track',
mediaSourceId: 'SV1',
bitrateKbps: 0,
bitrateWindowMs: 2001,
framesEncoded: 1,
targetBitrateKbps: 0,
},
{
direction: 'send',
kind: 'video',
ssrc: 1003,
rid: 'f',
active: true,
mid: '2',
trackIdentifier: 'cam-track',
mediaSourceId: 'SV1',
bitrateKbps: 0,
bitrateWindowMs: 2001,
framesEncoded: 1,
frameWidth: 1280,
frameHeight: 720,
targetBitrateKbps: 1133,
},
{
direction: 'send',
kind: 'video',
ssrc: 2001,
mid: '3',
trackIdentifier: 'screen-track',
mediaSourceId: 'SV2',
bitrateKbps: 3500,
},
];
}
describe('buildVoiceStatsForNerdsPresentation', () => {
it('classifies browser per-track stats once for overlay and copy consumers', () => {
const localParticipant = participant({
@@ -180,6 +239,203 @@ describe('buildVoiceStatsForNerdsPresentation', () => {
expect(presentation.remoteAudio).toBeNull();
});
it('classifies a firefox screen share whose outbound stats omit the track identifier', () => {
const localParticipant = participant({
microphone: 'b0d2f1a7-6c3e-4f8a-9b21-5d7c4e0a1f36',
screen_share: '2f4c8de1-9a07-4b53-8c6d-1e5b7a02d94f',
});
const remoteParticipant = participant({microphone: '8a106956-d202-4916-8e19-0a1672c64b73'});
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
{
direction: 'recv',
kind: 'audio',
mid: '2',
trackIdentifier: '{8a106956-d202-4916-8e19-0a1672c64b73}',
bitrateKbps: 46,
},
{direction: 'send', kind: 'audio', mid: '1', bitrateKbps: 30},
{direction: 'send', kind: 'video', mid: '3', bitrateKbps: 47, framesPerSecond: 58, frameWidth: 1920},
];
const presentation = buildVoiceStatsForNerdsPresentation({
connectionId: 'connection-a',
connectionQuality: 'excellent',
currentLatency: 37,
averageLatency: 41,
stats: voiceStats,
perTrackStats,
statsTimeSeries: timeSeries,
nativeStats: null,
publisherTransport: null,
subscriberTransport: null,
localParticipant,
remoteParticipants: [remoteParticipant],
});
expect(presentation.localScreenShare?.mid).toBe('3');
expect(presentation.localVideo).toBeNull();
});
it('classifies a firefox inbound screen share reported with a braced track identifier', () => {
const remoteParticipant = participant({screen_share: '34171e0d-174d-4d56-bb71-76b4fba330c3'});
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
{
direction: 'recv',
kind: 'video',
mid: '3',
trackIdentifier: '{34171e0d-174d-4d56-bb71-76b4fba330c3}',
bitrateKbps: 3585,
},
];
const presentation = buildVoiceStatsForNerdsPresentation({
connectionId: 'connection-a',
connectionQuality: 'excellent',
currentLatency: 37,
averageLatency: 41,
stats: voiceStats,
perTrackStats,
statsTimeSeries: timeSeries,
nativeStats: null,
publisherTransport: null,
subscriberTransport: null,
localParticipant: null,
remoteParticipants: [remoteParticipant],
});
expect(presentation.remoteScreenShare?.mid).toBe('3');
expect(presentation.remoteVideo).toBeNull();
});
it('exposes every camera simulcast layer next to the projected localVideo row', () => {
const localParticipant = participant({camera: 'cam-track', screen_share: 'screen-track'});
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = chromiumCameraSimulcastTracks();
const presentation = buildVoiceStatsForNerdsPresentation({
connectionId: 'connection-a',
connectionQuality: 'excellent',
currentLatency: 37,
averageLatency: 41,
stats: voiceStats,
perTrackStats,
statsTimeSeries: timeSeries,
nativeStats: null,
publisherTransport: null,
subscriberTransport: null,
localParticipant,
remoteParticipants: null,
});
expect(presentation.localVideo?.rid).toBe('q');
expect(presentation.localVideoLayers).toHaveLength(3);
expect(presentation.localVideoLayers.map((layer) => layer.rid)).toEqual(['q', 'h', 'f']);
expect(presentation.localVideoLayers.map((layer) => layer.active)).toEqual([false, false, true]);
expect(presentation.localVideoLayers.map((layer) => layer.bitrateWindowMs)).toEqual([2001, 2001, 2001]);
});
it('keeps the screen-share publication out of the camera layer view', () => {
const localParticipant = participant({camera: 'cam-track', screen_share: 'screen-track'});
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = chromiumCameraSimulcastTracks();
const presentation = buildVoiceStatsForNerdsPresentation({
connectionId: 'connection-a',
connectionQuality: 'excellent',
currentLatency: 37,
averageLatency: 41,
stats: voiceStats,
perTrackStats,
statsTimeSeries: timeSeries,
nativeStats: null,
publisherTransport: null,
subscriberTransport: null,
localParticipant,
remoteParticipants: null,
});
expect(presentation.localScreenShare?.trackIdentifier).toBe('screen-track');
expect(presentation.localVideoLayers).not.toContain(perTrackStats[3]);
});
it('reports a single camera layer when firefox omits rid and media source ids', () => {
const localParticipant = participant({camera: 'cam-track'});
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
{direction: 'send', kind: 'audio', mid: '1', bitrateKbps: 30},
{direction: 'send', kind: 'video', mid: '3', bitrateKbps: 47, framesPerSecond: 58, frameWidth: 1920},
];
const presentation = buildVoiceStatsForNerdsPresentation({
connectionId: 'connection-a',
connectionQuality: 'excellent',
currentLatency: 37,
averageLatency: 41,
stats: voiceStats,
perTrackStats,
statsTimeSeries: timeSeries,
nativeStats: null,
publisherTransport: null,
subscriberTransport: null,
localParticipant,
remoteParticipants: null,
});
expect(presentation.localVideo?.mid).toBe('3');
expect(presentation.localVideoLayers).toEqual([perTrackStats[1]]);
});
it('reports no camera layers rather than a false single layer when firefox simulcast is unidentifiable', () => {
const localParticipant = participant({camera: 'cam-track'});
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
{direction: 'send', kind: 'audio', mid: '1', bitrateKbps: 30},
{direction: 'send', kind: 'video', mid: '3', rid: 'q', bitrateKbps: 0},
{direction: 'send', kind: 'video', mid: '3', rid: 'h', bitrateKbps: 0},
{direction: 'send', kind: 'video', mid: '3', rid: 'f', bitrateKbps: 1133},
];
const presentation = buildVoiceStatsForNerdsPresentation({
connectionId: 'connection-a',
connectionQuality: 'excellent',
currentLatency: 37,
averageLatency: 41,
stats: voiceStats,
perTrackStats,
statsTimeSeries: timeSeries,
nativeStats: null,
publisherTransport: null,
subscriberTransport: null,
localParticipant,
remoteParticipants: null,
});
expect(presentation.localVideo).toBeNull();
expect(presentation.localVideoLayers).toEqual([]);
});
it('keeps the chromium simulcast projection byte-identical', () => {
const localParticipant = participant({camera: 'cam-track', screen_share: 'screen-track'});
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = chromiumCameraSimulcastTracks();
const presentation = buildVoiceStatsForNerdsPresentation({
connectionId: 'connection-a',
connectionQuality: 'excellent',
currentLatency: 37,
averageLatency: 41,
stats: voiceStats,
perTrackStats,
statsTimeSeries: timeSeries,
nativeStats: null,
publisherTransport: null,
subscriberTransport: null,
localParticipant,
remoteParticipants: null,
});
expect(presentation.localVideo).toBe(perTrackStats[0]);
expect(presentation.localScreenShare).toBe(perTrackStats[3]);
expect(presentation.localAudio).toBeNull();
expect(presentation.remoteVideo).toBeNull();
expect(presentation.network.videoSendBitrateKbps).toBe(1200);
});
it('uses the canonical v2 native stats projection when native stats are available', () => {
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
{direction: 'send', kind: 'video', trackIdentifier: 'browser-camera', bitrateKbps: 500},
@@ -309,3 +565,87 @@ describe('buildVoiceStatsForNerdsPresentation', () => {
expect(presentation.network.rttMs).toBeNull();
});
});
describe('collectScreenShareAudioPublicationDiagnostics', () => {
it('reports the mute and upstream state of every local screen-share audio publication', () => {
const displayPublication = {
trackSid: 'TR_display',
source: 'screen_share_audio',
isMuted: false,
audioTrack: {
isUpstreamPaused: true,
mediaStreamTrack: {
id: 'display-audio',
readyState: 'live',
muted: true,
enabled: true,
} as MediaStreamTrack,
},
};
const devicePublication = {
trackSid: 'TR_device',
source: 'screen_share_audio',
isMuted: false,
audioTrack: {
isUpstreamPaused: false,
mediaStreamTrack: {
id: 'device-audio',
readyState: 'live',
muted: false,
enabled: true,
} as MediaStreamTrack,
},
};
const micPublication = {
trackSid: 'TR_mic',
source: 'microphone',
isMuted: false,
audioTrack: {
isUpstreamPaused: false,
mediaStreamTrack: {
id: 'mic-audio',
readyState: 'live',
muted: false,
enabled: true,
} as MediaStreamTrack,
},
};
const localParticipant: ParticipantPublicationLookup = {
getTrackPublication: (source) => (String(source) === 'screen_share_audio' ? displayPublication : undefined),
audioTrackPublications: new Map([
['TR_display', displayPublication],
['TR_device', devicePublication],
['TR_mic', micPublication],
]),
};
const diagnostics = collectScreenShareAudioPublicationDiagnostics(localParticipant);
expect(diagnostics).toEqual([
{
trackSid: 'TR_display',
source: 'screen_share_audio',
isMuted: false,
isUpstreamPaused: true,
mediaStreamTrackId: 'display-audio',
mediaStreamTrackReadyState: 'live',
mediaStreamTrackMuted: true,
mediaStreamTrackEnabled: true,
},
{
trackSid: 'TR_device',
source: 'screen_share_audio',
isMuted: false,
isUpstreamPaused: false,
mediaStreamTrackId: 'device-audio',
mediaStreamTrackReadyState: 'live',
mediaStreamTrackMuted: false,
mediaStreamTrackEnabled: true,
},
]);
});
it('reports nothing when there is no local participant', () => {
expect(collectScreenShareAudioPublicationDiagnostics(null)).toEqual([]);
});
});
@@ -6,7 +6,6 @@ import {
type VoiceTrackPublicationSourceLike,
VoiceTrackSource,
} from '@app/features/voice/engine/VoiceTrackSource';
import type {ScreenShareAudioPumpDiagnostics} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump';
import {
classifyVoiceEngineV2TrackStats,
selectVoiceEngineV2StatsPresentationProjection,
@@ -21,6 +20,17 @@ import {
} from '@fluxer/voice_engine_v2';
import type {Track} from 'livekit-client';
export interface ScreenShareAudioPublicationDiagnostic {
trackSid: string | null;
source: string | null;
isMuted: boolean | null;
isUpstreamPaused: boolean | null;
mediaStreamTrackId: string | null;
mediaStreamTrackReadyState: string | null;
mediaStreamTrackMuted: boolean | null;
mediaStreamTrackEnabled: boolean | null;
}
export interface StatsForNerdsData {
session: {
connectionId: string;
@@ -44,6 +54,7 @@ export interface StatsForNerdsData {
subscriberTransport: VoiceEngineV2TransportInfo | null;
};
localVideo: VoiceEngineV2PerTrackStats | null;
localVideoLayers: Array<VoiceEngineV2PerTrackStats>;
localAudio: VoiceEngineV2PerTrackStats | null;
localScreenShare: VoiceEngineV2PerTrackStats | null;
localScreenShareAudio: VoiceEngineV2PerTrackStats | null;
@@ -85,8 +96,8 @@ export interface StatsForNerdsData {
openH264Enabled: boolean;
};
screenShareAudioCapture: {
pump: ScreenShareAudioPumpDiagnostics;
nativeCapture: Record<string, unknown>;
publications: Array<ScreenShareAudioPublicationDiagnostic>;
};
appInfo: {
appVersion: string;
@@ -119,6 +130,7 @@ export interface VoiceStatsForNerdsPresentation {
session: StatsForNerdsData['session'];
network: StatsForNerdsData['network'];
localVideo: VoiceEngineV2PerTrackStats | null;
localVideoLayers: Array<VoiceEngineV2PerTrackStats>;
localAudio: VoiceEngineV2PerTrackStats | null;
localScreenShare: VoiceEngineV2PerTrackStats | null;
localScreenShareAudio: VoiceEngineV2PerTrackStats | null;
@@ -131,9 +143,12 @@ export interface VoiceStatsForNerdsPresentation {
interface PublicationTrackLike {
mediaStreamTrack?: MediaStreamTrack;
isUpstreamPaused?: boolean;
}
interface TrackPublicationLike extends VoiceTrackPublicationSourceLike {
trackSid?: string;
isMuted?: boolean;
audioTrack?: PublicationTrackLike | null;
videoTrack?: PublicationTrackLike | null;
track?: PublicationTrackLike | null;
@@ -234,6 +249,38 @@ function collectParticipantScreenShareAudioTrackIdentifiers(
return Array.from(identifiers);
}
function summarizeScreenShareAudioPublication(
publication: TrackPublicationLike,
): ScreenShareAudioPublicationDiagnostic {
const track = publication.audioTrack ?? publication.track ?? null;
const mediaStreamTrack = track?.mediaStreamTrack ?? null;
return {
trackSid: publication.trackSid ?? null,
source: typeof publication.source === 'string' ? publication.source : null,
isMuted: publication.isMuted ?? null,
isUpstreamPaused: track?.isUpstreamPaused ?? null,
mediaStreamTrackId: mediaStreamTrack?.id ?? null,
mediaStreamTrackReadyState: mediaStreamTrack?.readyState ?? null,
mediaStreamTrackMuted: mediaStreamTrack?.muted ?? null,
mediaStreamTrackEnabled: mediaStreamTrack?.enabled ?? null,
};
}
export function collectScreenShareAudioPublicationDiagnostics(
participant: ParticipantPublicationLookup | null | undefined,
): Array<ScreenShareAudioPublicationDiagnostic> {
const publications = new Set<TrackPublicationLike>();
const directPublication = participant?.getTrackPublication(SCREEN_SHARE_AUDIO_SOURCE);
if (directPublication) {
publications.add(directPublication);
}
for (const publication of participant?.audioTrackPublications?.values() ?? []) {
if (!isScreenShareAudioPublicationLike(publication)) continue;
publications.add(publication);
}
return Array.from(publications, summarizeScreenShareAudioPublication);
}
function collectRemoteScreenShareAudioTrackIdentifiers(
remoteParticipants: Iterable<ParticipantPublicationLookup> | null | undefined,
): Array<string> {
@@ -284,6 +331,25 @@ function getClassifiedTrack(
return perTrackStats[index] ?? null;
}
function getPublicationGroupKey(track: VoiceEngineV2PerTrackStats): string | null {
if (track.mediaSourceId !== undefined) return `source:${track.mediaSourceId}`;
if (track.trackIdentifier !== undefined) return `track:${track.trackIdentifier}`;
if (track.mid !== undefined) return `mid:${track.mid}`;
return null;
}
function selectLocalVideoLayers(
perTrackStats: ReadonlyArray<VoiceEngineV2PerTrackStats>,
localVideo: VoiceEngineV2PerTrackStats | null,
): Array<VoiceEngineV2PerTrackStats> {
if (!localVideo) return [];
const groupKey = getPublicationGroupKey(localVideo);
if (groupKey === null) return [localVideo];
return perTrackStats.filter(
(track) => track.direction === 'send' && track.kind === 'video' && getPublicationGroupKey(track) === groupKey,
);
}
function selectNativeStatsProjection(
nativeStats: VoiceEngineV2Stats | null,
): VoiceEngineV2StatsPresentationProjection | null {
@@ -366,6 +432,7 @@ export function buildVoiceStatsForNerdsPresentation(
subscriberTransport,
},
localVideo: nativeProjection ? nativeProjection.localVideo : localVideo,
localVideoLayers: selectLocalVideoLayers(perTrackStats, localVideo),
localAudio: nativeProjection ? nativeProjection.localAudio : localAudio,
localScreenShare: nativeProjection ? nativeProjection.localScreenShare : localScreenShare,
localScreenShareAudio: nativeProjection ? nativeProjection.localScreenShareAudio : localScreenShareAudio,
@@ -0,0 +1,141 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {describe, expect, it, vi} from 'vitest';
import {
endBridgeStats,
getBridgeStats,
getEndedBridgeCaptures,
recordBridgeFrame,
startBridgeStats,
} from './bridgeStats';
function recordFrames(captureId: string, count: number, peak: number, rms: number): void {
for (let index = 0; index < count; index++) {
recordBridgeFrame(captureId, {timestampUs: index * 10_000, durationUs: 10_000, peak, rms});
}
}
function findEndedCapture(captureId: string) {
return getEndedBridgeCaptures().find((capture) => capture.captureId === captureId);
}
describe('bridgeStats', () => {
it('retains the counters of a capture that a mid-share restart supersedes', () => {
startBridgeStats('generator', 'native-audio:failed', {prebufferTargetUs: 60_000, frameDurationUs: 10_000});
recordFrames('native-audio:failed', 1124, 0.42, 0.11);
startBridgeStats('generator', 'native-audio:replacement', {prebufferTargetUs: 60_000, frameDurationUs: 10_000});
expect(getBridgeStats().captureId).toBe('native-audio:replacement');
expect(getBridgeStats().framesReceived).toBe(0);
const retained = findEndedCapture('native-audio:failed');
expect(retained?.bridgeMode).toBe('generator');
expect(retained?.framesReceived).toBe(1124);
expect(retained?.nonSilentFrameCount).toBe(1124);
expect(retained?.lastFramePeak).toBe(0.42);
expect(retained?.endReason).toBe('superseded');
});
it('records the end reason of a capture the newer capture already superseded', () => {
startBridgeStats('generator', 'native-audio:ends-late');
recordFrames('native-audio:ends-late', 3, 0.2, 0.05);
startBridgeStats('generator', 'native-audio:ends-late-replacement');
endBridgeStats('native-audio:ends-late', 'ended', 'source-disappeared');
const retained = findEndedCapture('native-audio:ends-late');
expect(retained?.endReason).toBe('ended');
expect(retained?.endDetail).toBe('source-disappeared');
expect(retained?.endedAt).not.toBeNull();
expect(getBridgeStats().captureId).toBe('native-audio:ends-late-replacement');
expect(getBridgeStats().active).toBe(true);
});
it('still ends the active capture in place', () => {
startBridgeStats('script-processor', 'native-audio:active-end');
endBridgeStats('native-audio:active-end', 'cleanup', 'caller-stopped');
const stats = getBridgeStats();
expect(stats.active).toBe(false);
expect(stats.endReason).toBe('cleanup');
expect(stats.endDetail).toBe('caller-stopped');
expect(stats.endedAt).not.toBeNull();
});
it('keeps the first end reason of the active capture', () => {
startBridgeStats('generator', 'native-audio:double-end');
endBridgeStats('native-audio:double-end', 'ended', 'source-disappeared');
endBridgeStats('native-audio:double-end', 'cleanup', 'caller-stopped');
expect(getBridgeStats().endReason).toBe('ended');
expect(getBridgeStats().endDetail).toBe('source-disappeared');
});
it('flags a capture whose frames keep arriving while every one of them is silence', () => {
const startMs = Date.UTC(2026, 8, 8, 12, 0, 0);
vi.useFakeTimers();
try {
vi.setSystemTime(startMs);
startBridgeStats('generator', 'native-audio:all-silent', {frameDurationUs: 10_000});
for (let index = 0; index < 400; index++) {
vi.setSystemTime(startMs + index * 100);
recordBridgeFrame('native-audio:all-silent', {
timestampUs: index * 10_000,
durationUs: 10_000,
peak: 0,
rms: 0,
});
}
const stats = getBridgeStats();
expect(stats.framesReceived).toBe(400);
expect(stats.nonSilentFrameCount).toBe(0);
expect(stats.lastNonSilentFrameAt).toBeNull();
expect(stats.silentFrameStreak).toBe(400);
expect(stats.maxSilentRunMs).toBe(39_900);
expect(stats.sustainedSilenceWarned).toBe(true);
} finally {
vi.useRealTimers();
}
});
it('does not flag a capture that keeps producing audible frames', () => {
const startMs = Date.UTC(2026, 8, 8, 13, 0, 0);
vi.useFakeTimers();
try {
vi.setSystemTime(startMs);
startBridgeStats('generator', 'native-audio:audible', {frameDurationUs: 10_000});
for (let index = 0; index < 400; index++) {
vi.setSystemTime(startMs + index * 100);
const audible = index % 4 === 0;
recordBridgeFrame('native-audio:audible', {
timestampUs: index * 10_000,
durationUs: 10_000,
peak: audible ? 0.3 : 0,
rms: audible ? 0.1 : 0,
});
}
const stats = getBridgeStats();
expect(stats.nonSilentFrameCount).toBe(100);
expect(stats.lastNonSilentFrameAt).toBe(startMs + 396 * 100);
expect(stats.silentFrameStreak).toBe(3);
expect(stats.maxSilentRunMs).toBe(300);
expect(stats.sustainedSilenceWarned).toBe(false);
} finally {
vi.useRealTimers();
}
});
it('bounds the retained capture history', () => {
for (let index = 0; index < 12; index++) {
startBridgeStats('generator', `native-audio:bounded-${index}`);
}
expect(getEndedBridgeCaptures().length).toBeLessThanOrEqual(8);
expect(findEndedCapture('native-audio:bounded-11')).toBeUndefined();
expect(findEndedCapture('native-audio:bounded-10')).toBeDefined();
});
});
@@ -1,22 +1,54 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Logger} from '@app/features/platform/utils/AppLogger';
import {
initialBridgeStats,
type NativeAudioBridgeEndedCapture,
type NativeAudioBridgeFrameMetrics,
type NativeAudioBridgeStats,
} from '@app/features/voice/utils/native_audio_capture_bridge/shared';
const logger = new Logger('NativeAudioBridgeStats');
const MAX_ENDED_BRIDGE_CAPTURES = 8;
const SUPERSEDED_END_REASON = 'superseded';
const SUSTAINED_SILENCE_WARN_MS = 30_000;
let bridgeStats: NativeAudioBridgeStats = {...initialBridgeStats};
let endedBridgeCaptures: Array<NativeAudioBridgeEndedCapture> = [];
export function getBridgeStats(): NativeAudioBridgeStats {
return {...bridgeStats};
}
export function getEndedBridgeCaptures(): Array<NativeAudioBridgeEndedCapture> {
return endedBridgeCaptures.map((capture) => ({...capture}));
}
function retireBridgeStats(): void {
if (bridgeStats.captureId === null) return;
endedBridgeCaptures.push({
captureId: bridgeStats.captureId,
bridgeMode: bridgeStats.bridgeMode,
startedAt: bridgeStats.startedAt,
endedAt: bridgeStats.endedAt ?? Date.now(),
endReason: bridgeStats.endReason ?? SUPERSEDED_END_REASON,
endDetail: bridgeStats.endDetail,
framesReceived: bridgeStats.framesReceived,
nonSilentFrameCount: bridgeStats.nonSilentFrameCount,
lastFramePeak: bridgeStats.lastFramePeak,
});
if (endedBridgeCaptures.length > MAX_ENDED_BRIDGE_CAPTURES) {
endedBridgeCaptures = endedBridgeCaptures.slice(-MAX_ENDED_BRIDGE_CAPTURES);
}
}
export function startBridgeStats(
mode: 'generator' | 'script-processor',
captureId: string,
options: {prebufferTargetUs?: number; frameDurationUs?: number} = {},
): void {
retireBridgeStats();
bridgeStats = {
...initialBridgeStats,
active: true,
@@ -57,6 +89,24 @@ export function recordBridgeFrame(captureId: string, metrics: NativeAudioBridgeF
bridgeStats.maxFrameRms = Math.max(bridgeStats.maxFrameRms, bridgeStats.lastFrameRms);
if (peak >= 0.0005 || rms >= 0.0001) {
bridgeStats.nonSilentFrameCount += 1;
bridgeStats.lastNonSilentFrameAt = now;
bridgeStats.silentFrameStreak = 0;
} else {
bridgeStats.silentFrameStreak += 1;
const silentSinceMs = bridgeStats.lastNonSilentFrameAt ?? bridgeStats.startedAt ?? now;
const silentRunMs = Math.max(0, now - silentSinceMs);
bridgeStats.maxSilentRunMs = Math.max(bridgeStats.maxSilentRunMs, silentRunMs);
if (!bridgeStats.sustainedSilenceWarned && silentRunMs >= SUSTAINED_SILENCE_WARN_MS) {
bridgeStats.sustainedSilenceWarned = true;
logger.warn('Native audio capture is delivering frames that contain only silence', {
captureId: bridgeStats.captureId,
bridgeMode: bridgeStats.bridgeMode,
framesReceived: bridgeStats.framesReceived,
nonSilentFrameCount: bridgeStats.nonSilentFrameCount,
silentFrameStreak: bridgeStats.silentFrameStreak,
silentRunMs,
});
}
}
}
}
@@ -95,9 +145,16 @@ export function recordBridgeQueue(captureId: string, pendingFrames: number, buff
}
export function endBridgeStats(captureId: string, reason: string | null, detail: string | null): void {
if (!bridgeStats.active || bridgeStats.captureId !== captureId) return;
bridgeStats.active = false;
bridgeStats.endReason = reason;
bridgeStats.endDetail = detail;
bridgeStats.endedAt = Date.now();
if (bridgeStats.active && bridgeStats.captureId === captureId) {
bridgeStats.active = false;
bridgeStats.endReason = reason;
bridgeStats.endDetail = detail;
bridgeStats.endedAt = Date.now();
return;
}
const ended = endedBridgeCaptures.find((capture) => capture.captureId === captureId);
if (!ended || ended.endReason !== SUPERSEDED_END_REASON) return;
ended.endReason = reason;
ended.endDetail = detail;
ended.endedAt = Date.now();
}
@@ -122,6 +122,10 @@ export interface NativeAudioBridgeStats {
maxFramePeak: number;
maxFrameRms: number;
nonSilentFrameCount: number;
lastNonSilentFrameAt: number | null;
silentFrameStreak: number;
maxSilentRunMs: number;
sustainedSilenceWarned: boolean;
prebufferTargetMs: number | null;
frameDurationMs: number | null;
endReason: string | null;
@@ -129,6 +133,18 @@ export interface NativeAudioBridgeStats {
endedAt: number | null;
}
export interface NativeAudioBridgeEndedCapture {
captureId: string;
bridgeMode: 'generator' | 'script-processor' | null;
startedAt: number | null;
endedAt: number | null;
endReason: string | null;
endDetail: string | null;
framesReceived: number;
nonSilentFrameCount: number;
lastFramePeak: number | null;
}
export interface NativeAudioBridgeFrameMetrics {
timestampUs?: number;
durationUs?: number;
@@ -156,6 +172,10 @@ export const initialBridgeStats: NativeAudioBridgeStats = {
maxFramePeak: 0,
maxFrameRms: 0,
nonSilentFrameCount: 0,
lastNonSilentFrameAt: null,
silentFrameStreak: 0,
maxSilentRunMs: 0,
sustainedSilenceWarned: false,
prebufferTargetMs: null,
frameDurationMs: null,
endReason: null,
@@ -119,5 +119,177 @@
"remoteScreenShareTrackIndex": 3,
"remoteScreenShareAudioTrackIndex": null
}
},
{
"name": "classifies a firefox screen share whose outbound stats omit the track identifier",
"input": {
"tracks": [
{
"direction": "recv",
"kind": "audio",
"trackIdentifier": "{8a106956-d202-4916-8e19-0a1672c64b73}",
"bitrateKbps": 46
},
{"direction": "send", "kind": "audio", "bitrateKbps": 30},
{"direction": "send", "kind": "video", "bitrateKbps": 47}
],
"publications": {
"localCameraTrackId": null,
"localMicrophoneTrackId": "b0d2f1a7-6c3e-4f8a-9b21-5d7c4e0a1f36",
"localScreenShareTrackId": "2f4c8de1-9a07-4b53-8c6d-1e5b7a02d94f",
"localScreenShareAudioTrackId": null,
"remoteMicrophoneTrackIds": ["8a106956-d202-4916-8e19-0a1672c64b73"],
"remoteScreenShareTrackIds": [],
"remoteScreenShareAudioTrackIds": []
}
},
"expected": {
"localVideoTrackIndex": null,
"localAudioTrackIndex": 1,
"localScreenShareTrackIndex": 2,
"localScreenShareAudioTrackIndex": null,
"remoteVideoTrackIndex": null,
"remoteAudioTrackIndex": 0,
"remoteScreenShareTrackIndex": null,
"remoteScreenShareAudioTrackIndex": null
}
},
{
"name": "classifies a firefox inbound screen share reported with a braced track identifier",
"input": {
"tracks": [
{
"direction": "recv",
"kind": "audio",
"trackIdentifier": "{9010d313-264b-4c51-9f13-9a6aefafce83}",
"bitrateKbps": 29
},
{
"direction": "recv",
"kind": "video",
"trackIdentifier": "{34171e0d-174d-4d56-bb71-76b4fba330c3}",
"bitrateKbps": 3585
},
{"direction": "send", "kind": "audio", "bitrateKbps": 44}
],
"publications": {
"localCameraTrackId": null,
"localMicrophoneTrackId": "7c5a1b98-3d24-4e6f-8a10-9b2c3d4e5f60",
"localScreenShareTrackId": null,
"localScreenShareAudioTrackId": null,
"remoteMicrophoneTrackIds": ["9010d313-264b-4c51-9f13-9a6aefafce83"],
"remoteScreenShareTrackIds": ["34171e0d-174d-4d56-bb71-76b4fba330c3"],
"remoteScreenShareAudioTrackIds": []
}
},
"expected": {
"localVideoTrackIndex": null,
"localAudioTrackIndex": 2,
"localScreenShareTrackIndex": null,
"localScreenShareAudioTrackIndex": null,
"remoteVideoTrackIndex": null,
"remoteAudioTrackIndex": 0,
"remoteScreenShareTrackIndex": 1,
"remoteScreenShareAudioTrackIndex": null
}
},
{
"name": "leaves both local video roles unresolved when unidentified sends could be camera or screen share",
"input": {
"tracks": [
{"direction": "send", "kind": "video", "bitrateKbps": 1600},
{"direction": "send", "kind": "video", "bitrateKbps": 500}
],
"publications": {
"localCameraTrackId": "c1e7d3b0-88f4-45a9-9c62-0ab5d1e2f374",
"localMicrophoneTrackId": null,
"localScreenShareTrackId": "2f4c8de1-9a07-4b53-8c6d-1e5b7a02d94f",
"localScreenShareAudioTrackId": null,
"remoteMicrophoneTrackIds": [],
"remoteScreenShareTrackIds": [],
"remoteScreenShareAudioTrackIds": []
}
},
"expected": {
"localVideoTrackIndex": null,
"localAudioTrackIndex": null,
"localScreenShareTrackIndex": null,
"localScreenShareAudioTrackIndex": null,
"remoteVideoTrackIndex": null,
"remoteAudioTrackIndex": null,
"remoteScreenShareTrackIndex": null,
"remoteScreenShareAudioTrackIndex": null
}
},
{
"name": "keeps chromium outbound identifier matching for a screen share published without a camera",
"input": {
"tracks": [
{
"direction": "send",
"kind": "video",
"trackIdentifier": "97079fec-a3af-4bc0-a8e6-ad142dc05c83",
"bitrateKbps": 1600
},
{"direction": "send", "kind": "audio", "bitrateKbps": 30}
],
"publications": {
"localCameraTrackId": null,
"localMicrophoneTrackId": null,
"localScreenShareTrackId": "97079fec-a3af-4bc0-a8e6-ad142dc05c83",
"localScreenShareAudioTrackId": null,
"remoteMicrophoneTrackIds": [],
"remoteScreenShareTrackIds": [],
"remoteScreenShareAudioTrackIds": []
}
},
"expected": {
"localVideoTrackIndex": null,
"localAudioTrackIndex": 1,
"localScreenShareTrackIndex": 0,
"localScreenShareAudioTrackIndex": null,
"remoteVideoTrackIndex": null,
"remoteAudioTrackIndex": null,
"remoteScreenShareTrackIndex": null,
"remoteScreenShareAudioTrackIndex": null
}
},
{
"name": "keeps chromium identified sends eligible for the camera role while a screen share is published",
"input": {
"tracks": [
{
"direction": "send",
"kind": "video",
"trackIdentifier": "97079fec-a3af-4bc0-a8e6-ad142dc05c83",
"bitrateKbps": 1600
},
{
"direction": "send",
"kind": "video",
"trackIdentifier": "1d0a4c62-5b3e-4f71-88d9-6c2a7e50b431",
"bitrateKbps": 400
}
],
"publications": {
"localCameraTrackId": null,
"localMicrophoneTrackId": null,
"localScreenShareTrackId": "97079fec-a3af-4bc0-a8e6-ad142dc05c83",
"localScreenShareAudioTrackId": null,
"remoteMicrophoneTrackIds": [],
"remoteScreenShareTrackIds": [],
"remoteScreenShareAudioTrackIds": []
}
},
"expected": {
"localVideoTrackIndex": 1,
"localAudioTrackIndex": null,
"localScreenShareTrackIndex": 0,
"localScreenShareAudioTrackIndex": null,
"remoteVideoTrackIndex": null,
"remoteAudioTrackIndex": null,
"remoteScreenShareTrackIndex": null,
"remoteScreenShareAudioTrackIndex": null
}
}
]
@@ -513,24 +513,55 @@ function normalizedId(id: string | null | undefined): string | null {
return trimmed ? trimmed : null;
}
function canonicalId(id: string | null | undefined): string | null {
const normalized = normalizedId(id);
if (normalized === null) return null;
const unwrapped =
normalized.startsWith('{') && normalized.endsWith('}') ? normalized.slice(1, -1).trim() : normalized;
return unwrapped ? unwrapped.toLowerCase() : null;
}
function idList(ids: ReadonlyArray<string>): Array<string> {
return ids.map((id) => normalizedId(id)).filter((id): id is string => id !== null);
return ids.map((id) => canonicalId(id)).filter((id): id is string => id !== null);
}
function matchesId(candidate: IndexedVoiceEngineV2StatsTrack, id: string | null): boolean {
const trackId = normalizedId(candidate.track.trackIdentifier);
return trackId !== null && id !== null && trackId === id;
const trackId = canonicalId(candidate.track.trackIdentifier);
const publicationId = canonicalId(id);
return trackId !== null && publicationId !== null && trackId === publicationId;
}
function idInList(candidate: IndexedVoiceEngineV2StatsTrack, ids: ReadonlyArray<string>): boolean {
const trackId = normalizedId(candidate.track.trackIdentifier);
const trackId = canonicalId(candidate.track.trackIdentifier);
return trackId !== null && ids.includes(trackId);
}
function hasId(candidate: IndexedVoiceEngineV2StatsTrack): boolean {
return canonicalId(candidate.track.trackIdentifier) !== null;
}
function hasRid(candidate: IndexedVoiceEngineV2StatsTrack): boolean {
return normalizedId(candidate.track.rid) !== null;
}
function couldBeLocalScreenShare(
candidate: IndexedVoiceEngineV2StatsTrack,
localScreenShareTrackId: string | null,
): boolean {
if (localScreenShareTrackId === null) return false;
return matchesId(candidate, localScreenShareTrackId) || !hasId(candidate);
}
function unidentifiedLocalScreenShareIndex(
sentVideoTracks: ReadonlyArray<IndexedVoiceEngineV2StatsTrack>,
localCameraTrackId: string | null,
localScreenShareTrackId: string | null,
): number | null {
if (localScreenShareTrackId === null || localCameraTrackId !== null || sentVideoTracks.length !== 1) return null;
const candidate = sentVideoTracks[0];
return candidate && !hasId(candidate) ? candidate.index : null;
}
function filterById(
candidates: ReadonlyArray<IndexedVoiceEngineV2StatsTrack>,
id: string | null,
@@ -565,19 +596,25 @@ export function classifyVoiceEngineV2TrackStats(
const remoteScreenShareTracks = receivedVideoTracks.filter((track) => idInList(track, remoteScreenShareTrackIds));
const remoteScreenShareTrackIndex = pickActiveTrackIndex(remoteScreenShareTracks);
const fallbackRemoteVideoTracks = receivedVideoTracks.filter((track) => !idInList(track, remoteScreenShareTrackIds));
const localScreenShareFallbackIndex = unidentifiedLocalScreenShareIndex(
sentVideoTracks,
localCameraTrackId,
localScreenShareTrackId,
);
return {
localVideoTrackIndex:
pickActiveTrackIndex(filterById(sentVideoTracks, localCameraTrackId)) ??
pickActiveTrackIndex(
sentVideoTracks.filter((track) => !hasRid(track) && !matchesId(track, localScreenShareTrackId)),
sentVideoTracks.filter((track) => !hasRid(track) && !couldBeLocalScreenShare(track, localScreenShareTrackId)),
),
localAudioTrackIndex:
pickActiveTrackIndex(filterById(sentAudioTracks, localMicrophoneTrackId)) ??
pickActiveTrackIndex(sentAudioTracks.filter((track) => !matchesId(track, localScreenShareAudioTrackId))),
localScreenShareTrackIndex:
pickActiveTrackIndex(filterById(sentVideoTracks, localScreenShareTrackId)) ??
pickActiveTrackIndex(sentVideoTracks.filter((track) => hasRid(track) && !matchesId(track, localCameraTrackId))),
pickActiveTrackIndex(sentVideoTracks.filter((track) => hasRid(track) && !matchesId(track, localCameraTrackId))) ??
localScreenShareFallbackIndex,
localScreenShareAudioTrackIndex: pickActiveTrackIndex(filterById(sentAudioTracks, localScreenShareAudioTrackId)),
remoteVideoTrackIndex:
pickActiveTrackIndex(fallbackRemoteVideoTracks) ??
@@ -499,12 +499,14 @@ export interface VoiceEngineV2PerTrackStats {
kind: 'audio' | 'video' | 'unknown';
ssrc?: number;
rid?: string;
active?: boolean;
mid?: string;
trackIdentifier?: string;
mediaSourceId?: string;
codec?: string;
payloadType?: number;
bitrateKbps: number;
bitrateWindowMs?: number;
packetsLost?: number;
packetsLossPercent?: number;
jitterMs?: number;