diff --git a/fluxer_app/src/features/voice/commands/VoiceSettingsCommands.test.ts b/fluxer_app/src/features/voice/commands/VoiceSettingsCommands.test.ts new file mode 100644 index 000000000..ef75a9ea0 --- /dev/null +++ b/fluxer_app/src/features/voice/commands/VoiceSettingsCommands.test.ts @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {beforeEach, describe, expect, it, vi} from 'vitest'; + +const refreshMicrophoneFromSettings = vi.fn(); +const refreshCameraBackgroundFromSettings = vi.fn(); +const refreshCameraCaptureFromSettings = vi.fn(); +const refreshScreenShareCodecNegotiationFromSettings = vi.fn(); + +const settings = { + cameraResolution: 720, + videoDeviceId: 'camera', + screenshareResolution: 1080, + videoFrameRate: 60, + streamingMode: 'smooth', + screenShareContentHint: 'auto', + preferredScreenShareCodec: 'auto', + screenShareAv1OptIn: true, + screenShareHevcOptIn: false, + screenShareEncoderMode: 'auto', + screenShareSoftwareQuality: 'auto', + screenShareScalabilityMode: 'auto', + screenShareBackupCodecMode: 'auto', + openH264Enabled: false, +}; + +vi.mock('@app/features/voice/engine/MediaEngineFacade', () => ({ + default: { + room: null, + refreshMicrophoneFromSettings, + refreshCameraBackgroundFromSettings, + refreshCameraCaptureFromSettings, + refreshScreenShareCodecNegotiationFromSettings, + setScreenShareAudioMuted: vi.fn(), + applyAllLocalAudioPreferences: vi.fn(), + applyLocalInputVolume: vi.fn(), + }, +})); + +vi.mock('@app/features/voice/engine/ScreenShareUnderperformance', () => ({ + noteDeliberateScreenShareQualityChange: vi.fn(), +})); + +vi.mock('@app/features/voice/utils/VoiceProcessingProfile', () => ({ + getActiveInputDeviceLabel: () => null, +})); + +vi.mock('@app/features/voice/state/VoiceSettings', () => ({ + default: { + updateSettings: (patch: Record) => Object.assign(settings, patch), + get cameraResolution() { + return settings.cameraResolution; + }, + get videoDeviceId() { + return settings.videoDeviceId; + }, + getScreenshareResolution: () => settings.screenshareResolution, + getVideoFrameRate: () => settings.videoFrameRate, + getStreamingMode: () => settings.streamingMode, + getScreenShareContentHint: () => settings.screenShareContentHint, + getPreferredScreenShareCodec: () => settings.preferredScreenShareCodec, + getScreenShareAv1OptIn: () => settings.screenShareAv1OptIn, + getScreenShareHevcOptIn: () => settings.screenShareHevcOptIn, + getScreenShareEncoderMode: () => settings.screenShareEncoderMode, + getScreenShareSoftwareQuality: () => settings.screenShareSoftwareQuality, + getScreenShareScalabilityMode: () => settings.screenShareScalabilityMode, + getScreenShareBackupCodecMode: () => settings.screenShareBackupCodecMode, + getOpenH264Enabled: () => settings.openH264Enabled, + }, +})); + +const VoiceSettingsCommands = await import('./VoiceSettingsCommands'); + +describe('screen-share codec settings reactions', () => { + beforeEach(() => { + settings.screenShareAv1OptIn = true; + settings.screenShareHevcOptIn = false; + settings.preferredScreenShareCodec = 'auto'; + refreshScreenShareCodecNegotiationFromSettings.mockClear(); + }); + + it('renegotiates the live screen share when the AV1 opt-in is turned off', () => { + VoiceSettingsCommands.update({screenShareAv1OptIn: false}); + expect(refreshScreenShareCodecNegotiationFromSettings).toHaveBeenCalledTimes(1); + }); + + it('renegotiates the live screen share when the HEVC opt-in changes', () => { + VoiceSettingsCommands.update({screenShareHevcOptIn: true}); + expect(refreshScreenShareCodecNegotiationFromSettings).toHaveBeenCalledTimes(1); + }); + + it('renegotiates the live screen share when the preferred codec changes', () => { + VoiceSettingsCommands.update({preferredScreenShareCodec: 'vp9'}); + expect(refreshScreenShareCodecNegotiationFromSettings).toHaveBeenCalledTimes(1); + }); + + it('leaves the negotiation alone for unrelated screen share settings', () => { + VoiceSettingsCommands.update({videoFrameRate: 30}); + expect(refreshScreenShareCodecNegotiationFromSettings).not.toHaveBeenCalled(); + }); +}); diff --git a/fluxer_app/src/features/voice/commands/VoiceSettingsCommands.ts b/fluxer_app/src/features/voice/commands/VoiceSettingsCommands.ts index a9de6afa4..bc69d199b 100644 --- a/fluxer_app/src/features/voice/commands/VoiceSettingsCommands.ts +++ b/fluxer_app/src/features/voice/commands/VoiceSettingsCommands.ts @@ -103,6 +103,11 @@ const CAMERA_BACKGROUND_REFRESH_KEYS: Array = [ 'mirrorCamera', ]; const CAMERA_CAPTURE_REFRESH_KEYS: Array = ['cameraResolution', 'videoDeviceId']; +const SCREEN_SHARE_CODEC_NEGOTIATION_REFRESH_KEYS: Array = [ + 'preferredScreenShareCodec', + 'screenShareAv1OptIn', + 'screenShareHevcOptIn', +]; function refreshMicrophone(): void { MediaEngine.refreshMicrophoneFromSettings(); } @@ -115,6 +120,10 @@ function refreshCameraCapture(): void { MediaEngine.refreshCameraCaptureFromSettings(); } +function refreshScreenShareCodecNegotiation(): void { + MediaEngine.refreshScreenShareCodecNegotiationFromSettings(); +} + function shouldRefreshMicrophone(settings: VoiceSettingsPatch): boolean { return MICROPHONE_REFRESH_KEYS.some((key) => settings[key] !== undefined); } @@ -123,6 +132,10 @@ function shouldRefreshCameraBackground(settings: VoiceSettingsPatch): boolean { return CAMERA_BACKGROUND_REFRESH_KEYS.some((key) => settings[key] !== undefined); } +function shouldRefreshScreenShareCodecNegotiation(settings: VoiceSettingsPatch): boolean { + return SCREEN_SHARE_CODEC_NEGOTIATION_REFRESH_KEYS.some((key) => settings[key] !== undefined); +} + function readCameraCaptureRefreshValues(): VoiceSettingsPatch { return { cameraResolution: VoiceSettings.cameraResolution, @@ -196,6 +209,9 @@ function applyUpdatedVoiceSettings( if (shouldRefreshCameraCapture(settings, cameraCaptureBefore, readCameraCaptureRefreshValues())) { refreshCameraCapture(); } + if (shouldRefreshScreenShareCodecNegotiation(settings)) { + refreshScreenShareCodecNegotiation(); + } } export function setVoiceProcessingModeForDeviceLabel(label: string, mode: VoiceProcessingMode): void { diff --git a/fluxer_app/src/features/voice/components/useStatsForNerds.ts b/fluxer_app/src/features/voice/components/useStatsForNerds.ts index 01a28f6f8..128c51804 100644 --- a/fluxer_app/src/features/voice/components/useStatsForNerds.ts +++ b/fluxer_app/src/features/voice/components/useStatsForNerds.ts @@ -9,7 +9,6 @@ import MediaEngine, {useMediaEngineVersion} from '@app/features/voice/engine/Med import ScreenShareCodecNegotiation, { getScreenShareCodecPreferenceOrder, } from '@app/features/voice/engine/ScreenShareCodecNegotiation'; -import {getScreenShareAudioPumpDiagnostics} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump'; import {getPublishedScreenShareMaxBitrateBps} from '@app/features/voice/engine/voice_screen_share_manager/shared'; import VoiceSettings from '@app/features/voice/state/VoiceSettings'; import {getNativeAudioCaptureDiagnosticState} from '@app/features/voice/utils/NativeAudioCaptureBridge'; @@ -17,6 +16,7 @@ import {getScreenShareBitrateBps, resolveStreamingModeSettings} from '@app/featu import {hasHigherVideoQuality} from '@app/features/voice/utils/VideoQualityEntitlement'; import { buildVoiceStatsForNerdsPresentation, + collectScreenShareAudioPublicationDiagnostics, type StatsForNerdsData, } from '@app/features/voice/utils/VoiceStatsForNerdsPresenter'; import type {VoiceEngineV2PerTrackStats, VoiceEngineV2TransportInfo} from '@fluxer/voice_engine_v2'; @@ -216,8 +216,8 @@ export function useStatsForNerds({enabled = true}: UseStatsForNerdsOptions = {}) openH264Enabled: VoiceSettings.getOpenH264Enabled(), }, screenShareAudioCapture: { - pump: getScreenShareAudioPumpDiagnostics(), nativeCapture: getNativeAudioCaptureDiagnosticState(), + publications: collectScreenShareAudioPublicationDiagnostics(localParticipant), }, appInfo: { appVersion: Config.PUBLIC_BUILD_VERSION ?? 'dev', diff --git a/fluxer_app/src/features/voice/diagnostics/VoiceDebugStatsHtml.test.tsx b/fluxer_app/src/features/voice/diagnostics/VoiceDebugStatsHtml.test.tsx index ee123b115..a28a4d7b1 100644 --- a/fluxer_app/src/features/voice/diagnostics/VoiceDebugStatsHtml.test.tsx +++ b/fluxer_app/src/features/voice/diagnostics/VoiceDebugStatsHtml.test.tsx @@ -27,6 +27,7 @@ function createStatsData(): StatsForNerdsData { subscriberTransport: null, }, localVideo: null, + localVideoLayers: [], localAudio: null, localScreenShare: null, localScreenShareAudio: null, @@ -68,20 +69,8 @@ function createStatsData(): StatsForNerdsData { openH264Enabled: true, }, screenShareAudioCapture: { - pump: { - active: false, - captureId: null, - sampleRate: null, - channels: null, - usesNativeSink: false, - publishStrategy: 'none', - publishedFormatKey: null, - eagerPublish: null, - eagerPublishError: null, - droppedPushFrames: 0, - pendingPushFrames: 0, - }, nativeCapture: {}, + publications: [], }, appInfo: { appVersion: 'dev', @@ -111,6 +100,50 @@ function createStatsData(): StatsForNerdsData { }; } +function createNativeCaptureRecord(): Record { + return { + armedCapture: null, + activeBridge: {captureId: 'native-audio:live'}, + supersededBridge: null, + lastStartedCapture: null, + lastArmFailure: null, + bridgeStats: { + active: true, + bridgeMode: 'generator', + captureId: 'native-audio:live', + startedAt: 1772000000000, + lastFrameAt: 1772000011240, + lastFrameTimestampUs: 11240000, + framesReceived: 1124, + framesDropped: 0, + lateFrameCount: 0, + rebufferCount: 0, + maxFrameArrivalGapMs: 32, + maxFrameTimestampGapMs: 10, + maxPendingFrames: 3, + maxBufferedDurationMs: 30, + lastFramePeak: 0.42, + lastFrameRms: 0.11, + maxFramePeak: 0.71, + maxFrameRms: 0.19, + nonSilentFrameCount: 1118, + prebufferTargetMs: 60, + frameDurationMs: 10, + endReason: null, + endDetail: null, + endedAt: null, + }, + endedBridgeCaptures: [], + lifecycleFaults: [], + }; +} + +function screenShareAudioCaptureSection(html: string): string { + const match = /]*>]*>screenShareAudioCapture \(native\)<\/h3>[\s\S]*?<\/section>/.exec(html); + expect(match).not.toBeNull(); + return match?.[0] ?? ''; +} + describe('renderVoiceDebugStatsHtml', () => { it('renders pure SVG sparklines for key popout metrics', () => { const html = renderVoiceDebugStatsHtml(createStatsData(), '2026-06-10T18:00:00.000Z'); @@ -124,4 +157,57 @@ describe('renderVoiceDebugStatsHtml', () => { expect(html).toContain('heapUsedMB sparkline'); expect(html).toContain('mainProcessCpuPercent sparkline'); }); + + it('does not render a screen-share audio pump section while the native capture is live', () => { + const data = createStatsData(); + data.screenShareAudioCapture = {nativeCapture: createNativeCaptureRecord(), publications: []}; + + const html = renderVoiceDebugStatsHtml(data, '2026-06-10T18:00:00.000Z'); + + expect(html).toContain('screenShareAudioCapture (native)'); + expect(html).toContain('framesReceived'); + expect(html).not.toContain('screenShareAudioCapture (pump)'); + expect(html).not.toContain('publishStrategy'); + expect(html).not.toContain('droppedPushFrames'); + expect(html).not.toContain('usesNativeSink'); + }); + + it('renders the screen-share audio capture section identically for Chromium and Firefox dumps', () => { + const chromium = createStatsData(); + chromium.screenShareAudioCapture = {nativeCapture: createNativeCaptureRecord(), publications: []}; + chromium.system.platform = 'Win32'; + chromium.system.userAgent = + '(Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36'; + const firefox = createStatsData(); + firefox.screenShareAudioCapture = {nativeCapture: createNativeCaptureRecord(), publications: []}; + firefox.system.platform = 'Win32'; + firefox.system.userAgent = '(Windows NT 10.0; Win64; x64; rv:155.0) Gecko/20100101 Firefox/155.0'; + firefox.appInfo.electronVersion = null; + firefox.appInfo.chromiumVersion = null; + firefox.appInfo.hardwareAccelerationEnabled = null; + + const chromiumHtml = renderVoiceDebugStatsHtml(chromium, '2026-06-10T18:00:00.000Z'); + const firefoxHtml = renderVoiceDebugStatsHtml(firefox, '2026-06-10T18:00:00.000Z'); + + expect(screenShareAudioCaptureSection(chromiumHtml)).toBe(screenShareAudioCaptureSection(firefoxHtml)); + expect(chromiumHtml).not.toContain('screenShareAudioCapture (pump)'); + expect(firefoxHtml).not.toContain('screenShareAudioCapture (pump)'); + }); + + it('renders every camera simulcast layer instead of only the projected localVideo row', () => { + const data = createStatsData(); + data.localVideoLayers = [ + {direction: 'send', kind: 'video', ssrc: 1001, rid: 'q', active: false, bitrateKbps: 0, bitrateWindowMs: 2001}, + {direction: 'send', kind: 'video', ssrc: 1002, rid: 'h', active: false, bitrateKbps: 0, bitrateWindowMs: 2001}, + {direction: 'send', kind: 'video', ssrc: 1003, rid: 'f', active: true, bitrateKbps: 1133, bitrateWindowMs: 2001}, + ]; + data.localVideo = data.localVideoLayers[0]; + + const html = renderVoiceDebugStatsHtml(data, '2026-06-10T18:00:00.000Z'); + + expect(html).toContain('track:localVideoLayers[0]'); + expect(html).toContain('track:localVideoLayers[1]'); + expect(html).toContain('track:localVideoLayers[2]'); + expect(html).toContain('bitrateWindowMs'); + }); }); diff --git a/fluxer_app/src/features/voice/diagnostics/VoiceDebugStatsHtml.tsx b/fluxer_app/src/features/voice/diagnostics/VoiceDebugStatsHtml.tsx index 02f214bcd..4724d48e4 100644 --- a/fluxer_app/src/features/voice/diagnostics/VoiceDebugStatsHtml.tsx +++ b/fluxer_app/src/features/voice/diagnostics/VoiceDebugStatsHtml.tsx @@ -233,6 +233,10 @@ function TrackTables({data}: TrackTablesProps) { const tracks: Array<[string, VoiceEngineV2PerTrackStats | null]> = [ ['track:localAudio', data.localAudio], ['track:localVideo', data.localVideo], + ...data.localVideoLayers.map((layer, index): [string, VoiceEngineV2PerTrackStats | null] => [ + `track:localVideoLayers[${index}]`, + layer, + ]), ['track:localScreenShare', data.localScreenShare], ['track:localScreenShareAudio', data.localScreenShareAudio], ['track:remoteAudio', data.remoteAudio], @@ -372,11 +376,6 @@ function VoiceDebugStats({data, generatedAtIso}: VoiceDebugStatsProps) { record={asStatsRecord(data.screenShareSettings)} data-flx="voice.diagnostics.voice-debug-stats-html.voice-debug-stats.stats-table--7" /> - { + logger.warn('Failed to refresh screen share codec negotiation from settings', {error}); + }); + } + + private async refreshScreenShareCodecNegotiationFromCurrentEngine(): Promise { + await ScreenShareCodecNegotiation.publishLocalCapabilities(this.room, 'manual'); + } + private reconcileLocalAudioStateInBackground(reason: string): void { const previous = this.localAudioReconcileCoalescer; const next = transitionVoiceLocalAudioReconcileCoalescerSnapshot(previous, {type: 'run.requested', reason}); diff --git a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts index d8fb37c40..5f76ca5c2 100644 --- a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts +++ b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts @@ -1,13 +1,17 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import type {FluxerCodecAdvertisement} from '@app/features/voice/engine/ScreenShareCodecNegotiation'; +import type { + FluxerCodecAdvertisement, + FluxerSelectProtocolMessage, +} from '@app/features/voice/engine/ScreenShareCodecNegotiation'; import type {HardwareEncodeReport} from '@app/features/voice/utils/GpuEncoderCapabilities'; -import {beforeEach, describe, expect, it, vi} from 'vitest'; +import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; let av1OptIn = false; let hevcOptIn = false; let preferredScreenShareCodec = 'auto'; let gpuReport: HardwareEncodeReport | null = null; +let firefoxBrowser = false; vi.mock('@app/features/voice/state/VoiceSettings', () => ({ default: { @@ -26,7 +30,7 @@ vi.mock('@app/features/ui/utils/NativeUtils', () => ({ guessPlatform: () => 'windows', isChromiumBrowser: () => true, isDesktop: () => true, - isFirefoxBrowser: () => false, + isFirefoxBrowser: () => firefoxBrowser, })); vi.mock('@app/features/voice/utils/GpuEncoderCapabilities', () => ({ @@ -48,6 +52,8 @@ vi.mock('@app/features/voice/utils/OpenH264Status', () => ({ })); vi.mock('@app/features/voice/utils/VideoDecoderCapabilities', () => ({ + clearScreenShareDecodeFailures: () => undefined, + getScreenShareDecodeFailures: () => new Set(), getVideoDecoderExclusionsSync: () => [], loadVideoDecoderExclusions: async () => [], })); @@ -216,6 +222,92 @@ describe('screen-share codec negotiation with the HEVC opt-in off', () => { }); }); +describe('screen-share codec negotiation on Gecko', () => { + beforeEach(() => { + av1OptIn = false; + hevcOptIn = false; + preferredScreenShareCodec = 'auto'; + gpuReport = null; + firefoxBrowser = true; + resetCachedCodecCapabilities(); + }); + + afterEach(() => { + firefoxBrowser = false; + resetCachedCodecCapabilities(); + }); + + it('ranks VP8 ahead of H.264 because Gecko always software-encodes H.264', () => { + expect(getScreenShareCodecPreferenceOrder()).toEqual(['vp8', 'h264']); + }); + + it('negotiates VP8 when both ends can encode and decode VP8 and H.264', () => { + const local = buildLocalCodecAdvertisements(); + const remote = [buildLocalCodecAdvertisements()]; + expect(computeNegotiatedVideoCodec(local, remote, 0, getScreenShareCodecPreferenceOrder()).codec).toBe('vp8'); + }); + + it('still lets an explicit H.264 pin win over the Gecko order', () => { + expect(getScreenShareCodecPreferenceOrder('h264')[0]).toBe('h264'); + }); +}); + +describe('advertising local capabilities over the negotiation topic', () => { + let room: FakeRoom; + + beforeEach(() => { + av1OptIn = false; + hevcOptIn = false; + preferredScreenShareCodec = 'auto'; + gpuReport = {av1: 'hardware', h265: 'hardware', h264: 'hardware', vp9: 'software', vp8: 'software'}; + resetCachedCodecCapabilities(); + room = new FakeRoom(); + ScreenShareCodecNegotiation.setSelectionChangeListener(null); + }); + + async function publishAndCaptureSelectProtocol(): Promise<{ + payload: Uint8Array; + message: FluxerSelectProtocolMessage; + }> { + Object.defineProperty(globalThis, 'window', {configurable: true, writable: true, value: {}}); + try { + ScreenShareCodecNegotiation.bind(room as never); + await ScreenShareCodecNegotiation.publishLocalCapabilities(room as never, 'manual'); + } finally { + Reflect.deleteProperty(globalThis, 'window'); + } + const payloads = (room.localParticipant.publishData.mock.calls as Array>).map( + (call) => call[0] as Uint8Array, + ); + const payload = payloads.findLast( + (candidate) => (JSON.parse(new TextDecoder().decode(candidate)) as {op: number}).op === 1, + ); + if (!payload) throw new Error('no select-protocol message was published'); + return {payload, message: JSON.parse(new TextDecoder().decode(payload)) as FluxerSelectProtocolMessage}; + } + + it('advertises no experiments because nothing on either side reads them', async () => { + const {message} = await publishAndCaptureSelectProtocol(); + expect(message.op).toBe(1); + expect(message.d.experiments).toEqual([]); + ScreenShareCodecNegotiation.dispose(); + }); + + it('keeps the empty experiments list acceptable to the receive-side validator', async () => { + const {payload} = await publishAndCaptureSelectProtocol(); + ScreenShareCodecNegotiation.dispose(); + const peerRoom = new FakeRoom(); + peerRoom.remoteParticipants.set('peer', {identity: 'peer'}); + ScreenShareCodecNegotiation.bind(peerRoom as never); + const listener = vi.fn(); + ScreenShareCodecNegotiation.setSelectionChangeListener(listener); + peerRoom.emit(RoomEvent.DataReceived, payload, {identity: 'peer'}, undefined, SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(listener).toHaveBeenCalled(); + ScreenShareCodecNegotiation.dispose(); + }); +}); + describe('reacting to a mid-session codec selection change', () => { let room: FakeRoom; @@ -267,6 +359,47 @@ describe('reacting to a mid-session codec selection change', () => { ScreenShareCodecNegotiation.dispose(); }); + async function withWindowDefined(run: () => Promise): Promise { + Object.defineProperty(globalThis, 'window', {configurable: true, writable: true, value: {}}); + try { + await run(); + } finally { + Reflect.deleteProperty(globalThis, 'window'); + } + } + + it('stops reusing a live AV1 selection once the AV1 opt-in is turned off mid-share', async () => { + av1OptIn = true; + resetCachedCodecCapabilities(); + await establishBaseline(); + expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('av1'); + av1OptIn = false; + resetCachedCodecCapabilities(); + expect(ScreenShareCodecNegotiation.selectScreenShareCodec('auto')).not.toBe('av1'); + expect(ScreenShareCodecNegotiation.selectNativeScreenShareCodec('auto')).not.toBe('av1'); + expect(getScreenShareCodecPreferenceOrder()).toContain(ScreenShareCodecNegotiation.selectScreenShareCodec('auto')); + ScreenShareCodecNegotiation.dispose(); + }); + + it('renegotiates the live share off AV1 when the opt-in toggle refreshes the local capabilities', async () => { + av1OptIn = true; + resetCachedCodecCapabilities(); + await establishBaseline(); + expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('av1'); + const listener = vi.fn(); + ScreenShareCodecNegotiation.setSelectionChangeListener(listener); + av1OptIn = false; + resetCachedCodecCapabilities(); + await withWindowDefined(async () => { + await ScreenShareCodecNegotiation.publishLocalCapabilities(room as never, 'manual'); + }); + const negotiated = ScreenShareCodecNegotiation.getSelectedCodec(); + expect(negotiated).not.toBe('av1'); + expect(listener).toHaveBeenCalledTimes(1); + expect(listener).toHaveBeenCalledWith(room, negotiated, 'manual'); + ScreenShareCodecNegotiation.dispose(); + }); + it('does not notify the listener when the recomputed selection is unchanged', async () => { await establishBaseline(); room.remoteParticipants.set('peer-with-hevc', {identity: 'peer-with-hevc'}); diff --git a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts index 3ed6fb826..173acbb45 100644 --- a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts +++ b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts @@ -1,6 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import {Logger} from '@app/features/platform/utils/AppLogger'; +import {isFirefoxBrowser} from '@app/features/ui/utils/NativeUtils'; import VoiceSettings from '@app/features/voice/state/VoiceSettings'; import { type CodecCapabilityReport, @@ -17,6 +18,8 @@ import {loadGpuEncoderReport} from '@app/features/voice/utils/GpuEncoderCapabili import {loadNativeHardwareEncoderCapabilities} from '@app/features/voice/utils/NativeHardwareEncoderCapabilities'; import {loadOpenH264Status} from '@app/features/voice/utils/OpenH264Status'; import { + clearScreenShareDecodeFailures, + getScreenShareDecodeFailures, getVideoDecoderExclusionsSync, loadVideoDecoderExclusions, } from '@app/features/voice/utils/VideoDecoderCapabilities'; @@ -39,6 +42,7 @@ const RTP_PAYLOAD_TYPE_MAX = 255; const CODEC_PRIORITY_MAX = 65_535; const CODEC_PREFERENCE: ReadonlyArray = ['av1', 'h265', 'h264', 'vp9', 'vp8']; const SOFTWARE_CODEC_PREFERENCE: ReadonlyArray = ['av1', 'vp9', 'h264', 'vp8', 'h265']; +const GECKO_SOFTWARE_CODEC_PREFERENCE: ReadonlyArray = ['vp8', 'h264']; const COMPATIBILITY_FALLBACK_CODEC_PREFERENCE: ReadonlyArray = ['h264', 'vp9', 'vp8']; const BASELINE_VIDEO_CODEC: VideoCodec = 'vp8'; const VIDEO_CODEC_NAMES: Record = { @@ -163,6 +167,9 @@ function hasReceiverCapability(codec: VideoCodec): boolean | null { function getLocalDecodeCapabilities(): Record { const exclusions = new Set(getVideoDecoderExclusionsSync() ?? []); + for (const codec of getScreenShareDecodeFailures()) { + exclusions.add(codec); + } const result: Record = { av1: false, h265: false, @@ -172,7 +179,7 @@ function getLocalDecodeCapabilities(): Record { }; for (const codec of CODEC_PREFERENCE) { const advertised = hasReceiverCapability(codec); - result[codec] = advertised === null ? result[codec] : advertised && !exclusions.has(codec); + result[codec] = (advertised === null ? result[codec] : advertised) && !exclusions.has(codec); } return result; } @@ -181,8 +188,11 @@ export function getScreenShareCodecPreferenceOrder( preference: CodecPreference = VoiceSettings.getPreferredScreenShareCodec(), ): ReadonlyArray { const encoderMode = resolveEffectiveScreenShareEncoderMode(VoiceSettings.getScreenShareEncoderMode()); - const automaticOrder = - encoderMode === 'software' ? SOFTWARE_CODEC_PREFERENCE : getHardwareFirstScreenShareCodecPreferenceOrder(); + const automaticOrder = isFirefoxBrowser() + ? GECKO_SOFTWARE_CODEC_PREFERENCE + : encoderMode === 'software' + ? SOFTWARE_CODEC_PREFERENCE + : getHardwareFirstScreenShareCodecPreferenceOrder(); const order = preference !== 'auto' ? [preference, ...automaticOrder.filter((codec) => codec !== preference)] : automaticOrder; return order.filter((codec) => isVideoCodecAllowedForPublish(codec)); @@ -537,6 +547,7 @@ class ScreenShareCodecNegotiation { } private canUseSelectedCodecForCurrentParticipants(codec: VideoCodec): boolean { + if (!isVideoCodecAllowedForPublish(codec)) return false; if (!this.canLocalEncode(codec)) return false; const {knownRemoteCodecs, unknownParticipants} = this.getRemoteCodecInputs(); if (unknownParticipants > 0) return false; @@ -631,6 +642,7 @@ class ScreenShareCodecNegotiation { this.remoteCodecsByIdentity.clear(); this.mediaSessionId = createId('media'); this.negotiationSnapshot = createScreenShareCodecNegotiationSnapshot(); + clearScreenShareDecodeFailures(); } async publishLocalCapabilities( @@ -665,7 +677,7 @@ class ScreenShareCodecNegotiation { }, codecs: this.localCodecs, rtc_connection_id: this.rtcConnectionId, - experiments: ['fixed_keyframe_interval', 'maintain_framerate', 'opus_red', 'transport_cc', 'loss_based_bwe_v2'], + experiments: [], }, }; await this.publishMessage(room, message); diff --git a/fluxer_app/src/features/voice/engine/ScreenShareDecodeStallRenegotiation.test.ts b/fluxer_app/src/features/voice/engine/ScreenShareDecodeStallRenegotiation.test.ts new file mode 100644 index 000000000..774d9326b --- /dev/null +++ b/fluxer_app/src/features/voice/engine/ScreenShareDecodeStallRenegotiation.test.ts @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {FluxerCodecAdvertisement} from '@app/features/voice/engine/ScreenShareCodecNegotiation'; +import type {HardwareEncodeReport} from '@app/features/voice/utils/GpuEncoderCapabilities'; +import {beforeEach, describe, expect, it, vi} from 'vitest'; + +const gpuReport: HardwareEncodeReport = { + av1: 'hardware', + h265: 'hardware', + h264: 'hardware', + vp9: 'software', + vp8: 'software', +}; + +vi.mock('@app/features/voice/state/VoiceSettings', () => ({ + default: { + getScreenShareAv1OptIn: () => false, + getScreenShareHevcOptIn: () => false, + getPreferredScreenShareCodec: () => 'auto', + getScreenShareEncoderMode: () => 'auto', + }, +})); + +vi.mock('@app/features/devtools/utils/DesktopTroubleshootingUtils', () => ({ + getCachedDesktopTroubleshootingSettings: () => null, +})); + +vi.mock('@app/features/ui/utils/NativeUtils', () => ({ + guessPlatform: () => 'windows', + isChromiumBrowser: () => true, + isDesktop: () => true, + isFirefoxBrowser: () => false, +})); + +vi.mock('@app/features/voice/utils/GpuEncoderCapabilities', () => ({ + getGpuEncoderReportSync: () => gpuReport, + loadGpuEncoderReport: async () => gpuReport, +})); + +vi.mock('@app/features/voice/utils/NativeHardwareEncoderCapabilities', () => ({ + getNativeHardwareEncoderCapabilitiesSync: () => null, + hasNativeHardwareEncoder: () => false, + resetNativeHardwareEncoderCapabilities: () => undefined, + loadNativeHardwareEncoderCapabilities: async () => null, +})); + +vi.mock('@app/features/voice/utils/OpenH264Status', () => ({ + getOpenH264StatusSync: () => null, + resetOpenH264Status: () => undefined, + loadOpenH264Status: async () => null, +})); + +const VIDEO_CAPABILITIES = { + codecs: [ + {mimeType: 'video/VP8'}, + {mimeType: 'video/VP9'}, + {mimeType: 'video/H264'}, + {mimeType: 'video/H265'}, + {mimeType: 'video/AV1'}, + ], +}; + +Object.defineProperty(globalThis, 'RTCRtpSender', { + configurable: true, + writable: true, + value: {getCapabilities: () => VIDEO_CAPABILITIES}, +}); + +Object.defineProperty(globalThis, 'RTCRtpReceiver', { + configurable: true, + writable: true, + value: {getCapabilities: () => VIDEO_CAPABILITIES}, +}); + +const { + default: ScreenShareCodecNegotiation, + buildLocalCodecAdvertisements, + computeNegotiatedVideoCodec, +} = await import('./ScreenShareCodecNegotiation'); +const {findStalledVideoDecoder} = await import('@app/features/voice/utils/ScreenShareCodecDiagnostics'); +const {getVideoDecoderExclusionsSync, markScreenShareDecodeFailure, resetVideoDecoderExclusions} = await import( + '@app/features/voice/utils/VideoDecoderCapabilities' +); +const {resetCachedCodecCapabilities} = await import('@app/features/voice/utils/CodecCapabilityDetector'); + +function stalledScreenShareStats(mimeType: string): RTCStatsReport { + const entries: Array> = [ + {id: 'codec-1', type: 'codec', mimeType}, + { + id: 'inbound-1', + type: 'inbound-rtp', + kind: 'video', + codecId: 'codec-1', + packetsReceived: 4200, + bytesReceived: 3_500_000, + framesReceived: 180, + framesDecoded: 0, + }, + ]; + return new Map(entries.map((entry) => [entry.id as string, entry])) as unknown as RTCStatsReport; +} + +function decodeAdvertisedFor(name: 'H264' | 'VP8' | 'VP9'): boolean | undefined { + return buildLocalCodecAdvertisements().find((codec) => codec.name === name)?.decode; +} + +function publisherThatEncodes(): Array { + return [ + {name: 'H264', type: 'video', payload_type: 102, priority: 1, encode: true, decode: true}, + {name: 'VP9', type: 'video', payload_type: 109, priority: 2, encode: true, decode: true}, + ]; +} + +describe('a stalled H.264 screen share decode', () => { + beforeEach(() => { + resetVideoDecoderExclusions(); + resetCachedCodecCapabilities(); + ScreenShareCodecNegotiation.dispose(); + }); + + it('is reported as an h264 decode stall by the stats diagnostic', () => { + expect(findStalledVideoDecoder(stalledScreenShareStats('video/H264'))?.codec).toBe('h264'); + }); + + it('withdraws the local H.264 decode advertisement so the publisher renegotiates away from it', () => { + const stall = findStalledVideoDecoder(stalledScreenShareStats('video/H264')); + if (!stall) throw new Error('expected the stats diagnostic to report a stalled decoder'); + expect(stall.codec).toBe('h264'); + expect(decodeAdvertisedFor('H264')).toBe(true); + expect( + computeNegotiatedVideoCodec(publisherThatEncodes(), [buildLocalCodecAdvertisements()], 0, ['h264', 'vp9']).codec, + ).toBe('h264'); + + expect(markScreenShareDecodeFailure(stall.codec, 'screen-share-decode-stalled')).toBe(true); + + expect(decodeAdvertisedFor('H264')).toBe(false); + expect( + computeNegotiatedVideoCodec(publisherThatEncodes(), [buildLocalCodecAdvertisements()], 0, ['h264', 'vp9']).codec, + ).toBe('vp9'); + }); + + it('never removes h264 or vp8 from the SDP-level subscriber exclusions', () => { + markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled'); + expect(getVideoDecoderExclusionsSync() ?? []).not.toContain('h264'); + expect(getVideoDecoderExclusionsSync() ?? []).not.toContain('vp8'); + }); + + it('keeps a non-baseline runtime stall out of the SDP-level subscriber exclusions too', () => { + expect(markScreenShareDecodeFailure('av1', 'screen-share-decode-stalled')).toBe(true); + expect(getVideoDecoderExclusionsSync() ?? []).not.toContain('av1'); + }); + + it('restores the H.264 decode advertisement when the room is torn down', () => { + expect(markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled')).toBe(true); + expect(decodeAdvertisedFor('H264')).toBe(false); + ScreenShareCodecNegotiation.dispose(); + expect(decodeAdvertisedFor('H264')).toBe(true); + }); + + it('stops after one codec change instead of cycling through the remaining codecs', () => { + expect(markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled')).toBe(true); + expect(markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled')).toBe(false); + expect(markScreenShareDecodeFailure('vp8', 'screen-share-decode-stalled')).toBe(false); + expect(decodeAdvertisedFor('VP8')).toBe(true); + }); +}); diff --git a/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts b/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts index 1a6698d20..75f9404c9 100644 --- a/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts +++ b/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts @@ -18,7 +18,7 @@ import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource'; import ParticipantVolume from '@app/features/voice/state/ParticipantVolume'; import {ScreenShareWatchErrorCode, ScreenShareWatchFailures} from '@app/features/voice/state/ScreenShareWatchFailures'; import {scheduleScreenShareDecoderVerification} from '@app/features/voice/utils/ScreenShareCodecDiagnostics'; -import {markVideoDecoderRuntimeFailure} from '@app/features/voice/utils/VideoDecoderCapabilities'; +import {markScreenShareDecodeFailure} from '@app/features/voice/utils/VideoDecoderCapabilities'; import {parseVoiceParticipantIdentity} from '@app/features/voice/utils/VoiceParticipantIdentity'; import type { LocalParticipant, @@ -266,7 +266,7 @@ export function bindRoomEvents( screenShareDecoderVerificationTimers.delete(trackSid); }, (failure) => { - if (!markVideoDecoderRuntimeFailure(failure.codec, 'screen-share-decode-stalled')) return; + if (!markScreenShareDecodeFailure(failure.codec, 'screen-share-decode-stalled')) return; void ScreenShareCodecNegotiation.publishLocalCapabilities(room, 'manual').catch((error) => { logger.warn('Failed to publish updated codec capabilities after decode stall', { error, diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump.ts deleted file mode 100644 index 87d854bee..000000000 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump.ts +++ /dev/null @@ -1,31 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -export interface ScreenShareAudioPumpDiagnostics { - active: boolean; - captureId: string | null; - sampleRate: number | null; - channels: number | null; - usesNativeSink: boolean; - publishStrategy: 'eager' | 'lazy' | 'none'; - publishedFormatKey: string | null; - eagerPublish: 'succeeded' | 'failed' | 'skipped' | null; - eagerPublishError: string | null; - droppedPushFrames: number; - pendingPushFrames: number; -} - -export function getScreenShareAudioPumpDiagnostics(): ScreenShareAudioPumpDiagnostics { - return { - active: false, - captureId: null, - sampleRate: null, - channels: null, - usesNativeSink: false, - publishStrategy: 'none', - publishedFormatKey: null, - eagerPublish: null, - eagerPublishError: null, - droppedPushFrames: 0, - pendingPushFrames: 0, - }; -} diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.test.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.test.ts new file mode 100644 index 000000000..d8d7beb19 --- /dev/null +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.test.ts @@ -0,0 +1,337 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import { + type PerTrackStats, + VoiceEngineV2AppStatsHostAdapter, + type VoiceEngineV2AppStatsHostAdapterScheduler, +} from '@app/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter'; +import {classifyVoiceEngineV2TrackStats} from '@fluxer/voice_engine_v2'; +import type {Room} from 'livekit-client'; +import {describe, expect, it} from 'vitest'; + +interface FakeScheduler extends VoiceEngineV2AppStatsHostAdapterScheduler { + runStatsTick(): void; +} + +function createScheduler(): FakeScheduler { + const handlers = new Map void>(); + return { + setInterval(handler, intervalMs) { + handlers.set(intervalMs, handler); + return intervalMs; + }, + clearInterval(handle) { + handlers.delete(handle as number); + }, + runStatsTick() { + handlers.get(2000)?.(); + }, + }; +} + +function chromiumCameraSimulcastReports(bytesSentByRid: Record): Map { + return new Map([ + [ + 'SV1', + { + type: 'media-source', + id: 'SV1', + kind: 'video', + trackIdentifier: 'cam-track', + width: 1280, + height: 720, + framesPerSecond: 30, + frames: 1801, + }, + ], + ['C1', {type: 'codec', id: 'C1', mimeType: 'video/H264', payloadType: 108}], + [ + 'OT01V1', + { + type: 'outbound-rtp', + id: 'OT01V1', + kind: 'video', + ssrc: 1001, + rid: 'q', + mid: '2', + active: false, + mediaSourceId: 'SV1', + codecId: 'C1', + bytesSent: bytesSentByRid.q, + framesEncoded: 1, + targetBitrate: 0, + qualityLimitationReason: 'none', + }, + ], + [ + 'OT01V2', + { + type: 'outbound-rtp', + id: 'OT01V2', + kind: 'video', + ssrc: 1002, + rid: 'h', + mid: '2', + active: false, + mediaSourceId: 'SV1', + codecId: 'C1', + bytesSent: bytesSentByRid.h, + framesEncoded: 1, + targetBitrate: 0, + qualityLimitationReason: 'none', + }, + ], + [ + 'OT01V3', + { + type: 'outbound-rtp', + id: 'OT01V3', + kind: 'video', + ssrc: 1003, + rid: 'f', + mid: '2', + active: true, + mediaSourceId: 'SV1', + codecId: 'C1', + bytesSent: bytesSentByRid.f, + framesEncoded: 1, + frameWidth: 1280, + frameHeight: 720, + targetBitrate: 1133000, + qualityLimitationReason: 'none', + }, + ], + ]); +} + +async function collectChromiumCameraRows( + samples: ReadonlyArray<{now: number; bytesSentByRid: Record}>, +): Promise> { + const scheduler = createScheduler(); + let sampleIndex = 0; + let now = samples[0].now; + const publisher = { + getStats() { + return Promise.resolve(chromiumCameraSimulcastReports(samples[sampleIndex].bytesSentByRid)); + }, + }; + const adapter = new VoiceEngineV2AppStatsHostAdapter({now: () => now, scheduler}); + adapter.setRoom({engine: {pcManager: {publisher}}, numParticipants: 2} as unknown as Room); + adapter.startStatsTracking(); + for (sampleIndex = 0; sampleIndex < samples.length; sampleIndex += 1) { + now = samples[sampleIndex].now; + scheduler.runStatsTick(); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + } + const rows = adapter.perTrackStats; + adapter.cleanup(); + return rows; +} + +describe('VoiceEngineV2AppStatsHostAdapter chromium outbound rows', () => { + it('collects the outbound-rtp active flag for every simulcast layer', async () => { + const rows = await collectChromiumCameraRows([{now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}}]); + + expect(rows.map((row) => row.rid)).toEqual(['q', 'h', 'f']); + expect(rows.map((row) => row.active)).toEqual([false, false, true]); + }); + + it('omits the bitrate window on a first observation so a zero bitrate is not read as measured', async () => { + const rows = await collectChromiumCameraRows([{now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}}]); + + expect(rows.map((row) => row.bitrateKbps)).toEqual([0, 0, 0]); + expect(rows.map((row) => row.bitrateWindowMs)).toEqual([undefined, undefined, undefined]); + }); + + it('reports the bitrate window once a delta was measured so a zero bitrate is falsifiable', async () => { + const rows = await collectChromiumCameraRows([ + {now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}}, + {now: 3001, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}}, + ]); + + expect(rows.map((row) => row.bitrateKbps)).toEqual([0, 0, 0]); + expect(rows.map((row) => row.bitrateWindowMs)).toEqual([2001, 2001, 2001]); + }); + + it('keeps every pre-existing chromium row field byte-identical', async () => { + const rows = await collectChromiumCameraRows([ + {now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}}, + {now: 3000, bytesSentByRid: {q: 0, h: 0, f: 5_500_000}}, + ]); + const {active, bitrateWindowMs, ...legacy} = rows[2]; + + expect(legacy).toEqual({ + direction: 'send', + kind: 'video', + ssrc: 1003, + rid: 'f', + mid: '2', + trackIdentifier: 'cam-track', + mediaSourceId: 'SV1', + codec: 'video/H264', + payloadType: 108, + bitrateKbps: 2000, + framesEncoded: 1, + frameWidth: 1280, + frameHeight: 720, + sourceFrameWidth: 1280, + sourceFrameHeight: 720, + sourceFramesPerSecond: 30, + sourceFrames: 1801, + targetBitrateKbps: 1133, + qualityLimitationReason: 'none', + encoderAcceleration: 'unknown', + }); + }); +}); + +function firefoxCameraAndScreenShareReports(): Map { + return new Map([ + ['C1', {type: 'codec', id: 'C1', mimeType: 'video/H264', payloadType: 108}], + ['C2', {type: 'codec', id: 'C2', mimeType: 'audio/opus', payloadType: 109}], + [ + 'outbound_rtp_audio_1', + { + type: 'outbound-rtp', + id: 'outbound_rtp_audio_1', + kind: 'audio', + ssrc: 724915693, + mid: '1', + codecId: 'C2', + bytesSent: 100_000, + }, + ], + [ + 'outbound_rtp_video_3', + { + type: 'outbound-rtp', + id: 'outbound_rtp_video_3', + kind: 'video', + ssrc: 4089824766, + mid: '3', + codecId: 'C1', + bytesSent: 5_000_000, + framesEncoded: 900, + }, + ], + [ + 'outbound_rtp_video_4', + { + type: 'outbound-rtp', + id: 'outbound_rtp_video_4', + kind: 'video', + ssrc: 1477030229, + mid: '4', + codecId: 'C1', + bytesSent: 2_000_000, + framesEncoded: 600, + }, + ], + [ + 'inbound_rtp_audio_2', + { + type: 'inbound-rtp', + id: 'inbound_rtp_audio_2', + kind: 'audio', + ssrc: 1661789229, + mid: '2', + codecId: 'C2', + trackIdentifier: '{8a106956-d202-4916-8e19-0a1672c64b73}', + bytesReceived: 50_000, + }, + ], + ]); +} + +async function collectFirefoxRows( + transceivers: ReadonlyArray<{mid: string | null; sender?: {track?: {id: string} | null} | null}> | null, +): Promise> { + const scheduler = createScheduler(); + const publisher = { + getStats() { + return Promise.resolve(firefoxCameraAndScreenShareReports()); + }, + ...(transceivers === null ? {} : {getTransceivers: () => transceivers}), + }; + const adapter = new VoiceEngineV2AppStatsHostAdapter({now: () => 1000, scheduler}); + adapter.setRoom({engine: {pcManager: {publisher}}, numParticipants: 2} as unknown as Room); + adapter.startStatsTracking(); + scheduler.runStatsTick(); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + const rows = adapter.perTrackStats; + adapter.cleanup(); + return rows; +} + +describe('VoiceEngineV2AppStatsHostAdapter firefox outbound rows', () => { + it('backfills the outbound trackIdentifier from the transceiver mid when the report carries neither identity member', async () => { + const rows = await collectFirefoxRows([ + {mid: '1', sender: {track: {id: '{mic-track}'}}}, + {mid: '2', sender: {track: null}}, + {mid: '3', sender: {track: {id: '{camera-track}'}}}, + {mid: '4', sender: {track: {id: '{screen-track}'}}}, + ]); + + expect(rows.map((row) => [row.direction, row.kind, row.mid, row.trackIdentifier])).toEqual([ + ['send', 'audio', '1', '{mic-track}'], + ['send', 'video', '3', '{camera-track}'], + ['send', 'video', '4', '{screen-track}'], + ['recv', 'audio', '2', '{8a106956-d202-4916-8e19-0a1672c64b73}'], + ]); + }); + + it('leaves an outbound row unidentified when no transceiver carries its mid', async () => { + const rows = await collectFirefoxRows([{mid: '3', sender: {track: {id: '{camera-track}'}}}]); + + expect(rows.map((row) => row.trackIdentifier)).toEqual([ + undefined, + '{camera-track}', + undefined, + '{8a106956-d202-4916-8e19-0a1672c64b73}', + ]); + }); + + it('keeps the inbound trackIdentifier when the source exposes no transceivers at all', async () => { + const rows = await collectFirefoxRows(null); + + expect(rows.map((row) => row.trackIdentifier)).toEqual([ + undefined, + undefined, + undefined, + '{8a106956-d202-4916-8e19-0a1672c64b73}', + ]); + }); + + it('lets the classifier tell a firefox camera apart from a simultaneous screen share', async () => { + const rows = await collectFirefoxRows([ + {mid: '1', sender: {track: {id: '{mic-track}'}}}, + {mid: '3', sender: {track: {id: '{camera-track}'}}}, + {mid: '4', sender: {track: {id: '{screen-track}'}}}, + ]); + const classification = classifyVoiceEngineV2TrackStats({ + tracks: rows.map((row) => ({ + direction: row.direction, + kind: row.kind, + rid: row.rid, + trackIdentifier: row.trackIdentifier, + bitrateKbps: row.bitrateKbps, + })), + publications: { + localCameraTrackId: '{camera-track}', + localMicrophoneTrackId: '{mic-track}', + localScreenShareTrackId: '{screen-track}', + localScreenShareAudioTrackId: null, + remoteMicrophoneTrackIds: [], + remoteScreenShareTrackIds: [], + remoteScreenShareAudioTrackIds: [], + }, + }); + + expect(rows[classification.localVideoTrackIndex!].mid).toBe('3'); + expect(rows[classification.localScreenShareTrackIndex!].mid).toBe('4'); + }); +}); diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.ts index 71875c3f2..6d27f3adc 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter.ts @@ -54,12 +54,14 @@ export interface PerTrackStats { kind: 'audio' | 'video' | 'unknown'; ssrc?: number; rid?: string; + active?: boolean; mid?: string; trackIdentifier?: string; mediaSourceId?: string; codec?: string; payloadType?: number; bitrateKbps: number; + bitrateWindowMs?: number; packetsLost?: number; packetsLossPercent?: number; jitterMs?: number; @@ -160,6 +162,12 @@ type RoomWithEngine = Room & { interface StatsSource { getStats(): Promise; + getTransceivers?(): ReadonlyArray; +} + +interface StatsTransceiver { + mid: string | null; + sender?: {track?: {id: string} | null} | null; } interface StatsReportMap { @@ -186,6 +194,7 @@ interface RTCStatsEntry { payloadType?: number; ssrc?: number; rid?: string; + active?: boolean; mid?: string; trackId?: string; trackIdentifier?: string; @@ -248,6 +257,11 @@ const LATENCY_UPDATE_INTERVAL_MS = 2000; const STATS_CLOCK_INTERVAL_MS = 1000; const STATS_UPDATE_INTERVAL_MS = 2000; +interface BitrateSample { + bitrateKbps: number; + windowMs?: number; +} + function normalizeTrackKind(kind: string | undefined): PerTrackStats['kind'] { if (kind === 'audio' || kind === 'video') return kind; return 'unknown'; @@ -264,28 +278,32 @@ function getReportKind(report: RTCStatsEntry, reportsById: Map, -): number { +): BitrateSample { if (typeof currentBytes !== 'number' || !Number.isFinite(currentBytes)) { - return 0; + return {bitrateKbps: 0}; } const previous = rtpCounters.get(reportId); let bitrateKbps = 0; + let windowMs: number | undefined; if (previous?.bytes !== undefined) { const dt = (now - previous.timestamp) / 1000; const db = currentBytes - previous.bytes; - if (dt > 0 && db >= 0) bitrateKbps = (db * 8) / 1000 / dt; + if (dt > 0 && db >= 0) { + bitrateKbps = (db * 8) / 1000 / dt; + windowMs = Math.round(now - previous.timestamp); + } } rtpCounters.set(reportId, { ...previous, bytes: currentBytes, timestamp: now, }); - return bitrateKbps; + return {bitrateKbps, windowMs}; } function hasUsableLossDeltas(lostDelta: number, receivedDelta: number): boolean { @@ -344,6 +362,7 @@ function hasUsableJitterBuffer(report: RTCStatsEntry): boolean { function buildOutboundTrackExtras(report: RTCStatsEntry): Partial { return { + active: report.active, retransmittedPacketsSent: report.retransmittedPacketsSent, retransmittedBytesSent: report.retransmittedBytesSent, keyFramesEncoded: report.keyFramesEncoded, @@ -379,11 +398,12 @@ function buildPerTrackStat(args: { now: number; rtpCounters: Map; reportsById: Map; + midToSenderTrackId: Map; }): PerTrackStats { - const {report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById} = args; + const {report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById, midToSenderTrackId} = args; const id = `${sourceId}:${report.type}:${report.id}`; const currentBytes = isOutbound ? report.bytesSent : report.bytesReceived; - const bitrateKbps = getBitrateKbps(id, currentBytes, now, rtpCounters); + const bitrate = getBitrateSample(id, currentBytes, now, rtpCounters); const codec = report.codecId ? reportsById.get(report.codecId) : undefined; const mediaSource = report.mediaSourceId ? reportsById.get(report.mediaSourceId) : undefined; const packetsLossPercent = isInbound ? getPacketLossPercent(id, report, now, rtpCounters) : undefined; @@ -402,11 +422,15 @@ function buildPerTrackStat(args: { ssrc: report.ssrc, rid: report.rid, mid: report.mid, - trackIdentifier: report.trackIdentifier ?? mediaSource?.trackIdentifier, + trackIdentifier: + report.trackIdentifier ?? + mediaSource?.trackIdentifier ?? + (isOutbound && report.mid ? midToSenderTrackId.get(report.mid) : undefined), mediaSourceId: report.mediaSourceId, codec: codec?.mimeType, payloadType: codec?.payloadType, - bitrateKbps: Math.round(bitrateKbps), + bitrateKbps: Math.round(bitrate.bitrateKbps), + bitrateWindowMs: bitrate.windowMs, packetsLost: report.packetsLost, packetsLossPercent: packetsLossPercent !== undefined ? Math.round(packetsLossPercent * 10) / 10 : undefined, jitterMs: typeof report.jitter === 'number' ? Math.round(report.jitter * 1000 * 10) / 10 : undefined, @@ -485,6 +509,11 @@ async function collectFromStatsSource( transport: TransportInfo | null; }> { const reports = await source.getStats(); + const midToSenderTrackId = new Map(); + for (const transceiver of source.getTransceivers?.() ?? []) { + const senderTrackId = transceiver.sender?.track?.id; + if (transceiver.mid && senderTrackId) midToSenderTrackId.set(transceiver.mid, senderTrackId); + } const reportsById = new Map(); for (const report of reports.values()) { const statsEntry = report as RTCStatsEntry; @@ -511,7 +540,9 @@ async function collectFromStatsSource( if (!isOutbound && !isInbound) continue; const id = `${sourceId}:${report.type}:${report.id}`; activeCounterIds.add(id); - tracks.push(buildPerTrackStat({report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById})); + tracks.push( + buildPerTrackStat({report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById, midToSenderTrackId}), + ); } if (!activePair && transportReport?.selectedCandidatePairId) { const selectedPair = reportsById.get(transportReport.selectedCandidatePairId); diff --git a/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.test.ts b/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.test.ts new file mode 100644 index 000000000..ab6bed850 --- /dev/null +++ b/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.test.ts @@ -0,0 +1,61 @@ +// @vitest-environment happy-dom +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {findStalledVideoEncoder} from '@app/features/voice/engine/voice_screen_share_manager/shared'; +import {describe, expect, test, vi} from 'vitest'; + +vi.mock('@app/features/voice/utils/VideoQualityEntitlement', () => ({hasHigherVideoQuality: () => false})); + +function makeStats(entries: ReadonlyArray>): RTCStatsReport { + return new Map(entries.map((entry) => [entry.id as string, entry])) as unknown as RTCStatsReport; +} + +const CODEC_REPORT = {type: 'codec', id: 'codec-h264', mimeType: 'video/H264'}; +const SOURCE_REPORT = {type: 'media-source', id: 'source-1', kind: 'video', frames: 240, framesPerSecond: 30}; + +function outboundReport(overrides: Record): Record { + return { + type: 'outbound-rtp', + id: 'outbound-1', + kind: 'video', + codecId: 'codec-h264', + mediaSourceId: 'source-1', + framesEncoded: 0, + framesSent: 0, + ...overrides, + }; +} + +describe('findStalledVideoEncoder', () => { + test('reports a stall when the encoding is active and the source is producing frames', () => { + const stats = makeStats([CODEC_REPORT, SOURCE_REPORT, outboundReport({active: true})]); + expect(findStalledVideoEncoder(stats, 'h264')).toEqual({ + codec: 'h264', + framesEncoded: 0, + framesSent: 0, + sourceFrames: 240, + sourceFramesPerSecond: 30, + }); + }); + + test('ignores an encoding the SFU deactivated even though the source is producing frames', () => { + const stats = makeStats([CODEC_REPORT, SOURCE_REPORT, outboundReport({active: false})]); + expect(findStalledVideoEncoder(stats, 'h264')).toBeNull(); + }); + + test('still reports a stall when active is absent, as on Firefox', () => { + const stats = makeStats([CODEC_REPORT, SOURCE_REPORT, outboundReport({})]); + expect(findStalledVideoEncoder(stats, 'h264')?.codec).toBe('h264'); + }); + + test('skips a deactivated layer and reports the still-active sibling instead', () => { + const stats = makeStats([ + CODEC_REPORT, + SOURCE_REPORT, + {type: 'media-source', id: 'source-low', kind: 'video', frames: 90, framesPerSecond: 15}, + outboundReport({id: 'outbound-low', mediaSourceId: 'source-low', active: false}), + outboundReport({id: 'outbound-high', active: true}), + ]); + expect(findStalledVideoEncoder(stats, 'h264')?.sourceFrames).toBe(240); + }); +}); diff --git a/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.ts b/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.ts index be791405f..92ced32df 100644 --- a/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.ts +++ b/fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.ts @@ -469,6 +469,7 @@ interface OutboundVideoStatsEntry { mediaType?: string; codecId?: string; mediaSourceId?: string; + active?: boolean; framesEncoded?: number; framesSent?: number; encoderImplementation?: string; @@ -592,6 +593,7 @@ export function findStalledVideoEncoder(stats: RTCStatsReport, codec?: VideoCode } for (const report of reports) { if (getStatsKind(report, reportsById) !== 'video') continue; + if (report.active === false) continue; const mimeType = report.codecId ? reportsById.get(report.codecId)?.mimeType : undefined; if (report.codecId && !codecMatchesTarget(mimeType, codec)) continue; const resolvedCodec = codec ?? getVideoCodecFromMimeType(mimeType); diff --git a/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.test.ts b/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.test.ts new file mode 100644 index 000000000..1e0db0bf5 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.test.ts @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {getNativeAudioCaptureDiagnosticState} from '@app/features/voice/utils/NativeAudioCaptureBridge'; +import {recordBridgeFrame, startBridgeStats} from '@app/features/voice/utils/native_audio_capture_bridge/bridgeStats'; +import type { + NativeAudioBridgeEndedCapture, + NativeAudioBridgeStats, +} from '@app/features/voice/utils/native_audio_capture_bridge/shared'; +import {describe, expect, it} from 'vitest'; + +describe('getNativeAudioCaptureDiagnosticState', () => { + it('keeps the six fields real dumps already carry and appends the retained histories', () => { + expect(Object.keys(getNativeAudioCaptureDiagnosticState())).toEqual([ + 'armedCapture', + 'activeBridge', + 'supersededBridge', + 'lastStartedCapture', + 'lastArmFailure', + 'bridgeStats', + 'endedBridgeCaptures', + 'lifecycleFaults', + ]); + }); + + it('emits the superseded capture next to the live one instead of only the replacement', () => { + startBridgeStats('generator', 'native-audio:diag-failed', {prebufferTargetUs: 60_000, frameDurationUs: 10_000}); + recordBridgeFrame('native-audio:diag-failed', {timestampUs: 0, durationUs: 10_000, peak: 0.31, rms: 0.08}); + startBridgeStats('generator', 'native-audio:diag-live', {prebufferTargetUs: 60_000, frameDurationUs: 10_000}); + + const state = getNativeAudioCaptureDiagnosticState(); + + expect((state.bridgeStats as NativeAudioBridgeStats).captureId).toBe('native-audio:diag-live'); + const ended = state.endedBridgeCaptures as Array; + const retained = ended.find((capture) => capture.captureId === 'native-audio:diag-failed'); + expect(retained?.framesReceived).toBe(1); + expect(retained?.nonSilentFrameCount).toBe(1); + expect(retained?.endReason).toBe('superseded'); + }); + + it('exposes a native-audio fault history that outlives the lifecycle unbind', () => { + expect(Array.isArray(getNativeAudioCaptureDiagnosticState().lifecycleFaults)).toBe(true); + }); +}); diff --git a/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.ts b/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.ts index 304fb6272..836807bb5 100644 --- a/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.ts +++ b/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.ts @@ -4,7 +4,10 @@ import {Logger} from '@app/features/platform/utils/AppLogger'; import {getElectronAPI} from '@app/features/ui/utils/NativeUtils'; import type {VoiceEngineV2AppSourceLifecycleBridge} from '@app/features/voice/engine/v2/VoiceEngineV2AppSourceLifecycleBridge'; import {getNativeAudioErrorDetail} from '@app/features/voice/utils/NativeAudioFailureUtils'; -import {getBridgeStats as getNativeAudioBridgeStats} from '@app/features/voice/utils/native_audio_capture_bridge/bridgeStats'; +import { + getEndedBridgeCaptures, + getBridgeStats as getNativeAudioBridgeStats, +} from '@app/features/voice/utils/native_audio_capture_bridge/bridgeStats'; import {createGeneratorBridge} from '@app/features/voice/utils/native_audio_capture_bridge/createGeneratorBridge'; import {createScriptProcessorBridge} from '@app/features/voice/utils/native_audio_capture_bridge/createScriptProcessorBridge'; import { @@ -67,8 +70,18 @@ interface NativeAudioStartedCaptureDiagnostic { includeSelfWindowAudio?: boolean; } +interface NativeAudioLifecycleFaultDiagnostic { + captureId: string; + sourceId: string; + message: string; + atMs: number; +} + +const MAX_RETAINED_LIFECYCLE_FAULTS = 8; + let lastStartedCapture: NativeAudioStartedCaptureDiagnostic | null = null; let sourceLifecycleBridge: VoiceEngineV2AppSourceLifecycleBridge | null = null; +let lifecycleFaults: Array = []; const lifecycleBoundCaptureIds = new Set(); export {getNativeAudioBridgeStats}; @@ -93,9 +106,19 @@ function bindNativeAudioCaptureLifecycle(captureId: string): void { } } +function recordLifecycleFault(captureId: string, message: string): void { + lifecycleFaults.push({captureId, sourceId: `native-audio-tap:${captureId}`, message, atMs: Date.now()}); + if (lifecycleFaults.length > MAX_RETAINED_LIFECYCLE_FAULTS) { + lifecycleFaults = lifecycleFaults.slice(-MAX_RETAINED_LIFECYCLE_FAULTS); + } +} + function unbindNativeAudioCaptureLifecycle(captureId: string, faulted: boolean): void { if (!lifecycleBoundCaptureIds.has(captureId)) return; const bridge = sourceLifecycleBridge; + if (faulted) { + recordLifecycleFault(captureId, 'native-audio-tap-track-ended'); + } if (bridge) { if (faulted) { bridge.reportLifecycle({captureId, kind: 'error', message: 'native-audio-tap-track-ended'}); @@ -167,6 +190,8 @@ export function getNativeAudioCaptureDiagnosticState(): Record lastStartedCapture: started, lastArmFailure: getLastNativeAudioArmFailure(), bridgeStats: getNativeAudioBridgeStats(), + endedBridgeCaptures: getEndedBridgeCaptures(), + lifecycleFaults: lifecycleFaults.map((fault) => ({...fault})), }; } @@ -1499,5 +1524,6 @@ export function resetNativeAudioCaptureBridgeForTests(): void { activeBridge = null; supersededBridge = null; sourceLifecycleBridge = null; + lifecycleFaults = []; lifecycleBoundCaptureIds.clear(); } diff --git a/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts b/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts index 58347fdba..943dc7aa8 100644 --- a/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts +++ b/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts @@ -15,7 +15,6 @@ import MediaEngine from '@app/features/voice/engine/MediaEngineFacade'; import ScreenShareCodecNegotiation, { getScreenShareCodecPreferenceOrder, } from '@app/features/voice/engine/ScreenShareCodecNegotiation'; -import {getScreenShareAudioPumpDiagnostics} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump'; import {getNativeEngineAudioTrackPumpStats} from '@app/features/voice/engine/voice_screen_share_manager/NativeEngineAudioTrackPump'; import {getPublishedScreenShareMaxBitrateBps} from '@app/features/voice/engine/voice_screen_share_manager/shared'; import VoiceSettings from '@app/features/voice/state/VoiceSettings'; @@ -36,6 +35,7 @@ import {getScreenShareBitrateBps, resolveStreamingModeSettings} from '@app/featu import {hasHigherVideoQuality} from '@app/features/voice/utils/VideoQualityEntitlement'; import { buildVoiceStatsForNerdsPresentation, + collectScreenShareAudioPublicationDiagnostics, type StatsForNerdsData, } from '@app/features/voice/utils/VoiceStatsForNerdsPresenter'; import type {NativeAudioApplication, VirtmicNode} from '@app/types/electron.d'; @@ -519,8 +519,8 @@ export function collectStatsForNerdsSnapshot(): StatsForNerdsData { openH264Enabled: VoiceSettings.getOpenH264Enabled(), }, screenShareAudioCapture: { - pump: getScreenShareAudioPumpDiagnostics(), nativeCapture: getNativeAudioCaptureDiagnosticState(), + publications: collectScreenShareAudioPublicationDiagnostics(localParticipant), }, appInfo: { appVersion: Config.PUBLIC_BUILD_VERSION ?? 'dev', diff --git a/fluxer_app/src/features/voice/utils/VideoDecoderCapabilities.ts b/fluxer_app/src/features/voice/utils/VideoDecoderCapabilities.ts index 486068b51..56dbd9f7e 100644 --- a/fluxer_app/src/features/voice/utils/VideoDecoderCapabilities.ts +++ b/fluxer_app/src/features/voice/utils/VideoDecoderCapabilities.ts @@ -45,9 +45,11 @@ const CODEC_PROBES: ReadonlyArray = [ }, ]; +const SCREEN_SHARE_DECODE_FAILURES_MAX = 1; + let cachedExclusions: Array | null = null; let pendingProbe: Promise> | null = null; -const runtimeExcludedCodecs = new Set(); +const screenShareDecodeFailures = new Set(); interface VideoDecoderLike { isConfigSupported(config: WebCodecsConfig): Promise<{supported: boolean}>; @@ -80,15 +82,6 @@ function isBaselineWebRtcDecodeCodec(codec: VideoCodec): boolean { return codec === 'h264' || codec === 'vp8'; } -function mergeRuntimeExclusions(exclusions: Array | null): Array | null { - if (runtimeExcludedCodecs.size === 0) return exclusions; - const merged = new Set(exclusions ?? []); - for (const codec of runtimeExcludedCodecs) { - merged.add(codec); - } - return [...merged]; -} - function getPlatformPolicyExclusions(): Array | null { if ((globalThis as Record).RTCRtpReceiver === undefined) return null; const excluded: Array = []; @@ -145,17 +138,17 @@ async function probeAllCodecs(): Promise> { } export function loadVideoDecoderExclusions(): Promise> { - if (cachedExclusions) return Promise.resolve(mergeRuntimeExclusions(cachedExclusions) ?? []); + if (cachedExclusions) return Promise.resolve(cachedExclusions); if (pendingProbe) return pendingProbe; pendingProbe = probeAllCodecs() .then((result) => { cachedExclusions = result; - return mergeRuntimeExclusions(result) ?? []; + return result; }) .catch((error) => { logger.warn('Video decoder probe failed entirely, not excluding any codecs', {error}); cachedExclusions = []; - return mergeRuntimeExclusions([]) ?? []; + return []; }) .finally(() => { pendingProbe = null; @@ -164,21 +157,29 @@ export function loadVideoDecoderExclusions(): Promise> { } export function getVideoDecoderExclusionsSync(): Array | null { - return mergeRuntimeExclusions(cachedExclusions ?? getPlatformPolicyExclusions()); + return cachedExclusions ?? getPlatformPolicyExclusions(); } -export function markVideoDecoderRuntimeFailure(codec: VideoCodec, reason: string): boolean { - if (isBaselineWebRtcDecodeCodec(codec)) return false; - if (runtimeExcludedCodecs.has(codec)) return false; - runtimeExcludedCodecs.add(codec); - logger.warn('Excluding codec from subscriber decode after runtime failure', {codec, reason}); +export function markScreenShareDecodeFailure(codec: VideoCodec, reason: string): boolean { + if (screenShareDecodeFailures.has(codec)) return false; + if (screenShareDecodeFailures.size >= SCREEN_SHARE_DECODE_FAILURES_MAX) return false; + screenShareDecodeFailures.add(codec); + logger.warn('Withdrawing decode advertisement for a codec after a screen share decode stall', {codec, reason}); return true; } +export function getScreenShareDecodeFailures(): ReadonlySet { + return screenShareDecodeFailures; +} + +export function clearScreenShareDecodeFailures(): void { + screenShareDecodeFailures.clear(); +} + export function resetVideoDecoderExclusions(): void { cachedExclusions = null; pendingProbe = null; - runtimeExcludedCodecs.clear(); + screenShareDecodeFailures.clear(); } if (typeof window !== 'undefined') { diff --git a/fluxer_app/src/features/voice/utils/VoiceStatsForNerdsPresenter.test.ts b/fluxer_app/src/features/voice/utils/VoiceStatsForNerdsPresenter.test.ts index de5ffa238..da97e430e 100644 --- a/fluxer_app/src/features/voice/utils/VoiceStatsForNerdsPresenter.test.ts +++ b/fluxer_app/src/features/voice/utils/VoiceStatsForNerdsPresenter.test.ts @@ -2,6 +2,7 @@ import { buildVoiceStatsForNerdsPresentation, + collectScreenShareAudioPublicationDiagnostics, type ParticipantPublicationLookup, } from '@app/features/voice/utils/VoiceStatsForNerdsPresenter'; import type { @@ -86,6 +87,64 @@ function nativeStats(overrides: Partial): VoiceEngineV2Stats return {rttMs: null, outbound: [], inbound: [], ...overrides}; } +function chromiumCameraSimulcastTracks(): Array { + return [ + { + direction: 'send', + kind: 'video', + ssrc: 1001, + rid: 'q', + active: false, + mid: '2', + trackIdentifier: 'cam-track', + mediaSourceId: 'SV1', + bitrateKbps: 0, + bitrateWindowMs: 2001, + framesEncoded: 1, + targetBitrateKbps: 0, + }, + { + direction: 'send', + kind: 'video', + ssrc: 1002, + rid: 'h', + active: false, + mid: '2', + trackIdentifier: 'cam-track', + mediaSourceId: 'SV1', + bitrateKbps: 0, + bitrateWindowMs: 2001, + framesEncoded: 1, + targetBitrateKbps: 0, + }, + { + direction: 'send', + kind: 'video', + ssrc: 1003, + rid: 'f', + active: true, + mid: '2', + trackIdentifier: 'cam-track', + mediaSourceId: 'SV1', + bitrateKbps: 0, + bitrateWindowMs: 2001, + framesEncoded: 1, + frameWidth: 1280, + frameHeight: 720, + targetBitrateKbps: 1133, + }, + { + direction: 'send', + kind: 'video', + ssrc: 2001, + mid: '3', + trackIdentifier: 'screen-track', + mediaSourceId: 'SV2', + bitrateKbps: 3500, + }, + ]; +} + describe('buildVoiceStatsForNerdsPresentation', () => { it('classifies browser per-track stats once for overlay and copy consumers', () => { const localParticipant = participant({ @@ -180,6 +239,203 @@ describe('buildVoiceStatsForNerdsPresentation', () => { expect(presentation.remoteAudio).toBeNull(); }); + it('classifies a firefox screen share whose outbound stats omit the track identifier', () => { + const localParticipant = participant({ + microphone: 'b0d2f1a7-6c3e-4f8a-9b21-5d7c4e0a1f36', + screen_share: '2f4c8de1-9a07-4b53-8c6d-1e5b7a02d94f', + }); + const remoteParticipant = participant({microphone: '8a106956-d202-4916-8e19-0a1672c64b73'}); + const perTrackStats: Array = [ + { + direction: 'recv', + kind: 'audio', + mid: '2', + trackIdentifier: '{8a106956-d202-4916-8e19-0a1672c64b73}', + bitrateKbps: 46, + }, + {direction: 'send', kind: 'audio', mid: '1', bitrateKbps: 30}, + {direction: 'send', kind: 'video', mid: '3', bitrateKbps: 47, framesPerSecond: 58, frameWidth: 1920}, + ]; + + const presentation = buildVoiceStatsForNerdsPresentation({ + connectionId: 'connection-a', + connectionQuality: 'excellent', + currentLatency: 37, + averageLatency: 41, + stats: voiceStats, + perTrackStats, + statsTimeSeries: timeSeries, + nativeStats: null, + publisherTransport: null, + subscriberTransport: null, + localParticipant, + remoteParticipants: [remoteParticipant], + }); + + expect(presentation.localScreenShare?.mid).toBe('3'); + expect(presentation.localVideo).toBeNull(); + }); + + it('classifies a firefox inbound screen share reported with a braced track identifier', () => { + const remoteParticipant = participant({screen_share: '34171e0d-174d-4d56-bb71-76b4fba330c3'}); + const perTrackStats: Array = [ + { + direction: 'recv', + kind: 'video', + mid: '3', + trackIdentifier: '{34171e0d-174d-4d56-bb71-76b4fba330c3}', + bitrateKbps: 3585, + }, + ]; + + const presentation = buildVoiceStatsForNerdsPresentation({ + connectionId: 'connection-a', + connectionQuality: 'excellent', + currentLatency: 37, + averageLatency: 41, + stats: voiceStats, + perTrackStats, + statsTimeSeries: timeSeries, + nativeStats: null, + publisherTransport: null, + subscriberTransport: null, + localParticipant: null, + remoteParticipants: [remoteParticipant], + }); + + expect(presentation.remoteScreenShare?.mid).toBe('3'); + expect(presentation.remoteVideo).toBeNull(); + }); + + it('exposes every camera simulcast layer next to the projected localVideo row', () => { + const localParticipant = participant({camera: 'cam-track', screen_share: 'screen-track'}); + const perTrackStats: Array = chromiumCameraSimulcastTracks(); + + const presentation = buildVoiceStatsForNerdsPresentation({ + connectionId: 'connection-a', + connectionQuality: 'excellent', + currentLatency: 37, + averageLatency: 41, + stats: voiceStats, + perTrackStats, + statsTimeSeries: timeSeries, + nativeStats: null, + publisherTransport: null, + subscriberTransport: null, + localParticipant, + remoteParticipants: null, + }); + + expect(presentation.localVideo?.rid).toBe('q'); + expect(presentation.localVideoLayers).toHaveLength(3); + expect(presentation.localVideoLayers.map((layer) => layer.rid)).toEqual(['q', 'h', 'f']); + expect(presentation.localVideoLayers.map((layer) => layer.active)).toEqual([false, false, true]); + expect(presentation.localVideoLayers.map((layer) => layer.bitrateWindowMs)).toEqual([2001, 2001, 2001]); + }); + + it('keeps the screen-share publication out of the camera layer view', () => { + const localParticipant = participant({camera: 'cam-track', screen_share: 'screen-track'}); + const perTrackStats: Array = chromiumCameraSimulcastTracks(); + + const presentation = buildVoiceStatsForNerdsPresentation({ + connectionId: 'connection-a', + connectionQuality: 'excellent', + currentLatency: 37, + averageLatency: 41, + stats: voiceStats, + perTrackStats, + statsTimeSeries: timeSeries, + nativeStats: null, + publisherTransport: null, + subscriberTransport: null, + localParticipant, + remoteParticipants: null, + }); + + expect(presentation.localScreenShare?.trackIdentifier).toBe('screen-track'); + expect(presentation.localVideoLayers).not.toContain(perTrackStats[3]); + }); + + it('reports a single camera layer when firefox omits rid and media source ids', () => { + const localParticipant = participant({camera: 'cam-track'}); + const perTrackStats: Array = [ + {direction: 'send', kind: 'audio', mid: '1', bitrateKbps: 30}, + {direction: 'send', kind: 'video', mid: '3', bitrateKbps: 47, framesPerSecond: 58, frameWidth: 1920}, + ]; + + const presentation = buildVoiceStatsForNerdsPresentation({ + connectionId: 'connection-a', + connectionQuality: 'excellent', + currentLatency: 37, + averageLatency: 41, + stats: voiceStats, + perTrackStats, + statsTimeSeries: timeSeries, + nativeStats: null, + publisherTransport: null, + subscriberTransport: null, + localParticipant, + remoteParticipants: null, + }); + + expect(presentation.localVideo?.mid).toBe('3'); + expect(presentation.localVideoLayers).toEqual([perTrackStats[1]]); + }); + + it('reports no camera layers rather than a false single layer when firefox simulcast is unidentifiable', () => { + const localParticipant = participant({camera: 'cam-track'}); + const perTrackStats: Array = [ + {direction: 'send', kind: 'audio', mid: '1', bitrateKbps: 30}, + {direction: 'send', kind: 'video', mid: '3', rid: 'q', bitrateKbps: 0}, + {direction: 'send', kind: 'video', mid: '3', rid: 'h', bitrateKbps: 0}, + {direction: 'send', kind: 'video', mid: '3', rid: 'f', bitrateKbps: 1133}, + ]; + + const presentation = buildVoiceStatsForNerdsPresentation({ + connectionId: 'connection-a', + connectionQuality: 'excellent', + currentLatency: 37, + averageLatency: 41, + stats: voiceStats, + perTrackStats, + statsTimeSeries: timeSeries, + nativeStats: null, + publisherTransport: null, + subscriberTransport: null, + localParticipant, + remoteParticipants: null, + }); + + expect(presentation.localVideo).toBeNull(); + expect(presentation.localVideoLayers).toEqual([]); + }); + + it('keeps the chromium simulcast projection byte-identical', () => { + const localParticipant = participant({camera: 'cam-track', screen_share: 'screen-track'}); + const perTrackStats: Array = chromiumCameraSimulcastTracks(); + + const presentation = buildVoiceStatsForNerdsPresentation({ + connectionId: 'connection-a', + connectionQuality: 'excellent', + currentLatency: 37, + averageLatency: 41, + stats: voiceStats, + perTrackStats, + statsTimeSeries: timeSeries, + nativeStats: null, + publisherTransport: null, + subscriberTransport: null, + localParticipant, + remoteParticipants: null, + }); + + expect(presentation.localVideo).toBe(perTrackStats[0]); + expect(presentation.localScreenShare).toBe(perTrackStats[3]); + expect(presentation.localAudio).toBeNull(); + expect(presentation.remoteVideo).toBeNull(); + expect(presentation.network.videoSendBitrateKbps).toBe(1200); + }); + it('uses the canonical v2 native stats projection when native stats are available', () => { const perTrackStats: Array = [ {direction: 'send', kind: 'video', trackIdentifier: 'browser-camera', bitrateKbps: 500}, @@ -309,3 +565,87 @@ describe('buildVoiceStatsForNerdsPresentation', () => { expect(presentation.network.rttMs).toBeNull(); }); }); + +describe('collectScreenShareAudioPublicationDiagnostics', () => { + it('reports the mute and upstream state of every local screen-share audio publication', () => { + const displayPublication = { + trackSid: 'TR_display', + source: 'screen_share_audio', + isMuted: false, + audioTrack: { + isUpstreamPaused: true, + mediaStreamTrack: { + id: 'display-audio', + readyState: 'live', + muted: true, + enabled: true, + } as MediaStreamTrack, + }, + }; + const devicePublication = { + trackSid: 'TR_device', + source: 'screen_share_audio', + isMuted: false, + audioTrack: { + isUpstreamPaused: false, + mediaStreamTrack: { + id: 'device-audio', + readyState: 'live', + muted: false, + enabled: true, + } as MediaStreamTrack, + }, + }; + const micPublication = { + trackSid: 'TR_mic', + source: 'microphone', + isMuted: false, + audioTrack: { + isUpstreamPaused: false, + mediaStreamTrack: { + id: 'mic-audio', + readyState: 'live', + muted: false, + enabled: true, + } as MediaStreamTrack, + }, + }; + const localParticipant: ParticipantPublicationLookup = { + getTrackPublication: (source) => (String(source) === 'screen_share_audio' ? displayPublication : undefined), + audioTrackPublications: new Map([ + ['TR_display', displayPublication], + ['TR_device', devicePublication], + ['TR_mic', micPublication], + ]), + }; + + const diagnostics = collectScreenShareAudioPublicationDiagnostics(localParticipant); + + expect(diagnostics).toEqual([ + { + trackSid: 'TR_display', + source: 'screen_share_audio', + isMuted: false, + isUpstreamPaused: true, + mediaStreamTrackId: 'display-audio', + mediaStreamTrackReadyState: 'live', + mediaStreamTrackMuted: true, + mediaStreamTrackEnabled: true, + }, + { + trackSid: 'TR_device', + source: 'screen_share_audio', + isMuted: false, + isUpstreamPaused: false, + mediaStreamTrackId: 'device-audio', + mediaStreamTrackReadyState: 'live', + mediaStreamTrackMuted: false, + mediaStreamTrackEnabled: true, + }, + ]); + }); + + it('reports nothing when there is no local participant', () => { + expect(collectScreenShareAudioPublicationDiagnostics(null)).toEqual([]); + }); +}); diff --git a/fluxer_app/src/features/voice/utils/VoiceStatsForNerdsPresenter.ts b/fluxer_app/src/features/voice/utils/VoiceStatsForNerdsPresenter.ts index 8441f154b..42b289497 100644 --- a/fluxer_app/src/features/voice/utils/VoiceStatsForNerdsPresenter.ts +++ b/fluxer_app/src/features/voice/utils/VoiceStatsForNerdsPresenter.ts @@ -6,7 +6,6 @@ import { type VoiceTrackPublicationSourceLike, VoiceTrackSource, } from '@app/features/voice/engine/VoiceTrackSource'; -import type {ScreenShareAudioPumpDiagnostics} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump'; import { classifyVoiceEngineV2TrackStats, selectVoiceEngineV2StatsPresentationProjection, @@ -21,6 +20,17 @@ import { } from '@fluxer/voice_engine_v2'; import type {Track} from 'livekit-client'; +export interface ScreenShareAudioPublicationDiagnostic { + trackSid: string | null; + source: string | null; + isMuted: boolean | null; + isUpstreamPaused: boolean | null; + mediaStreamTrackId: string | null; + mediaStreamTrackReadyState: string | null; + mediaStreamTrackMuted: boolean | null; + mediaStreamTrackEnabled: boolean | null; +} + export interface StatsForNerdsData { session: { connectionId: string; @@ -44,6 +54,7 @@ export interface StatsForNerdsData { subscriberTransport: VoiceEngineV2TransportInfo | null; }; localVideo: VoiceEngineV2PerTrackStats | null; + localVideoLayers: Array; localAudio: VoiceEngineV2PerTrackStats | null; localScreenShare: VoiceEngineV2PerTrackStats | null; localScreenShareAudio: VoiceEngineV2PerTrackStats | null; @@ -85,8 +96,8 @@ export interface StatsForNerdsData { openH264Enabled: boolean; }; screenShareAudioCapture: { - pump: ScreenShareAudioPumpDiagnostics; nativeCapture: Record; + publications: Array; }; appInfo: { appVersion: string; @@ -119,6 +130,7 @@ export interface VoiceStatsForNerdsPresentation { session: StatsForNerdsData['session']; network: StatsForNerdsData['network']; localVideo: VoiceEngineV2PerTrackStats | null; + localVideoLayers: Array; localAudio: VoiceEngineV2PerTrackStats | null; localScreenShare: VoiceEngineV2PerTrackStats | null; localScreenShareAudio: VoiceEngineV2PerTrackStats | null; @@ -131,9 +143,12 @@ export interface VoiceStatsForNerdsPresentation { interface PublicationTrackLike { mediaStreamTrack?: MediaStreamTrack; + isUpstreamPaused?: boolean; } interface TrackPublicationLike extends VoiceTrackPublicationSourceLike { + trackSid?: string; + isMuted?: boolean; audioTrack?: PublicationTrackLike | null; videoTrack?: PublicationTrackLike | null; track?: PublicationTrackLike | null; @@ -234,6 +249,38 @@ function collectParticipantScreenShareAudioTrackIdentifiers( return Array.from(identifiers); } +function summarizeScreenShareAudioPublication( + publication: TrackPublicationLike, +): ScreenShareAudioPublicationDiagnostic { + const track = publication.audioTrack ?? publication.track ?? null; + const mediaStreamTrack = track?.mediaStreamTrack ?? null; + return { + trackSid: publication.trackSid ?? null, + source: typeof publication.source === 'string' ? publication.source : null, + isMuted: publication.isMuted ?? null, + isUpstreamPaused: track?.isUpstreamPaused ?? null, + mediaStreamTrackId: mediaStreamTrack?.id ?? null, + mediaStreamTrackReadyState: mediaStreamTrack?.readyState ?? null, + mediaStreamTrackMuted: mediaStreamTrack?.muted ?? null, + mediaStreamTrackEnabled: mediaStreamTrack?.enabled ?? null, + }; +} + +export function collectScreenShareAudioPublicationDiagnostics( + participant: ParticipantPublicationLookup | null | undefined, +): Array { + const publications = new Set(); + const directPublication = participant?.getTrackPublication(SCREEN_SHARE_AUDIO_SOURCE); + if (directPublication) { + publications.add(directPublication); + } + for (const publication of participant?.audioTrackPublications?.values() ?? []) { + if (!isScreenShareAudioPublicationLike(publication)) continue; + publications.add(publication); + } + return Array.from(publications, summarizeScreenShareAudioPublication); +} + function collectRemoteScreenShareAudioTrackIdentifiers( remoteParticipants: Iterable | null | undefined, ): Array { @@ -284,6 +331,25 @@ function getClassifiedTrack( return perTrackStats[index] ?? null; } +function getPublicationGroupKey(track: VoiceEngineV2PerTrackStats): string | null { + if (track.mediaSourceId !== undefined) return `source:${track.mediaSourceId}`; + if (track.trackIdentifier !== undefined) return `track:${track.trackIdentifier}`; + if (track.mid !== undefined) return `mid:${track.mid}`; + return null; +} + +function selectLocalVideoLayers( + perTrackStats: ReadonlyArray, + localVideo: VoiceEngineV2PerTrackStats | null, +): Array { + if (!localVideo) return []; + const groupKey = getPublicationGroupKey(localVideo); + if (groupKey === null) return [localVideo]; + return perTrackStats.filter( + (track) => track.direction === 'send' && track.kind === 'video' && getPublicationGroupKey(track) === groupKey, + ); +} + function selectNativeStatsProjection( nativeStats: VoiceEngineV2Stats | null, ): VoiceEngineV2StatsPresentationProjection | null { @@ -366,6 +432,7 @@ export function buildVoiceStatsForNerdsPresentation( subscriberTransport, }, localVideo: nativeProjection ? nativeProjection.localVideo : localVideo, + localVideoLayers: selectLocalVideoLayers(perTrackStats, localVideo), localAudio: nativeProjection ? nativeProjection.localAudio : localAudio, localScreenShare: nativeProjection ? nativeProjection.localScreenShare : localScreenShare, localScreenShareAudio: nativeProjection ? nativeProjection.localScreenShareAudio : localScreenShareAudio, diff --git a/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/bridgeStats.test.ts b/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/bridgeStats.test.ts new file mode 100644 index 000000000..29defd223 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/bridgeStats.test.ts @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {describe, expect, it, vi} from 'vitest'; +import { + endBridgeStats, + getBridgeStats, + getEndedBridgeCaptures, + recordBridgeFrame, + startBridgeStats, +} from './bridgeStats'; + +function recordFrames(captureId: string, count: number, peak: number, rms: number): void { + for (let index = 0; index < count; index++) { + recordBridgeFrame(captureId, {timestampUs: index * 10_000, durationUs: 10_000, peak, rms}); + } +} + +function findEndedCapture(captureId: string) { + return getEndedBridgeCaptures().find((capture) => capture.captureId === captureId); +} + +describe('bridgeStats', () => { + it('retains the counters of a capture that a mid-share restart supersedes', () => { + startBridgeStats('generator', 'native-audio:failed', {prebufferTargetUs: 60_000, frameDurationUs: 10_000}); + recordFrames('native-audio:failed', 1124, 0.42, 0.11); + + startBridgeStats('generator', 'native-audio:replacement', {prebufferTargetUs: 60_000, frameDurationUs: 10_000}); + + expect(getBridgeStats().captureId).toBe('native-audio:replacement'); + expect(getBridgeStats().framesReceived).toBe(0); + const retained = findEndedCapture('native-audio:failed'); + expect(retained?.bridgeMode).toBe('generator'); + expect(retained?.framesReceived).toBe(1124); + expect(retained?.nonSilentFrameCount).toBe(1124); + expect(retained?.lastFramePeak).toBe(0.42); + expect(retained?.endReason).toBe('superseded'); + }); + + it('records the end reason of a capture the newer capture already superseded', () => { + startBridgeStats('generator', 'native-audio:ends-late'); + recordFrames('native-audio:ends-late', 3, 0.2, 0.05); + startBridgeStats('generator', 'native-audio:ends-late-replacement'); + + endBridgeStats('native-audio:ends-late', 'ended', 'source-disappeared'); + + const retained = findEndedCapture('native-audio:ends-late'); + expect(retained?.endReason).toBe('ended'); + expect(retained?.endDetail).toBe('source-disappeared'); + expect(retained?.endedAt).not.toBeNull(); + expect(getBridgeStats().captureId).toBe('native-audio:ends-late-replacement'); + expect(getBridgeStats().active).toBe(true); + }); + + it('still ends the active capture in place', () => { + startBridgeStats('script-processor', 'native-audio:active-end'); + + endBridgeStats('native-audio:active-end', 'cleanup', 'caller-stopped'); + + const stats = getBridgeStats(); + expect(stats.active).toBe(false); + expect(stats.endReason).toBe('cleanup'); + expect(stats.endDetail).toBe('caller-stopped'); + expect(stats.endedAt).not.toBeNull(); + }); + + it('keeps the first end reason of the active capture', () => { + startBridgeStats('generator', 'native-audio:double-end'); + endBridgeStats('native-audio:double-end', 'ended', 'source-disappeared'); + + endBridgeStats('native-audio:double-end', 'cleanup', 'caller-stopped'); + + expect(getBridgeStats().endReason).toBe('ended'); + expect(getBridgeStats().endDetail).toBe('source-disappeared'); + }); + + it('flags a capture whose frames keep arriving while every one of them is silence', () => { + const startMs = Date.UTC(2026, 8, 8, 12, 0, 0); + vi.useFakeTimers(); + try { + vi.setSystemTime(startMs); + startBridgeStats('generator', 'native-audio:all-silent', {frameDurationUs: 10_000}); + for (let index = 0; index < 400; index++) { + vi.setSystemTime(startMs + index * 100); + recordBridgeFrame('native-audio:all-silent', { + timestampUs: index * 10_000, + durationUs: 10_000, + peak: 0, + rms: 0, + }); + } + + const stats = getBridgeStats(); + expect(stats.framesReceived).toBe(400); + expect(stats.nonSilentFrameCount).toBe(0); + expect(stats.lastNonSilentFrameAt).toBeNull(); + expect(stats.silentFrameStreak).toBe(400); + expect(stats.maxSilentRunMs).toBe(39_900); + expect(stats.sustainedSilenceWarned).toBe(true); + } finally { + vi.useRealTimers(); + } + }); + + it('does not flag a capture that keeps producing audible frames', () => { + const startMs = Date.UTC(2026, 8, 8, 13, 0, 0); + vi.useFakeTimers(); + try { + vi.setSystemTime(startMs); + startBridgeStats('generator', 'native-audio:audible', {frameDurationUs: 10_000}); + for (let index = 0; index < 400; index++) { + vi.setSystemTime(startMs + index * 100); + const audible = index % 4 === 0; + recordBridgeFrame('native-audio:audible', { + timestampUs: index * 10_000, + durationUs: 10_000, + peak: audible ? 0.3 : 0, + rms: audible ? 0.1 : 0, + }); + } + + const stats = getBridgeStats(); + expect(stats.nonSilentFrameCount).toBe(100); + expect(stats.lastNonSilentFrameAt).toBe(startMs + 396 * 100); + expect(stats.silentFrameStreak).toBe(3); + expect(stats.maxSilentRunMs).toBe(300); + expect(stats.sustainedSilenceWarned).toBe(false); + } finally { + vi.useRealTimers(); + } + }); + + it('bounds the retained capture history', () => { + for (let index = 0; index < 12; index++) { + startBridgeStats('generator', `native-audio:bounded-${index}`); + } + + expect(getEndedBridgeCaptures().length).toBeLessThanOrEqual(8); + expect(findEndedCapture('native-audio:bounded-11')).toBeUndefined(); + expect(findEndedCapture('native-audio:bounded-10')).toBeDefined(); + }); +}); diff --git a/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/bridgeStats.ts b/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/bridgeStats.ts index 9501d31a2..9ee952ef0 100644 --- a/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/bridgeStats.ts +++ b/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/bridgeStats.ts @@ -1,22 +1,54 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +import {Logger} from '@app/features/platform/utils/AppLogger'; import { initialBridgeStats, + type NativeAudioBridgeEndedCapture, type NativeAudioBridgeFrameMetrics, type NativeAudioBridgeStats, } from '@app/features/voice/utils/native_audio_capture_bridge/shared'; +const logger = new Logger('NativeAudioBridgeStats'); + +const MAX_ENDED_BRIDGE_CAPTURES = 8; +const SUPERSEDED_END_REASON = 'superseded'; +const SUSTAINED_SILENCE_WARN_MS = 30_000; + let bridgeStats: NativeAudioBridgeStats = {...initialBridgeStats}; +let endedBridgeCaptures: Array = []; export function getBridgeStats(): NativeAudioBridgeStats { return {...bridgeStats}; } +export function getEndedBridgeCaptures(): Array { + return endedBridgeCaptures.map((capture) => ({...capture})); +} + +function retireBridgeStats(): void { + if (bridgeStats.captureId === null) return; + endedBridgeCaptures.push({ + captureId: bridgeStats.captureId, + bridgeMode: bridgeStats.bridgeMode, + startedAt: bridgeStats.startedAt, + endedAt: bridgeStats.endedAt ?? Date.now(), + endReason: bridgeStats.endReason ?? SUPERSEDED_END_REASON, + endDetail: bridgeStats.endDetail, + framesReceived: bridgeStats.framesReceived, + nonSilentFrameCount: bridgeStats.nonSilentFrameCount, + lastFramePeak: bridgeStats.lastFramePeak, + }); + if (endedBridgeCaptures.length > MAX_ENDED_BRIDGE_CAPTURES) { + endedBridgeCaptures = endedBridgeCaptures.slice(-MAX_ENDED_BRIDGE_CAPTURES); + } +} + export function startBridgeStats( mode: 'generator' | 'script-processor', captureId: string, options: {prebufferTargetUs?: number; frameDurationUs?: number} = {}, ): void { + retireBridgeStats(); bridgeStats = { ...initialBridgeStats, active: true, @@ -57,6 +89,24 @@ export function recordBridgeFrame(captureId: string, metrics: NativeAudioBridgeF bridgeStats.maxFrameRms = Math.max(bridgeStats.maxFrameRms, bridgeStats.lastFrameRms); if (peak >= 0.0005 || rms >= 0.0001) { bridgeStats.nonSilentFrameCount += 1; + bridgeStats.lastNonSilentFrameAt = now; + bridgeStats.silentFrameStreak = 0; + } else { + bridgeStats.silentFrameStreak += 1; + const silentSinceMs = bridgeStats.lastNonSilentFrameAt ?? bridgeStats.startedAt ?? now; + const silentRunMs = Math.max(0, now - silentSinceMs); + bridgeStats.maxSilentRunMs = Math.max(bridgeStats.maxSilentRunMs, silentRunMs); + if (!bridgeStats.sustainedSilenceWarned && silentRunMs >= SUSTAINED_SILENCE_WARN_MS) { + bridgeStats.sustainedSilenceWarned = true; + logger.warn('Native audio capture is delivering frames that contain only silence', { + captureId: bridgeStats.captureId, + bridgeMode: bridgeStats.bridgeMode, + framesReceived: bridgeStats.framesReceived, + nonSilentFrameCount: bridgeStats.nonSilentFrameCount, + silentFrameStreak: bridgeStats.silentFrameStreak, + silentRunMs, + }); + } } } } @@ -95,9 +145,16 @@ export function recordBridgeQueue(captureId: string, pendingFrames: number, buff } export function endBridgeStats(captureId: string, reason: string | null, detail: string | null): void { - if (!bridgeStats.active || bridgeStats.captureId !== captureId) return; - bridgeStats.active = false; - bridgeStats.endReason = reason; - bridgeStats.endDetail = detail; - bridgeStats.endedAt = Date.now(); + if (bridgeStats.active && bridgeStats.captureId === captureId) { + bridgeStats.active = false; + bridgeStats.endReason = reason; + bridgeStats.endDetail = detail; + bridgeStats.endedAt = Date.now(); + return; + } + const ended = endedBridgeCaptures.find((capture) => capture.captureId === captureId); + if (!ended || ended.endReason !== SUPERSEDED_END_REASON) return; + ended.endReason = reason; + ended.endDetail = detail; + ended.endedAt = Date.now(); } diff --git a/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/shared.ts b/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/shared.ts index 0d8b2608e..9bed519a9 100644 --- a/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/shared.ts +++ b/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/shared.ts @@ -122,6 +122,10 @@ export interface NativeAudioBridgeStats { maxFramePeak: number; maxFrameRms: number; nonSilentFrameCount: number; + lastNonSilentFrameAt: number | null; + silentFrameStreak: number; + maxSilentRunMs: number; + sustainedSilenceWarned: boolean; prebufferTargetMs: number | null; frameDurationMs: number | null; endReason: string | null; @@ -129,6 +133,18 @@ export interface NativeAudioBridgeStats { endedAt: number | null; } +export interface NativeAudioBridgeEndedCapture { + captureId: string; + bridgeMode: 'generator' | 'script-processor' | null; + startedAt: number | null; + endedAt: number | null; + endReason: string | null; + endDetail: string | null; + framesReceived: number; + nonSilentFrameCount: number; + lastFramePeak: number | null; +} + export interface NativeAudioBridgeFrameMetrics { timestampUs?: number; durationUs?: number; @@ -156,6 +172,10 @@ export const initialBridgeStats: NativeAudioBridgeStats = { maxFramePeak: 0, maxFrameRms: 0, nonSilentFrameCount: 0, + lastNonSilentFrameAt: null, + silentFrameStreak: 0, + maxSilentRunMs: 0, + sustainedSilenceWarned: false, prebufferTargetMs: null, frameDurationMs: null, endReason: null, diff --git a/packages/voice_engine_v2/fixtures/policies/voice_track_classification.json b/packages/voice_engine_v2/fixtures/policies/voice_track_classification.json index ad5c2f1ee..341c7c313 100644 --- a/packages/voice_engine_v2/fixtures/policies/voice_track_classification.json +++ b/packages/voice_engine_v2/fixtures/policies/voice_track_classification.json @@ -119,5 +119,177 @@ "remoteScreenShareTrackIndex": 3, "remoteScreenShareAudioTrackIndex": null } + }, + { + "name": "classifies a firefox screen share whose outbound stats omit the track identifier", + "input": { + "tracks": [ + { + "direction": "recv", + "kind": "audio", + "trackIdentifier": "{8a106956-d202-4916-8e19-0a1672c64b73}", + "bitrateKbps": 46 + }, + {"direction": "send", "kind": "audio", "bitrateKbps": 30}, + {"direction": "send", "kind": "video", "bitrateKbps": 47} + ], + "publications": { + "localCameraTrackId": null, + "localMicrophoneTrackId": "b0d2f1a7-6c3e-4f8a-9b21-5d7c4e0a1f36", + "localScreenShareTrackId": "2f4c8de1-9a07-4b53-8c6d-1e5b7a02d94f", + "localScreenShareAudioTrackId": null, + "remoteMicrophoneTrackIds": ["8a106956-d202-4916-8e19-0a1672c64b73"], + "remoteScreenShareTrackIds": [], + "remoteScreenShareAudioTrackIds": [] + } + }, + "expected": { + "localVideoTrackIndex": null, + "localAudioTrackIndex": 1, + "localScreenShareTrackIndex": 2, + "localScreenShareAudioTrackIndex": null, + "remoteVideoTrackIndex": null, + "remoteAudioTrackIndex": 0, + "remoteScreenShareTrackIndex": null, + "remoteScreenShareAudioTrackIndex": null + } + }, + { + "name": "classifies a firefox inbound screen share reported with a braced track identifier", + "input": { + "tracks": [ + { + "direction": "recv", + "kind": "audio", + "trackIdentifier": "{9010d313-264b-4c51-9f13-9a6aefafce83}", + "bitrateKbps": 29 + }, + { + "direction": "recv", + "kind": "video", + "trackIdentifier": "{34171e0d-174d-4d56-bb71-76b4fba330c3}", + "bitrateKbps": 3585 + }, + {"direction": "send", "kind": "audio", "bitrateKbps": 44} + ], + "publications": { + "localCameraTrackId": null, + "localMicrophoneTrackId": "7c5a1b98-3d24-4e6f-8a10-9b2c3d4e5f60", + "localScreenShareTrackId": null, + "localScreenShareAudioTrackId": null, + "remoteMicrophoneTrackIds": ["9010d313-264b-4c51-9f13-9a6aefafce83"], + "remoteScreenShareTrackIds": ["34171e0d-174d-4d56-bb71-76b4fba330c3"], + "remoteScreenShareAudioTrackIds": [] + } + }, + "expected": { + "localVideoTrackIndex": null, + "localAudioTrackIndex": 2, + "localScreenShareTrackIndex": null, + "localScreenShareAudioTrackIndex": null, + "remoteVideoTrackIndex": null, + "remoteAudioTrackIndex": 0, + "remoteScreenShareTrackIndex": 1, + "remoteScreenShareAudioTrackIndex": null + } + }, + { + "name": "leaves both local video roles unresolved when unidentified sends could be camera or screen share", + "input": { + "tracks": [ + {"direction": "send", "kind": "video", "bitrateKbps": 1600}, + {"direction": "send", "kind": "video", "bitrateKbps": 500} + ], + "publications": { + "localCameraTrackId": "c1e7d3b0-88f4-45a9-9c62-0ab5d1e2f374", + "localMicrophoneTrackId": null, + "localScreenShareTrackId": "2f4c8de1-9a07-4b53-8c6d-1e5b7a02d94f", + "localScreenShareAudioTrackId": null, + "remoteMicrophoneTrackIds": [], + "remoteScreenShareTrackIds": [], + "remoteScreenShareAudioTrackIds": [] + } + }, + "expected": { + "localVideoTrackIndex": null, + "localAudioTrackIndex": null, + "localScreenShareTrackIndex": null, + "localScreenShareAudioTrackIndex": null, + "remoteVideoTrackIndex": null, + "remoteAudioTrackIndex": null, + "remoteScreenShareTrackIndex": null, + "remoteScreenShareAudioTrackIndex": null + } + }, + { + "name": "keeps chromium outbound identifier matching for a screen share published without a camera", + "input": { + "tracks": [ + { + "direction": "send", + "kind": "video", + "trackIdentifier": "97079fec-a3af-4bc0-a8e6-ad142dc05c83", + "bitrateKbps": 1600 + }, + {"direction": "send", "kind": "audio", "bitrateKbps": 30} + ], + "publications": { + "localCameraTrackId": null, + "localMicrophoneTrackId": null, + "localScreenShareTrackId": "97079fec-a3af-4bc0-a8e6-ad142dc05c83", + "localScreenShareAudioTrackId": null, + "remoteMicrophoneTrackIds": [], + "remoteScreenShareTrackIds": [], + "remoteScreenShareAudioTrackIds": [] + } + }, + "expected": { + "localVideoTrackIndex": null, + "localAudioTrackIndex": 1, + "localScreenShareTrackIndex": 0, + "localScreenShareAudioTrackIndex": null, + "remoteVideoTrackIndex": null, + "remoteAudioTrackIndex": null, + "remoteScreenShareTrackIndex": null, + "remoteScreenShareAudioTrackIndex": null + } + }, + { + "name": "keeps chromium identified sends eligible for the camera role while a screen share is published", + "input": { + "tracks": [ + { + "direction": "send", + "kind": "video", + "trackIdentifier": "97079fec-a3af-4bc0-a8e6-ad142dc05c83", + "bitrateKbps": 1600 + }, + { + "direction": "send", + "kind": "video", + "trackIdentifier": "1d0a4c62-5b3e-4f71-88d9-6c2a7e50b431", + "bitrateKbps": 400 + } + ], + "publications": { + "localCameraTrackId": null, + "localMicrophoneTrackId": null, + "localScreenShareTrackId": "97079fec-a3af-4bc0-a8e6-ad142dc05c83", + "localScreenShareAudioTrackId": null, + "remoteMicrophoneTrackIds": [], + "remoteScreenShareTrackIds": [], + "remoteScreenShareAudioTrackIds": [] + } + }, + "expected": { + "localVideoTrackIndex": 1, + "localAudioTrackIndex": null, + "localScreenShareTrackIndex": 0, + "localScreenShareAudioTrackIndex": null, + "remoteVideoTrackIndex": null, + "remoteAudioTrackIndex": null, + "remoteScreenShareTrackIndex": null, + "remoteScreenShareAudioTrackIndex": null + } } ] diff --git a/packages/voice_engine_v2/src/policies/voiceStats.ts b/packages/voice_engine_v2/src/policies/voiceStats.ts index 7b3d61336..97d7a6bde 100644 --- a/packages/voice_engine_v2/src/policies/voiceStats.ts +++ b/packages/voice_engine_v2/src/policies/voiceStats.ts @@ -513,24 +513,55 @@ function normalizedId(id: string | null | undefined): string | null { return trimmed ? trimmed : null; } +function canonicalId(id: string | null | undefined): string | null { + const normalized = normalizedId(id); + if (normalized === null) return null; + const unwrapped = + normalized.startsWith('{') && normalized.endsWith('}') ? normalized.slice(1, -1).trim() : normalized; + return unwrapped ? unwrapped.toLowerCase() : null; +} + function idList(ids: ReadonlyArray): Array { - return ids.map((id) => normalizedId(id)).filter((id): id is string => id !== null); + return ids.map((id) => canonicalId(id)).filter((id): id is string => id !== null); } function matchesId(candidate: IndexedVoiceEngineV2StatsTrack, id: string | null): boolean { - const trackId = normalizedId(candidate.track.trackIdentifier); - return trackId !== null && id !== null && trackId === id; + const trackId = canonicalId(candidate.track.trackIdentifier); + const publicationId = canonicalId(id); + return trackId !== null && publicationId !== null && trackId === publicationId; } function idInList(candidate: IndexedVoiceEngineV2StatsTrack, ids: ReadonlyArray): boolean { - const trackId = normalizedId(candidate.track.trackIdentifier); + const trackId = canonicalId(candidate.track.trackIdentifier); return trackId !== null && ids.includes(trackId); } +function hasId(candidate: IndexedVoiceEngineV2StatsTrack): boolean { + return canonicalId(candidate.track.trackIdentifier) !== null; +} + function hasRid(candidate: IndexedVoiceEngineV2StatsTrack): boolean { return normalizedId(candidate.track.rid) !== null; } +function couldBeLocalScreenShare( + candidate: IndexedVoiceEngineV2StatsTrack, + localScreenShareTrackId: string | null, +): boolean { + if (localScreenShareTrackId === null) return false; + return matchesId(candidate, localScreenShareTrackId) || !hasId(candidate); +} + +function unidentifiedLocalScreenShareIndex( + sentVideoTracks: ReadonlyArray, + localCameraTrackId: string | null, + localScreenShareTrackId: string | null, +): number | null { + if (localScreenShareTrackId === null || localCameraTrackId !== null || sentVideoTracks.length !== 1) return null; + const candidate = sentVideoTracks[0]; + return candidate && !hasId(candidate) ? candidate.index : null; +} + function filterById( candidates: ReadonlyArray, id: string | null, @@ -565,19 +596,25 @@ export function classifyVoiceEngineV2TrackStats( const remoteScreenShareTracks = receivedVideoTracks.filter((track) => idInList(track, remoteScreenShareTrackIds)); const remoteScreenShareTrackIndex = pickActiveTrackIndex(remoteScreenShareTracks); const fallbackRemoteVideoTracks = receivedVideoTracks.filter((track) => !idInList(track, remoteScreenShareTrackIds)); + const localScreenShareFallbackIndex = unidentifiedLocalScreenShareIndex( + sentVideoTracks, + localCameraTrackId, + localScreenShareTrackId, + ); return { localVideoTrackIndex: pickActiveTrackIndex(filterById(sentVideoTracks, localCameraTrackId)) ?? pickActiveTrackIndex( - sentVideoTracks.filter((track) => !hasRid(track) && !matchesId(track, localScreenShareTrackId)), + sentVideoTracks.filter((track) => !hasRid(track) && !couldBeLocalScreenShare(track, localScreenShareTrackId)), ), localAudioTrackIndex: pickActiveTrackIndex(filterById(sentAudioTracks, localMicrophoneTrackId)) ?? pickActiveTrackIndex(sentAudioTracks.filter((track) => !matchesId(track, localScreenShareAudioTrackId))), localScreenShareTrackIndex: pickActiveTrackIndex(filterById(sentVideoTracks, localScreenShareTrackId)) ?? - pickActiveTrackIndex(sentVideoTracks.filter((track) => hasRid(track) && !matchesId(track, localCameraTrackId))), + pickActiveTrackIndex(sentVideoTracks.filter((track) => hasRid(track) && !matchesId(track, localCameraTrackId))) ?? + localScreenShareFallbackIndex, localScreenShareAudioTrackIndex: pickActiveTrackIndex(filterById(sentAudioTracks, localScreenShareAudioTrackId)), remoteVideoTrackIndex: pickActiveTrackIndex(fallbackRemoteVideoTracks) ?? diff --git a/packages/voice_engine_v2/src/protocol/types.ts b/packages/voice_engine_v2/src/protocol/types.ts index d2ca7969f..32dd415c8 100644 --- a/packages/voice_engine_v2/src/protocol/types.ts +++ b/packages/voice_engine_v2/src/protocol/types.ts @@ -499,12 +499,14 @@ export interface VoiceEngineV2PerTrackStats { kind: 'audio' | 'video' | 'unknown'; ssrc?: number; rid?: string; + active?: boolean; mid?: string; trackIdentifier?: string; mediaSourceId?: string; codec?: string; payloadType?: number; bitrateKbps: number; + bitrateWindowMs?: number; packetsLost?: number; packetsLossPercent?: number; jitterMs?: number;