mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-07 19:22:14 +09:00
fix(voice): report real state in voice diagnostics (#2600)
This commit is contained in:
@@ -0,0 +1,101 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {beforeEach, describe, expect, it, vi} from 'vitest';
|
||||
|
||||
const refreshMicrophoneFromSettings = vi.fn();
|
||||
const refreshCameraBackgroundFromSettings = vi.fn();
|
||||
const refreshCameraCaptureFromSettings = vi.fn();
|
||||
const refreshScreenShareCodecNegotiationFromSettings = vi.fn();
|
||||
|
||||
const settings = {
|
||||
cameraResolution: 720,
|
||||
videoDeviceId: 'camera',
|
||||
screenshareResolution: 1080,
|
||||
videoFrameRate: 60,
|
||||
streamingMode: 'smooth',
|
||||
screenShareContentHint: 'auto',
|
||||
preferredScreenShareCodec: 'auto',
|
||||
screenShareAv1OptIn: true,
|
||||
screenShareHevcOptIn: false,
|
||||
screenShareEncoderMode: 'auto',
|
||||
screenShareSoftwareQuality: 'auto',
|
||||
screenShareScalabilityMode: 'auto',
|
||||
screenShareBackupCodecMode: 'auto',
|
||||
openH264Enabled: false,
|
||||
};
|
||||
|
||||
vi.mock('@app/features/voice/engine/MediaEngineFacade', () => ({
|
||||
default: {
|
||||
room: null,
|
||||
refreshMicrophoneFromSettings,
|
||||
refreshCameraBackgroundFromSettings,
|
||||
refreshCameraCaptureFromSettings,
|
||||
refreshScreenShareCodecNegotiationFromSettings,
|
||||
setScreenShareAudioMuted: vi.fn(),
|
||||
applyAllLocalAudioPreferences: vi.fn(),
|
||||
applyLocalInputVolume: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('@app/features/voice/engine/ScreenShareUnderperformance', () => ({
|
||||
noteDeliberateScreenShareQualityChange: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('@app/features/voice/utils/VoiceProcessingProfile', () => ({
|
||||
getActiveInputDeviceLabel: () => null,
|
||||
}));
|
||||
|
||||
vi.mock('@app/features/voice/state/VoiceSettings', () => ({
|
||||
default: {
|
||||
updateSettings: (patch: Record<string, unknown>) => Object.assign(settings, patch),
|
||||
get cameraResolution() {
|
||||
return settings.cameraResolution;
|
||||
},
|
||||
get videoDeviceId() {
|
||||
return settings.videoDeviceId;
|
||||
},
|
||||
getScreenshareResolution: () => settings.screenshareResolution,
|
||||
getVideoFrameRate: () => settings.videoFrameRate,
|
||||
getStreamingMode: () => settings.streamingMode,
|
||||
getScreenShareContentHint: () => settings.screenShareContentHint,
|
||||
getPreferredScreenShareCodec: () => settings.preferredScreenShareCodec,
|
||||
getScreenShareAv1OptIn: () => settings.screenShareAv1OptIn,
|
||||
getScreenShareHevcOptIn: () => settings.screenShareHevcOptIn,
|
||||
getScreenShareEncoderMode: () => settings.screenShareEncoderMode,
|
||||
getScreenShareSoftwareQuality: () => settings.screenShareSoftwareQuality,
|
||||
getScreenShareScalabilityMode: () => settings.screenShareScalabilityMode,
|
||||
getScreenShareBackupCodecMode: () => settings.screenShareBackupCodecMode,
|
||||
getOpenH264Enabled: () => settings.openH264Enabled,
|
||||
},
|
||||
}));
|
||||
|
||||
const VoiceSettingsCommands = await import('./VoiceSettingsCommands');
|
||||
|
||||
describe('screen-share codec settings reactions', () => {
|
||||
beforeEach(() => {
|
||||
settings.screenShareAv1OptIn = true;
|
||||
settings.screenShareHevcOptIn = false;
|
||||
settings.preferredScreenShareCodec = 'auto';
|
||||
refreshScreenShareCodecNegotiationFromSettings.mockClear();
|
||||
});
|
||||
|
||||
it('renegotiates the live screen share when the AV1 opt-in is turned off', () => {
|
||||
VoiceSettingsCommands.update({screenShareAv1OptIn: false});
|
||||
expect(refreshScreenShareCodecNegotiationFromSettings).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('renegotiates the live screen share when the HEVC opt-in changes', () => {
|
||||
VoiceSettingsCommands.update({screenShareHevcOptIn: true});
|
||||
expect(refreshScreenShareCodecNegotiationFromSettings).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('renegotiates the live screen share when the preferred codec changes', () => {
|
||||
VoiceSettingsCommands.update({preferredScreenShareCodec: 'vp9'});
|
||||
expect(refreshScreenShareCodecNegotiationFromSettings).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('leaves the negotiation alone for unrelated screen share settings', () => {
|
||||
VoiceSettingsCommands.update({videoFrameRate: 30});
|
||||
expect(refreshScreenShareCodecNegotiationFromSettings).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -103,6 +103,11 @@ const CAMERA_BACKGROUND_REFRESH_KEYS: Array<keyof VoiceSettingsPatch> = [
|
||||
'mirrorCamera',
|
||||
];
|
||||
const CAMERA_CAPTURE_REFRESH_KEYS: Array<keyof VoiceSettingsPatch> = ['cameraResolution', 'videoDeviceId'];
|
||||
const SCREEN_SHARE_CODEC_NEGOTIATION_REFRESH_KEYS: Array<keyof VoiceSettingsPatch> = [
|
||||
'preferredScreenShareCodec',
|
||||
'screenShareAv1OptIn',
|
||||
'screenShareHevcOptIn',
|
||||
];
|
||||
function refreshMicrophone(): void {
|
||||
MediaEngine.refreshMicrophoneFromSettings();
|
||||
}
|
||||
@@ -115,6 +120,10 @@ function refreshCameraCapture(): void {
|
||||
MediaEngine.refreshCameraCaptureFromSettings();
|
||||
}
|
||||
|
||||
function refreshScreenShareCodecNegotiation(): void {
|
||||
MediaEngine.refreshScreenShareCodecNegotiationFromSettings();
|
||||
}
|
||||
|
||||
function shouldRefreshMicrophone(settings: VoiceSettingsPatch): boolean {
|
||||
return MICROPHONE_REFRESH_KEYS.some((key) => settings[key] !== undefined);
|
||||
}
|
||||
@@ -123,6 +132,10 @@ function shouldRefreshCameraBackground(settings: VoiceSettingsPatch): boolean {
|
||||
return CAMERA_BACKGROUND_REFRESH_KEYS.some((key) => settings[key] !== undefined);
|
||||
}
|
||||
|
||||
function shouldRefreshScreenShareCodecNegotiation(settings: VoiceSettingsPatch): boolean {
|
||||
return SCREEN_SHARE_CODEC_NEGOTIATION_REFRESH_KEYS.some((key) => settings[key] !== undefined);
|
||||
}
|
||||
|
||||
function readCameraCaptureRefreshValues(): VoiceSettingsPatch {
|
||||
return {
|
||||
cameraResolution: VoiceSettings.cameraResolution,
|
||||
@@ -196,6 +209,9 @@ function applyUpdatedVoiceSettings(
|
||||
if (shouldRefreshCameraCapture(settings, cameraCaptureBefore, readCameraCaptureRefreshValues())) {
|
||||
refreshCameraCapture();
|
||||
}
|
||||
if (shouldRefreshScreenShareCodecNegotiation(settings)) {
|
||||
refreshScreenShareCodecNegotiation();
|
||||
}
|
||||
}
|
||||
|
||||
export function setVoiceProcessingModeForDeviceLabel(label: string, mode: VoiceProcessingMode): void {
|
||||
|
||||
@@ -9,7 +9,6 @@ import MediaEngine, {useMediaEngineVersion} from '@app/features/voice/engine/Med
|
||||
import ScreenShareCodecNegotiation, {
|
||||
getScreenShareCodecPreferenceOrder,
|
||||
} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
|
||||
import {getScreenShareAudioPumpDiagnostics} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump';
|
||||
import {getPublishedScreenShareMaxBitrateBps} from '@app/features/voice/engine/voice_screen_share_manager/shared';
|
||||
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
|
||||
import {getNativeAudioCaptureDiagnosticState} from '@app/features/voice/utils/NativeAudioCaptureBridge';
|
||||
@@ -17,6 +16,7 @@ import {getScreenShareBitrateBps, resolveStreamingModeSettings} from '@app/featu
|
||||
import {hasHigherVideoQuality} from '@app/features/voice/utils/VideoQualityEntitlement';
|
||||
import {
|
||||
buildVoiceStatsForNerdsPresentation,
|
||||
collectScreenShareAudioPublicationDiagnostics,
|
||||
type StatsForNerdsData,
|
||||
} from '@app/features/voice/utils/VoiceStatsForNerdsPresenter';
|
||||
import type {VoiceEngineV2PerTrackStats, VoiceEngineV2TransportInfo} from '@fluxer/voice_engine_v2';
|
||||
@@ -216,8 +216,8 @@ export function useStatsForNerds({enabled = true}: UseStatsForNerdsOptions = {})
|
||||
openH264Enabled: VoiceSettings.getOpenH264Enabled(),
|
||||
},
|
||||
screenShareAudioCapture: {
|
||||
pump: getScreenShareAudioPumpDiagnostics(),
|
||||
nativeCapture: getNativeAudioCaptureDiagnosticState(),
|
||||
publications: collectScreenShareAudioPublicationDiagnostics(localParticipant),
|
||||
},
|
||||
appInfo: {
|
||||
appVersion: Config.PUBLIC_BUILD_VERSION ?? 'dev',
|
||||
|
||||
@@ -27,6 +27,7 @@ function createStatsData(): StatsForNerdsData {
|
||||
subscriberTransport: null,
|
||||
},
|
||||
localVideo: null,
|
||||
localVideoLayers: [],
|
||||
localAudio: null,
|
||||
localScreenShare: null,
|
||||
localScreenShareAudio: null,
|
||||
@@ -68,20 +69,8 @@ function createStatsData(): StatsForNerdsData {
|
||||
openH264Enabled: true,
|
||||
},
|
||||
screenShareAudioCapture: {
|
||||
pump: {
|
||||
active: false,
|
||||
captureId: null,
|
||||
sampleRate: null,
|
||||
channels: null,
|
||||
usesNativeSink: false,
|
||||
publishStrategy: 'none',
|
||||
publishedFormatKey: null,
|
||||
eagerPublish: null,
|
||||
eagerPublishError: null,
|
||||
droppedPushFrames: 0,
|
||||
pendingPushFrames: 0,
|
||||
},
|
||||
nativeCapture: {},
|
||||
publications: [],
|
||||
},
|
||||
appInfo: {
|
||||
appVersion: 'dev',
|
||||
@@ -111,6 +100,50 @@ function createStatsData(): StatsForNerdsData {
|
||||
};
|
||||
}
|
||||
|
||||
function createNativeCaptureRecord(): Record<string, unknown> {
|
||||
return {
|
||||
armedCapture: null,
|
||||
activeBridge: {captureId: 'native-audio:live'},
|
||||
supersededBridge: null,
|
||||
lastStartedCapture: null,
|
||||
lastArmFailure: null,
|
||||
bridgeStats: {
|
||||
active: true,
|
||||
bridgeMode: 'generator',
|
||||
captureId: 'native-audio:live',
|
||||
startedAt: 1772000000000,
|
||||
lastFrameAt: 1772000011240,
|
||||
lastFrameTimestampUs: 11240000,
|
||||
framesReceived: 1124,
|
||||
framesDropped: 0,
|
||||
lateFrameCount: 0,
|
||||
rebufferCount: 0,
|
||||
maxFrameArrivalGapMs: 32,
|
||||
maxFrameTimestampGapMs: 10,
|
||||
maxPendingFrames: 3,
|
||||
maxBufferedDurationMs: 30,
|
||||
lastFramePeak: 0.42,
|
||||
lastFrameRms: 0.11,
|
||||
maxFramePeak: 0.71,
|
||||
maxFrameRms: 0.19,
|
||||
nonSilentFrameCount: 1118,
|
||||
prebufferTargetMs: 60,
|
||||
frameDurationMs: 10,
|
||||
endReason: null,
|
||||
endDetail: null,
|
||||
endedAt: null,
|
||||
},
|
||||
endedBridgeCaptures: [],
|
||||
lifecycleFaults: [],
|
||||
};
|
||||
}
|
||||
|
||||
function screenShareAudioCaptureSection(html: string): string {
|
||||
const match = /<section[^>]*><h3[^>]*>screenShareAudioCapture \(native\)<\/h3>[\s\S]*?<\/section>/.exec(html);
|
||||
expect(match).not.toBeNull();
|
||||
return match?.[0] ?? '';
|
||||
}
|
||||
|
||||
describe('renderVoiceDebugStatsHtml', () => {
|
||||
it('renders pure SVG sparklines for key popout metrics', () => {
|
||||
const html = renderVoiceDebugStatsHtml(createStatsData(), '2026-06-10T18:00:00.000Z');
|
||||
@@ -124,4 +157,57 @@ describe('renderVoiceDebugStatsHtml', () => {
|
||||
expect(html).toContain('heapUsedMB sparkline');
|
||||
expect(html).toContain('mainProcessCpuPercent sparkline');
|
||||
});
|
||||
|
||||
it('does not render a screen-share audio pump section while the native capture is live', () => {
|
||||
const data = createStatsData();
|
||||
data.screenShareAudioCapture = {nativeCapture: createNativeCaptureRecord(), publications: []};
|
||||
|
||||
const html = renderVoiceDebugStatsHtml(data, '2026-06-10T18:00:00.000Z');
|
||||
|
||||
expect(html).toContain('screenShareAudioCapture (native)');
|
||||
expect(html).toContain('framesReceived');
|
||||
expect(html).not.toContain('screenShareAudioCapture (pump)');
|
||||
expect(html).not.toContain('publishStrategy');
|
||||
expect(html).not.toContain('droppedPushFrames');
|
||||
expect(html).not.toContain('usesNativeSink');
|
||||
});
|
||||
|
||||
it('renders the screen-share audio capture section identically for Chromium and Firefox dumps', () => {
|
||||
const chromium = createStatsData();
|
||||
chromium.screenShareAudioCapture = {nativeCapture: createNativeCaptureRecord(), publications: []};
|
||||
chromium.system.platform = 'Win32';
|
||||
chromium.system.userAgent =
|
||||
'(Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36';
|
||||
const firefox = createStatsData();
|
||||
firefox.screenShareAudioCapture = {nativeCapture: createNativeCaptureRecord(), publications: []};
|
||||
firefox.system.platform = 'Win32';
|
||||
firefox.system.userAgent = '(Windows NT 10.0; Win64; x64; rv:155.0) Gecko/20100101 Firefox/155.0';
|
||||
firefox.appInfo.electronVersion = null;
|
||||
firefox.appInfo.chromiumVersion = null;
|
||||
firefox.appInfo.hardwareAccelerationEnabled = null;
|
||||
|
||||
const chromiumHtml = renderVoiceDebugStatsHtml(chromium, '2026-06-10T18:00:00.000Z');
|
||||
const firefoxHtml = renderVoiceDebugStatsHtml(firefox, '2026-06-10T18:00:00.000Z');
|
||||
|
||||
expect(screenShareAudioCaptureSection(chromiumHtml)).toBe(screenShareAudioCaptureSection(firefoxHtml));
|
||||
expect(chromiumHtml).not.toContain('screenShareAudioCapture (pump)');
|
||||
expect(firefoxHtml).not.toContain('screenShareAudioCapture (pump)');
|
||||
});
|
||||
|
||||
it('renders every camera simulcast layer instead of only the projected localVideo row', () => {
|
||||
const data = createStatsData();
|
||||
data.localVideoLayers = [
|
||||
{direction: 'send', kind: 'video', ssrc: 1001, rid: 'q', active: false, bitrateKbps: 0, bitrateWindowMs: 2001},
|
||||
{direction: 'send', kind: 'video', ssrc: 1002, rid: 'h', active: false, bitrateKbps: 0, bitrateWindowMs: 2001},
|
||||
{direction: 'send', kind: 'video', ssrc: 1003, rid: 'f', active: true, bitrateKbps: 1133, bitrateWindowMs: 2001},
|
||||
];
|
||||
data.localVideo = data.localVideoLayers[0];
|
||||
|
||||
const html = renderVoiceDebugStatsHtml(data, '2026-06-10T18:00:00.000Z');
|
||||
|
||||
expect(html).toContain('track:localVideoLayers[0]');
|
||||
expect(html).toContain('track:localVideoLayers[1]');
|
||||
expect(html).toContain('track:localVideoLayers[2]');
|
||||
expect(html).toContain('bitrateWindowMs');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -233,6 +233,10 @@ function TrackTables({data}: TrackTablesProps) {
|
||||
const tracks: Array<[string, VoiceEngineV2PerTrackStats | null]> = [
|
||||
['track:localAudio', data.localAudio],
|
||||
['track:localVideo', data.localVideo],
|
||||
...data.localVideoLayers.map((layer, index): [string, VoiceEngineV2PerTrackStats | null] => [
|
||||
`track:localVideoLayers[${index}]`,
|
||||
layer,
|
||||
]),
|
||||
['track:localScreenShare', data.localScreenShare],
|
||||
['track:localScreenShareAudio', data.localScreenShareAudio],
|
||||
['track:remoteAudio', data.remoteAudio],
|
||||
@@ -372,11 +376,6 @@ function VoiceDebugStats({data, generatedAtIso}: VoiceDebugStatsProps) {
|
||||
record={asStatsRecord(data.screenShareSettings)}
|
||||
data-flx="voice.diagnostics.voice-debug-stats-html.voice-debug-stats.stats-table--7"
|
||||
/>
|
||||
<StatsTable
|
||||
title="screenShareAudioCapture (pump)"
|
||||
record={asStatsRecord(data.screenShareAudioCapture.pump)}
|
||||
data-flx="voice.diagnostics.voice-debug-stats-html.voice-debug-stats.stats-table--8"
|
||||
/>
|
||||
<StatsTable
|
||||
title="screenShareAudioCapture (native)"
|
||||
record={asStatsRecord(data.screenShareAudioCapture.nativeCapture)}
|
||||
|
||||
@@ -749,6 +749,16 @@ class MediaEngineFacade extends Store {
|
||||
await voiceEngineV2AppMediaExecutionAdapter.refreshCameraCapture();
|
||||
}
|
||||
|
||||
refreshScreenShareCodecNegotiationFromSettings(): void {
|
||||
void this.refreshScreenShareCodecNegotiationFromCurrentEngine().catch((error) => {
|
||||
logger.warn('Failed to refresh screen share codec negotiation from settings', {error});
|
||||
});
|
||||
}
|
||||
|
||||
private async refreshScreenShareCodecNegotiationFromCurrentEngine(): Promise<void> {
|
||||
await ScreenShareCodecNegotiation.publishLocalCapabilities(this.room, 'manual');
|
||||
}
|
||||
|
||||
private reconcileLocalAudioStateInBackground(reason: string): void {
|
||||
const previous = this.localAudioReconcileCoalescer;
|
||||
const next = transitionVoiceLocalAudioReconcileCoalescerSnapshot(previous, {type: 'run.requested', reason});
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {FluxerCodecAdvertisement} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
|
||||
import type {
|
||||
FluxerCodecAdvertisement,
|
||||
FluxerSelectProtocolMessage,
|
||||
} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
|
||||
import type {HardwareEncodeReport} from '@app/features/voice/utils/GpuEncoderCapabilities';
|
||||
import {beforeEach, describe, expect, it, vi} from 'vitest';
|
||||
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
|
||||
|
||||
let av1OptIn = false;
|
||||
let hevcOptIn = false;
|
||||
let preferredScreenShareCodec = 'auto';
|
||||
let gpuReport: HardwareEncodeReport | null = null;
|
||||
let firefoxBrowser = false;
|
||||
|
||||
vi.mock('@app/features/voice/state/VoiceSettings', () => ({
|
||||
default: {
|
||||
@@ -26,7 +30,7 @@ vi.mock('@app/features/ui/utils/NativeUtils', () => ({
|
||||
guessPlatform: () => 'windows',
|
||||
isChromiumBrowser: () => true,
|
||||
isDesktop: () => true,
|
||||
isFirefoxBrowser: () => false,
|
||||
isFirefoxBrowser: () => firefoxBrowser,
|
||||
}));
|
||||
|
||||
vi.mock('@app/features/voice/utils/GpuEncoderCapabilities', () => ({
|
||||
@@ -48,6 +52,8 @@ vi.mock('@app/features/voice/utils/OpenH264Status', () => ({
|
||||
}));
|
||||
|
||||
vi.mock('@app/features/voice/utils/VideoDecoderCapabilities', () => ({
|
||||
clearScreenShareDecodeFailures: () => undefined,
|
||||
getScreenShareDecodeFailures: () => new Set(),
|
||||
getVideoDecoderExclusionsSync: () => [],
|
||||
loadVideoDecoderExclusions: async () => [],
|
||||
}));
|
||||
@@ -216,6 +222,92 @@ describe('screen-share codec negotiation with the HEVC opt-in off', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('screen-share codec negotiation on Gecko', () => {
|
||||
beforeEach(() => {
|
||||
av1OptIn = false;
|
||||
hevcOptIn = false;
|
||||
preferredScreenShareCodec = 'auto';
|
||||
gpuReport = null;
|
||||
firefoxBrowser = true;
|
||||
resetCachedCodecCapabilities();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
firefoxBrowser = false;
|
||||
resetCachedCodecCapabilities();
|
||||
});
|
||||
|
||||
it('ranks VP8 ahead of H.264 because Gecko always software-encodes H.264', () => {
|
||||
expect(getScreenShareCodecPreferenceOrder()).toEqual(['vp8', 'h264']);
|
||||
});
|
||||
|
||||
it('negotiates VP8 when both ends can encode and decode VP8 and H.264', () => {
|
||||
const local = buildLocalCodecAdvertisements();
|
||||
const remote = [buildLocalCodecAdvertisements()];
|
||||
expect(computeNegotiatedVideoCodec(local, remote, 0, getScreenShareCodecPreferenceOrder()).codec).toBe('vp8');
|
||||
});
|
||||
|
||||
it('still lets an explicit H.264 pin win over the Gecko order', () => {
|
||||
expect(getScreenShareCodecPreferenceOrder('h264')[0]).toBe('h264');
|
||||
});
|
||||
});
|
||||
|
||||
describe('advertising local capabilities over the negotiation topic', () => {
|
||||
let room: FakeRoom;
|
||||
|
||||
beforeEach(() => {
|
||||
av1OptIn = false;
|
||||
hevcOptIn = false;
|
||||
preferredScreenShareCodec = 'auto';
|
||||
gpuReport = {av1: 'hardware', h265: 'hardware', h264: 'hardware', vp9: 'software', vp8: 'software'};
|
||||
resetCachedCodecCapabilities();
|
||||
room = new FakeRoom();
|
||||
ScreenShareCodecNegotiation.setSelectionChangeListener(null);
|
||||
});
|
||||
|
||||
async function publishAndCaptureSelectProtocol(): Promise<{
|
||||
payload: Uint8Array;
|
||||
message: FluxerSelectProtocolMessage;
|
||||
}> {
|
||||
Object.defineProperty(globalThis, 'window', {configurable: true, writable: true, value: {}});
|
||||
try {
|
||||
ScreenShareCodecNegotiation.bind(room as never);
|
||||
await ScreenShareCodecNegotiation.publishLocalCapabilities(room as never, 'manual');
|
||||
} finally {
|
||||
Reflect.deleteProperty(globalThis, 'window');
|
||||
}
|
||||
const payloads = (room.localParticipant.publishData.mock.calls as Array<Array<unknown>>).map(
|
||||
(call) => call[0] as Uint8Array,
|
||||
);
|
||||
const payload = payloads.findLast(
|
||||
(candidate) => (JSON.parse(new TextDecoder().decode(candidate)) as {op: number}).op === 1,
|
||||
);
|
||||
if (!payload) throw new Error('no select-protocol message was published');
|
||||
return {payload, message: JSON.parse(new TextDecoder().decode(payload)) as FluxerSelectProtocolMessage};
|
||||
}
|
||||
|
||||
it('advertises no experiments because nothing on either side reads them', async () => {
|
||||
const {message} = await publishAndCaptureSelectProtocol();
|
||||
expect(message.op).toBe(1);
|
||||
expect(message.d.experiments).toEqual([]);
|
||||
ScreenShareCodecNegotiation.dispose();
|
||||
});
|
||||
|
||||
it('keeps the empty experiments list acceptable to the receive-side validator', async () => {
|
||||
const {payload} = await publishAndCaptureSelectProtocol();
|
||||
ScreenShareCodecNegotiation.dispose();
|
||||
const peerRoom = new FakeRoom();
|
||||
peerRoom.remoteParticipants.set('peer', {identity: 'peer'});
|
||||
ScreenShareCodecNegotiation.bind(peerRoom as never);
|
||||
const listener = vi.fn();
|
||||
ScreenShareCodecNegotiation.setSelectionChangeListener(listener);
|
||||
peerRoom.emit(RoomEvent.DataReceived, payload, {identity: 'peer'}, undefined, SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC);
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(listener).toHaveBeenCalled();
|
||||
ScreenShareCodecNegotiation.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
describe('reacting to a mid-session codec selection change', () => {
|
||||
let room: FakeRoom;
|
||||
|
||||
@@ -267,6 +359,47 @@ describe('reacting to a mid-session codec selection change', () => {
|
||||
ScreenShareCodecNegotiation.dispose();
|
||||
});
|
||||
|
||||
async function withWindowDefined(run: () => Promise<void>): Promise<void> {
|
||||
Object.defineProperty(globalThis, 'window', {configurable: true, writable: true, value: {}});
|
||||
try {
|
||||
await run();
|
||||
} finally {
|
||||
Reflect.deleteProperty(globalThis, 'window');
|
||||
}
|
||||
}
|
||||
|
||||
it('stops reusing a live AV1 selection once the AV1 opt-in is turned off mid-share', async () => {
|
||||
av1OptIn = true;
|
||||
resetCachedCodecCapabilities();
|
||||
await establishBaseline();
|
||||
expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('av1');
|
||||
av1OptIn = false;
|
||||
resetCachedCodecCapabilities();
|
||||
expect(ScreenShareCodecNegotiation.selectScreenShareCodec('auto')).not.toBe('av1');
|
||||
expect(ScreenShareCodecNegotiation.selectNativeScreenShareCodec('auto')).not.toBe('av1');
|
||||
expect(getScreenShareCodecPreferenceOrder()).toContain(ScreenShareCodecNegotiation.selectScreenShareCodec('auto'));
|
||||
ScreenShareCodecNegotiation.dispose();
|
||||
});
|
||||
|
||||
it('renegotiates the live share off AV1 when the opt-in toggle refreshes the local capabilities', async () => {
|
||||
av1OptIn = true;
|
||||
resetCachedCodecCapabilities();
|
||||
await establishBaseline();
|
||||
expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('av1');
|
||||
const listener = vi.fn();
|
||||
ScreenShareCodecNegotiation.setSelectionChangeListener(listener);
|
||||
av1OptIn = false;
|
||||
resetCachedCodecCapabilities();
|
||||
await withWindowDefined(async () => {
|
||||
await ScreenShareCodecNegotiation.publishLocalCapabilities(room as never, 'manual');
|
||||
});
|
||||
const negotiated = ScreenShareCodecNegotiation.getSelectedCodec();
|
||||
expect(negotiated).not.toBe('av1');
|
||||
expect(listener).toHaveBeenCalledTimes(1);
|
||||
expect(listener).toHaveBeenCalledWith(room, negotiated, 'manual');
|
||||
ScreenShareCodecNegotiation.dispose();
|
||||
});
|
||||
|
||||
it('does not notify the listener when the recomputed selection is unchanged', async () => {
|
||||
await establishBaseline();
|
||||
room.remoteParticipants.set('peer-with-hevc', {identity: 'peer-with-hevc'});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {Logger} from '@app/features/platform/utils/AppLogger';
|
||||
import {isFirefoxBrowser} from '@app/features/ui/utils/NativeUtils';
|
||||
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
|
||||
import {
|
||||
type CodecCapabilityReport,
|
||||
@@ -17,6 +18,8 @@ import {loadGpuEncoderReport} from '@app/features/voice/utils/GpuEncoderCapabili
|
||||
import {loadNativeHardwareEncoderCapabilities} from '@app/features/voice/utils/NativeHardwareEncoderCapabilities';
|
||||
import {loadOpenH264Status} from '@app/features/voice/utils/OpenH264Status';
|
||||
import {
|
||||
clearScreenShareDecodeFailures,
|
||||
getScreenShareDecodeFailures,
|
||||
getVideoDecoderExclusionsSync,
|
||||
loadVideoDecoderExclusions,
|
||||
} from '@app/features/voice/utils/VideoDecoderCapabilities';
|
||||
@@ -39,6 +42,7 @@ const RTP_PAYLOAD_TYPE_MAX = 255;
|
||||
const CODEC_PRIORITY_MAX = 65_535;
|
||||
const CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['av1', 'h265', 'h264', 'vp9', 'vp8'];
|
||||
const SOFTWARE_CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['av1', 'vp9', 'h264', 'vp8', 'h265'];
|
||||
const GECKO_SOFTWARE_CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['vp8', 'h264'];
|
||||
const COMPATIBILITY_FALLBACK_CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['h264', 'vp9', 'vp8'];
|
||||
const BASELINE_VIDEO_CODEC: VideoCodec = 'vp8';
|
||||
const VIDEO_CODEC_NAMES: Record<VideoCodec, FluxerVideoCodecName> = {
|
||||
@@ -163,6 +167,9 @@ function hasReceiverCapability(codec: VideoCodec): boolean | null {
|
||||
|
||||
function getLocalDecodeCapabilities(): Record<VideoCodec, boolean> {
|
||||
const exclusions = new Set(getVideoDecoderExclusionsSync() ?? []);
|
||||
for (const codec of getScreenShareDecodeFailures()) {
|
||||
exclusions.add(codec);
|
||||
}
|
||||
const result: Record<VideoCodec, boolean> = {
|
||||
av1: false,
|
||||
h265: false,
|
||||
@@ -172,7 +179,7 @@ function getLocalDecodeCapabilities(): Record<VideoCodec, boolean> {
|
||||
};
|
||||
for (const codec of CODEC_PREFERENCE) {
|
||||
const advertised = hasReceiverCapability(codec);
|
||||
result[codec] = advertised === null ? result[codec] : advertised && !exclusions.has(codec);
|
||||
result[codec] = (advertised === null ? result[codec] : advertised) && !exclusions.has(codec);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -181,8 +188,11 @@ export function getScreenShareCodecPreferenceOrder(
|
||||
preference: CodecPreference = VoiceSettings.getPreferredScreenShareCodec(),
|
||||
): ReadonlyArray<VideoCodec> {
|
||||
const encoderMode = resolveEffectiveScreenShareEncoderMode(VoiceSettings.getScreenShareEncoderMode());
|
||||
const automaticOrder =
|
||||
encoderMode === 'software' ? SOFTWARE_CODEC_PREFERENCE : getHardwareFirstScreenShareCodecPreferenceOrder();
|
||||
const automaticOrder = isFirefoxBrowser()
|
||||
? GECKO_SOFTWARE_CODEC_PREFERENCE
|
||||
: encoderMode === 'software'
|
||||
? SOFTWARE_CODEC_PREFERENCE
|
||||
: getHardwareFirstScreenShareCodecPreferenceOrder();
|
||||
const order =
|
||||
preference !== 'auto' ? [preference, ...automaticOrder.filter((codec) => codec !== preference)] : automaticOrder;
|
||||
return order.filter((codec) => isVideoCodecAllowedForPublish(codec));
|
||||
@@ -537,6 +547,7 @@ class ScreenShareCodecNegotiation {
|
||||
}
|
||||
|
||||
private canUseSelectedCodecForCurrentParticipants(codec: VideoCodec): boolean {
|
||||
if (!isVideoCodecAllowedForPublish(codec)) return false;
|
||||
if (!this.canLocalEncode(codec)) return false;
|
||||
const {knownRemoteCodecs, unknownParticipants} = this.getRemoteCodecInputs();
|
||||
if (unknownParticipants > 0) return false;
|
||||
@@ -631,6 +642,7 @@ class ScreenShareCodecNegotiation {
|
||||
this.remoteCodecsByIdentity.clear();
|
||||
this.mediaSessionId = createId('media');
|
||||
this.negotiationSnapshot = createScreenShareCodecNegotiationSnapshot();
|
||||
clearScreenShareDecodeFailures();
|
||||
}
|
||||
|
||||
async publishLocalCapabilities(
|
||||
@@ -665,7 +677,7 @@ class ScreenShareCodecNegotiation {
|
||||
},
|
||||
codecs: this.localCodecs,
|
||||
rtc_connection_id: this.rtcConnectionId,
|
||||
experiments: ['fixed_keyframe_interval', 'maintain_framerate', 'opus_red', 'transport_cc', 'loss_based_bwe_v2'],
|
||||
experiments: [],
|
||||
},
|
||||
};
|
||||
await this.publishMessage(room, message);
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {FluxerCodecAdvertisement} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
|
||||
import type {HardwareEncodeReport} from '@app/features/voice/utils/GpuEncoderCapabilities';
|
||||
import {beforeEach, describe, expect, it, vi} from 'vitest';
|
||||
|
||||
const gpuReport: HardwareEncodeReport = {
|
||||
av1: 'hardware',
|
||||
h265: 'hardware',
|
||||
h264: 'hardware',
|
||||
vp9: 'software',
|
||||
vp8: 'software',
|
||||
};
|
||||
|
||||
vi.mock('@app/features/voice/state/VoiceSettings', () => ({
|
||||
default: {
|
||||
getScreenShareAv1OptIn: () => false,
|
||||
getScreenShareHevcOptIn: () => false,
|
||||
getPreferredScreenShareCodec: () => 'auto',
|
||||
getScreenShareEncoderMode: () => 'auto',
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('@app/features/devtools/utils/DesktopTroubleshootingUtils', () => ({
|
||||
getCachedDesktopTroubleshootingSettings: () => null,
|
||||
}));
|
||||
|
||||
vi.mock('@app/features/ui/utils/NativeUtils', () => ({
|
||||
guessPlatform: () => 'windows',
|
||||
isChromiumBrowser: () => true,
|
||||
isDesktop: () => true,
|
||||
isFirefoxBrowser: () => false,
|
||||
}));
|
||||
|
||||
vi.mock('@app/features/voice/utils/GpuEncoderCapabilities', () => ({
|
||||
getGpuEncoderReportSync: () => gpuReport,
|
||||
loadGpuEncoderReport: async () => gpuReport,
|
||||
}));
|
||||
|
||||
vi.mock('@app/features/voice/utils/NativeHardwareEncoderCapabilities', () => ({
|
||||
getNativeHardwareEncoderCapabilitiesSync: () => null,
|
||||
hasNativeHardwareEncoder: () => false,
|
||||
resetNativeHardwareEncoderCapabilities: () => undefined,
|
||||
loadNativeHardwareEncoderCapabilities: async () => null,
|
||||
}));
|
||||
|
||||
vi.mock('@app/features/voice/utils/OpenH264Status', () => ({
|
||||
getOpenH264StatusSync: () => null,
|
||||
resetOpenH264Status: () => undefined,
|
||||
loadOpenH264Status: async () => null,
|
||||
}));
|
||||
|
||||
const VIDEO_CAPABILITIES = {
|
||||
codecs: [
|
||||
{mimeType: 'video/VP8'},
|
||||
{mimeType: 'video/VP9'},
|
||||
{mimeType: 'video/H264'},
|
||||
{mimeType: 'video/H265'},
|
||||
{mimeType: 'video/AV1'},
|
||||
],
|
||||
};
|
||||
|
||||
Object.defineProperty(globalThis, 'RTCRtpSender', {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: {getCapabilities: () => VIDEO_CAPABILITIES},
|
||||
});
|
||||
|
||||
Object.defineProperty(globalThis, 'RTCRtpReceiver', {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: {getCapabilities: () => VIDEO_CAPABILITIES},
|
||||
});
|
||||
|
||||
const {
|
||||
default: ScreenShareCodecNegotiation,
|
||||
buildLocalCodecAdvertisements,
|
||||
computeNegotiatedVideoCodec,
|
||||
} = await import('./ScreenShareCodecNegotiation');
|
||||
const {findStalledVideoDecoder} = await import('@app/features/voice/utils/ScreenShareCodecDiagnostics');
|
||||
const {getVideoDecoderExclusionsSync, markScreenShareDecodeFailure, resetVideoDecoderExclusions} = await import(
|
||||
'@app/features/voice/utils/VideoDecoderCapabilities'
|
||||
);
|
||||
const {resetCachedCodecCapabilities} = await import('@app/features/voice/utils/CodecCapabilityDetector');
|
||||
|
||||
function stalledScreenShareStats(mimeType: string): RTCStatsReport {
|
||||
const entries: Array<Record<string, unknown>> = [
|
||||
{id: 'codec-1', type: 'codec', mimeType},
|
||||
{
|
||||
id: 'inbound-1',
|
||||
type: 'inbound-rtp',
|
||||
kind: 'video',
|
||||
codecId: 'codec-1',
|
||||
packetsReceived: 4200,
|
||||
bytesReceived: 3_500_000,
|
||||
framesReceived: 180,
|
||||
framesDecoded: 0,
|
||||
},
|
||||
];
|
||||
return new Map(entries.map((entry) => [entry.id as string, entry])) as unknown as RTCStatsReport;
|
||||
}
|
||||
|
||||
function decodeAdvertisedFor(name: 'H264' | 'VP8' | 'VP9'): boolean | undefined {
|
||||
return buildLocalCodecAdvertisements().find((codec) => codec.name === name)?.decode;
|
||||
}
|
||||
|
||||
function publisherThatEncodes(): Array<FluxerCodecAdvertisement> {
|
||||
return [
|
||||
{name: 'H264', type: 'video', payload_type: 102, priority: 1, encode: true, decode: true},
|
||||
{name: 'VP9', type: 'video', payload_type: 109, priority: 2, encode: true, decode: true},
|
||||
];
|
||||
}
|
||||
|
||||
describe('a stalled H.264 screen share decode', () => {
|
||||
beforeEach(() => {
|
||||
resetVideoDecoderExclusions();
|
||||
resetCachedCodecCapabilities();
|
||||
ScreenShareCodecNegotiation.dispose();
|
||||
});
|
||||
|
||||
it('is reported as an h264 decode stall by the stats diagnostic', () => {
|
||||
expect(findStalledVideoDecoder(stalledScreenShareStats('video/H264'))?.codec).toBe('h264');
|
||||
});
|
||||
|
||||
it('withdraws the local H.264 decode advertisement so the publisher renegotiates away from it', () => {
|
||||
const stall = findStalledVideoDecoder(stalledScreenShareStats('video/H264'));
|
||||
if (!stall) throw new Error('expected the stats diagnostic to report a stalled decoder');
|
||||
expect(stall.codec).toBe('h264');
|
||||
expect(decodeAdvertisedFor('H264')).toBe(true);
|
||||
expect(
|
||||
computeNegotiatedVideoCodec(publisherThatEncodes(), [buildLocalCodecAdvertisements()], 0, ['h264', 'vp9']).codec,
|
||||
).toBe('h264');
|
||||
|
||||
expect(markScreenShareDecodeFailure(stall.codec, 'screen-share-decode-stalled')).toBe(true);
|
||||
|
||||
expect(decodeAdvertisedFor('H264')).toBe(false);
|
||||
expect(
|
||||
computeNegotiatedVideoCodec(publisherThatEncodes(), [buildLocalCodecAdvertisements()], 0, ['h264', 'vp9']).codec,
|
||||
).toBe('vp9');
|
||||
});
|
||||
|
||||
it('never removes h264 or vp8 from the SDP-level subscriber exclusions', () => {
|
||||
markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled');
|
||||
expect(getVideoDecoderExclusionsSync() ?? []).not.toContain('h264');
|
||||
expect(getVideoDecoderExclusionsSync() ?? []).not.toContain('vp8');
|
||||
});
|
||||
|
||||
it('keeps a non-baseline runtime stall out of the SDP-level subscriber exclusions too', () => {
|
||||
expect(markScreenShareDecodeFailure('av1', 'screen-share-decode-stalled')).toBe(true);
|
||||
expect(getVideoDecoderExclusionsSync() ?? []).not.toContain('av1');
|
||||
});
|
||||
|
||||
it('restores the H.264 decode advertisement when the room is torn down', () => {
|
||||
expect(markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled')).toBe(true);
|
||||
expect(decodeAdvertisedFor('H264')).toBe(false);
|
||||
ScreenShareCodecNegotiation.dispose();
|
||||
expect(decodeAdvertisedFor('H264')).toBe(true);
|
||||
});
|
||||
|
||||
it('stops after one codec change instead of cycling through the remaining codecs', () => {
|
||||
expect(markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled')).toBe(true);
|
||||
expect(markScreenShareDecodeFailure('h264', 'screen-share-decode-stalled')).toBe(false);
|
||||
expect(markScreenShareDecodeFailure('vp8', 'screen-share-decode-stalled')).toBe(false);
|
||||
expect(decodeAdvertisedFor('VP8')).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -18,7 +18,7 @@ import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
|
||||
import ParticipantVolume from '@app/features/voice/state/ParticipantVolume';
|
||||
import {ScreenShareWatchErrorCode, ScreenShareWatchFailures} from '@app/features/voice/state/ScreenShareWatchFailures';
|
||||
import {scheduleScreenShareDecoderVerification} from '@app/features/voice/utils/ScreenShareCodecDiagnostics';
|
||||
import {markVideoDecoderRuntimeFailure} from '@app/features/voice/utils/VideoDecoderCapabilities';
|
||||
import {markScreenShareDecodeFailure} from '@app/features/voice/utils/VideoDecoderCapabilities';
|
||||
import {parseVoiceParticipantIdentity} from '@app/features/voice/utils/VoiceParticipantIdentity';
|
||||
import type {
|
||||
LocalParticipant,
|
||||
@@ -266,7 +266,7 @@ export function bindRoomEvents(
|
||||
screenShareDecoderVerificationTimers.delete(trackSid);
|
||||
},
|
||||
(failure) => {
|
||||
if (!markVideoDecoderRuntimeFailure(failure.codec, 'screen-share-decode-stalled')) return;
|
||||
if (!markScreenShareDecodeFailure(failure.codec, 'screen-share-decode-stalled')) return;
|
||||
void ScreenShareCodecNegotiation.publishLocalCapabilities(room, 'manual').catch((error) => {
|
||||
logger.warn('Failed to publish updated codec capabilities after decode stall', {
|
||||
error,
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
export interface ScreenShareAudioPumpDiagnostics {
|
||||
active: boolean;
|
||||
captureId: string | null;
|
||||
sampleRate: number | null;
|
||||
channels: number | null;
|
||||
usesNativeSink: boolean;
|
||||
publishStrategy: 'eager' | 'lazy' | 'none';
|
||||
publishedFormatKey: string | null;
|
||||
eagerPublish: 'succeeded' | 'failed' | 'skipped' | null;
|
||||
eagerPublishError: string | null;
|
||||
droppedPushFrames: number;
|
||||
pendingPushFrames: number;
|
||||
}
|
||||
|
||||
export function getScreenShareAudioPumpDiagnostics(): ScreenShareAudioPumpDiagnostics {
|
||||
return {
|
||||
active: false,
|
||||
captureId: null,
|
||||
sampleRate: null,
|
||||
channels: null,
|
||||
usesNativeSink: false,
|
||||
publishStrategy: 'none',
|
||||
publishedFormatKey: null,
|
||||
eagerPublish: null,
|
||||
eagerPublishError: null,
|
||||
droppedPushFrames: 0,
|
||||
pendingPushFrames: 0,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {
|
||||
type PerTrackStats,
|
||||
VoiceEngineV2AppStatsHostAdapter,
|
||||
type VoiceEngineV2AppStatsHostAdapterScheduler,
|
||||
} from '@app/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter';
|
||||
import {classifyVoiceEngineV2TrackStats} from '@fluxer/voice_engine_v2';
|
||||
import type {Room} from 'livekit-client';
|
||||
import {describe, expect, it} from 'vitest';
|
||||
|
||||
interface FakeScheduler extends VoiceEngineV2AppStatsHostAdapterScheduler {
|
||||
runStatsTick(): void;
|
||||
}
|
||||
|
||||
function createScheduler(): FakeScheduler {
|
||||
const handlers = new Map<number, () => void>();
|
||||
return {
|
||||
setInterval(handler, intervalMs) {
|
||||
handlers.set(intervalMs, handler);
|
||||
return intervalMs;
|
||||
},
|
||||
clearInterval(handle) {
|
||||
handlers.delete(handle as number);
|
||||
},
|
||||
runStatsTick() {
|
||||
handlers.get(2000)?.();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function chromiumCameraSimulcastReports(bytesSentByRid: Record<string, number>): Map<string, unknown> {
|
||||
return new Map<string, unknown>([
|
||||
[
|
||||
'SV1',
|
||||
{
|
||||
type: 'media-source',
|
||||
id: 'SV1',
|
||||
kind: 'video',
|
||||
trackIdentifier: 'cam-track',
|
||||
width: 1280,
|
||||
height: 720,
|
||||
framesPerSecond: 30,
|
||||
frames: 1801,
|
||||
},
|
||||
],
|
||||
['C1', {type: 'codec', id: 'C1', mimeType: 'video/H264', payloadType: 108}],
|
||||
[
|
||||
'OT01V1',
|
||||
{
|
||||
type: 'outbound-rtp',
|
||||
id: 'OT01V1',
|
||||
kind: 'video',
|
||||
ssrc: 1001,
|
||||
rid: 'q',
|
||||
mid: '2',
|
||||
active: false,
|
||||
mediaSourceId: 'SV1',
|
||||
codecId: 'C1',
|
||||
bytesSent: bytesSentByRid.q,
|
||||
framesEncoded: 1,
|
||||
targetBitrate: 0,
|
||||
qualityLimitationReason: 'none',
|
||||
},
|
||||
],
|
||||
[
|
||||
'OT01V2',
|
||||
{
|
||||
type: 'outbound-rtp',
|
||||
id: 'OT01V2',
|
||||
kind: 'video',
|
||||
ssrc: 1002,
|
||||
rid: 'h',
|
||||
mid: '2',
|
||||
active: false,
|
||||
mediaSourceId: 'SV1',
|
||||
codecId: 'C1',
|
||||
bytesSent: bytesSentByRid.h,
|
||||
framesEncoded: 1,
|
||||
targetBitrate: 0,
|
||||
qualityLimitationReason: 'none',
|
||||
},
|
||||
],
|
||||
[
|
||||
'OT01V3',
|
||||
{
|
||||
type: 'outbound-rtp',
|
||||
id: 'OT01V3',
|
||||
kind: 'video',
|
||||
ssrc: 1003,
|
||||
rid: 'f',
|
||||
mid: '2',
|
||||
active: true,
|
||||
mediaSourceId: 'SV1',
|
||||
codecId: 'C1',
|
||||
bytesSent: bytesSentByRid.f,
|
||||
framesEncoded: 1,
|
||||
frameWidth: 1280,
|
||||
frameHeight: 720,
|
||||
targetBitrate: 1133000,
|
||||
qualityLimitationReason: 'none',
|
||||
},
|
||||
],
|
||||
]);
|
||||
}
|
||||
|
||||
async function collectChromiumCameraRows(
|
||||
samples: ReadonlyArray<{now: number; bytesSentByRid: Record<string, number>}>,
|
||||
): Promise<Array<PerTrackStats>> {
|
||||
const scheduler = createScheduler();
|
||||
let sampleIndex = 0;
|
||||
let now = samples[0].now;
|
||||
const publisher = {
|
||||
getStats() {
|
||||
return Promise.resolve(chromiumCameraSimulcastReports(samples[sampleIndex].bytesSentByRid));
|
||||
},
|
||||
};
|
||||
const adapter = new VoiceEngineV2AppStatsHostAdapter({now: () => now, scheduler});
|
||||
adapter.setRoom({engine: {pcManager: {publisher}}, numParticipants: 2} as unknown as Room);
|
||||
adapter.startStatsTracking();
|
||||
for (sampleIndex = 0; sampleIndex < samples.length; sampleIndex += 1) {
|
||||
now = samples[sampleIndex].now;
|
||||
scheduler.runStatsTick();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
}
|
||||
const rows = adapter.perTrackStats;
|
||||
adapter.cleanup();
|
||||
return rows;
|
||||
}
|
||||
|
||||
describe('VoiceEngineV2AppStatsHostAdapter chromium outbound rows', () => {
|
||||
it('collects the outbound-rtp active flag for every simulcast layer', async () => {
|
||||
const rows = await collectChromiumCameraRows([{now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}}]);
|
||||
|
||||
expect(rows.map((row) => row.rid)).toEqual(['q', 'h', 'f']);
|
||||
expect(rows.map((row) => row.active)).toEqual([false, false, true]);
|
||||
});
|
||||
|
||||
it('omits the bitrate window on a first observation so a zero bitrate is not read as measured', async () => {
|
||||
const rows = await collectChromiumCameraRows([{now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}}]);
|
||||
|
||||
expect(rows.map((row) => row.bitrateKbps)).toEqual([0, 0, 0]);
|
||||
expect(rows.map((row) => row.bitrateWindowMs)).toEqual([undefined, undefined, undefined]);
|
||||
});
|
||||
|
||||
it('reports the bitrate window once a delta was measured so a zero bitrate is falsifiable', async () => {
|
||||
const rows = await collectChromiumCameraRows([
|
||||
{now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}},
|
||||
{now: 3001, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}},
|
||||
]);
|
||||
|
||||
expect(rows.map((row) => row.bitrateKbps)).toEqual([0, 0, 0]);
|
||||
expect(rows.map((row) => row.bitrateWindowMs)).toEqual([2001, 2001, 2001]);
|
||||
});
|
||||
|
||||
it('keeps every pre-existing chromium row field byte-identical', async () => {
|
||||
const rows = await collectChromiumCameraRows([
|
||||
{now: 1000, bytesSentByRid: {q: 0, h: 0, f: 5_000_000}},
|
||||
{now: 3000, bytesSentByRid: {q: 0, h: 0, f: 5_500_000}},
|
||||
]);
|
||||
const {active, bitrateWindowMs, ...legacy} = rows[2];
|
||||
|
||||
expect(legacy).toEqual({
|
||||
direction: 'send',
|
||||
kind: 'video',
|
||||
ssrc: 1003,
|
||||
rid: 'f',
|
||||
mid: '2',
|
||||
trackIdentifier: 'cam-track',
|
||||
mediaSourceId: 'SV1',
|
||||
codec: 'video/H264',
|
||||
payloadType: 108,
|
||||
bitrateKbps: 2000,
|
||||
framesEncoded: 1,
|
||||
frameWidth: 1280,
|
||||
frameHeight: 720,
|
||||
sourceFrameWidth: 1280,
|
||||
sourceFrameHeight: 720,
|
||||
sourceFramesPerSecond: 30,
|
||||
sourceFrames: 1801,
|
||||
targetBitrateKbps: 1133,
|
||||
qualityLimitationReason: 'none',
|
||||
encoderAcceleration: 'unknown',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
function firefoxCameraAndScreenShareReports(): Map<string, unknown> {
|
||||
return new Map<string, unknown>([
|
||||
['C1', {type: 'codec', id: 'C1', mimeType: 'video/H264', payloadType: 108}],
|
||||
['C2', {type: 'codec', id: 'C2', mimeType: 'audio/opus', payloadType: 109}],
|
||||
[
|
||||
'outbound_rtp_audio_1',
|
||||
{
|
||||
type: 'outbound-rtp',
|
||||
id: 'outbound_rtp_audio_1',
|
||||
kind: 'audio',
|
||||
ssrc: 724915693,
|
||||
mid: '1',
|
||||
codecId: 'C2',
|
||||
bytesSent: 100_000,
|
||||
},
|
||||
],
|
||||
[
|
||||
'outbound_rtp_video_3',
|
||||
{
|
||||
type: 'outbound-rtp',
|
||||
id: 'outbound_rtp_video_3',
|
||||
kind: 'video',
|
||||
ssrc: 4089824766,
|
||||
mid: '3',
|
||||
codecId: 'C1',
|
||||
bytesSent: 5_000_000,
|
||||
framesEncoded: 900,
|
||||
},
|
||||
],
|
||||
[
|
||||
'outbound_rtp_video_4',
|
||||
{
|
||||
type: 'outbound-rtp',
|
||||
id: 'outbound_rtp_video_4',
|
||||
kind: 'video',
|
||||
ssrc: 1477030229,
|
||||
mid: '4',
|
||||
codecId: 'C1',
|
||||
bytesSent: 2_000_000,
|
||||
framesEncoded: 600,
|
||||
},
|
||||
],
|
||||
[
|
||||
'inbound_rtp_audio_2',
|
||||
{
|
||||
type: 'inbound-rtp',
|
||||
id: 'inbound_rtp_audio_2',
|
||||
kind: 'audio',
|
||||
ssrc: 1661789229,
|
||||
mid: '2',
|
||||
codecId: 'C2',
|
||||
trackIdentifier: '{8a106956-d202-4916-8e19-0a1672c64b73}',
|
||||
bytesReceived: 50_000,
|
||||
},
|
||||
],
|
||||
]);
|
||||
}
|
||||
|
||||
async function collectFirefoxRows(
|
||||
transceivers: ReadonlyArray<{mid: string | null; sender?: {track?: {id: string} | null} | null}> | null,
|
||||
): Promise<Array<PerTrackStats>> {
|
||||
const scheduler = createScheduler();
|
||||
const publisher = {
|
||||
getStats() {
|
||||
return Promise.resolve(firefoxCameraAndScreenShareReports());
|
||||
},
|
||||
...(transceivers === null ? {} : {getTransceivers: () => transceivers}),
|
||||
};
|
||||
const adapter = new VoiceEngineV2AppStatsHostAdapter({now: () => 1000, scheduler});
|
||||
adapter.setRoom({engine: {pcManager: {publisher}}, numParticipants: 2} as unknown as Room);
|
||||
adapter.startStatsTracking();
|
||||
scheduler.runStatsTick();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
const rows = adapter.perTrackStats;
|
||||
adapter.cleanup();
|
||||
return rows;
|
||||
}
|
||||
|
||||
describe('VoiceEngineV2AppStatsHostAdapter firefox outbound rows', () => {
|
||||
it('backfills the outbound trackIdentifier from the transceiver mid when the report carries neither identity member', async () => {
|
||||
const rows = await collectFirefoxRows([
|
||||
{mid: '1', sender: {track: {id: '{mic-track}'}}},
|
||||
{mid: '2', sender: {track: null}},
|
||||
{mid: '3', sender: {track: {id: '{camera-track}'}}},
|
||||
{mid: '4', sender: {track: {id: '{screen-track}'}}},
|
||||
]);
|
||||
|
||||
expect(rows.map((row) => [row.direction, row.kind, row.mid, row.trackIdentifier])).toEqual([
|
||||
['send', 'audio', '1', '{mic-track}'],
|
||||
['send', 'video', '3', '{camera-track}'],
|
||||
['send', 'video', '4', '{screen-track}'],
|
||||
['recv', 'audio', '2', '{8a106956-d202-4916-8e19-0a1672c64b73}'],
|
||||
]);
|
||||
});
|
||||
|
||||
it('leaves an outbound row unidentified when no transceiver carries its mid', async () => {
|
||||
const rows = await collectFirefoxRows([{mid: '3', sender: {track: {id: '{camera-track}'}}}]);
|
||||
|
||||
expect(rows.map((row) => row.trackIdentifier)).toEqual([
|
||||
undefined,
|
||||
'{camera-track}',
|
||||
undefined,
|
||||
'{8a106956-d202-4916-8e19-0a1672c64b73}',
|
||||
]);
|
||||
});
|
||||
|
||||
it('keeps the inbound trackIdentifier when the source exposes no transceivers at all', async () => {
|
||||
const rows = await collectFirefoxRows(null);
|
||||
|
||||
expect(rows.map((row) => row.trackIdentifier)).toEqual([
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
'{8a106956-d202-4916-8e19-0a1672c64b73}',
|
||||
]);
|
||||
});
|
||||
|
||||
it('lets the classifier tell a firefox camera apart from a simultaneous screen share', async () => {
|
||||
const rows = await collectFirefoxRows([
|
||||
{mid: '1', sender: {track: {id: '{mic-track}'}}},
|
||||
{mid: '3', sender: {track: {id: '{camera-track}'}}},
|
||||
{mid: '4', sender: {track: {id: '{screen-track}'}}},
|
||||
]);
|
||||
const classification = classifyVoiceEngineV2TrackStats({
|
||||
tracks: rows.map((row) => ({
|
||||
direction: row.direction,
|
||||
kind: row.kind,
|
||||
rid: row.rid,
|
||||
trackIdentifier: row.trackIdentifier,
|
||||
bitrateKbps: row.bitrateKbps,
|
||||
})),
|
||||
publications: {
|
||||
localCameraTrackId: '{camera-track}',
|
||||
localMicrophoneTrackId: '{mic-track}',
|
||||
localScreenShareTrackId: '{screen-track}',
|
||||
localScreenShareAudioTrackId: null,
|
||||
remoteMicrophoneTrackIds: [],
|
||||
remoteScreenShareTrackIds: [],
|
||||
remoteScreenShareAudioTrackIds: [],
|
||||
},
|
||||
});
|
||||
|
||||
expect(rows[classification.localVideoTrackIndex!].mid).toBe('3');
|
||||
expect(rows[classification.localScreenShareTrackIndex!].mid).toBe('4');
|
||||
});
|
||||
});
|
||||
@@ -54,12 +54,14 @@ export interface PerTrackStats {
|
||||
kind: 'audio' | 'video' | 'unknown';
|
||||
ssrc?: number;
|
||||
rid?: string;
|
||||
active?: boolean;
|
||||
mid?: string;
|
||||
trackIdentifier?: string;
|
||||
mediaSourceId?: string;
|
||||
codec?: string;
|
||||
payloadType?: number;
|
||||
bitrateKbps: number;
|
||||
bitrateWindowMs?: number;
|
||||
packetsLost?: number;
|
||||
packetsLossPercent?: number;
|
||||
jitterMs?: number;
|
||||
@@ -160,6 +162,12 @@ type RoomWithEngine = Room & {
|
||||
|
||||
interface StatsSource {
|
||||
getStats(): Promise<StatsReportMap>;
|
||||
getTransceivers?(): ReadonlyArray<StatsTransceiver>;
|
||||
}
|
||||
|
||||
interface StatsTransceiver {
|
||||
mid: string | null;
|
||||
sender?: {track?: {id: string} | null} | null;
|
||||
}
|
||||
|
||||
interface StatsReportMap {
|
||||
@@ -186,6 +194,7 @@ interface RTCStatsEntry {
|
||||
payloadType?: number;
|
||||
ssrc?: number;
|
||||
rid?: string;
|
||||
active?: boolean;
|
||||
mid?: string;
|
||||
trackId?: string;
|
||||
trackIdentifier?: string;
|
||||
@@ -248,6 +257,11 @@ const LATENCY_UPDATE_INTERVAL_MS = 2000;
|
||||
const STATS_CLOCK_INTERVAL_MS = 1000;
|
||||
const STATS_UPDATE_INTERVAL_MS = 2000;
|
||||
|
||||
interface BitrateSample {
|
||||
bitrateKbps: number;
|
||||
windowMs?: number;
|
||||
}
|
||||
|
||||
function normalizeTrackKind(kind: string | undefined): PerTrackStats['kind'] {
|
||||
if (kind === 'audio' || kind === 'video') return kind;
|
||||
return 'unknown';
|
||||
@@ -264,28 +278,32 @@ function getReportKind(report: RTCStatsEntry, reportsById: Map<string, RTCStatsE
|
||||
return normalizeTrackKind(track?.kind ?? track?.mediaType ?? mediaSource?.kind ?? mediaSource?.mediaType);
|
||||
}
|
||||
|
||||
function getBitrateKbps(
|
||||
function getBitrateSample(
|
||||
reportId: string,
|
||||
currentBytes: number | undefined,
|
||||
now: number,
|
||||
rtpCounters: Map<string, VoiceStatsRtpCounter>,
|
||||
): number {
|
||||
): BitrateSample {
|
||||
if (typeof currentBytes !== 'number' || !Number.isFinite(currentBytes)) {
|
||||
return 0;
|
||||
return {bitrateKbps: 0};
|
||||
}
|
||||
const previous = rtpCounters.get(reportId);
|
||||
let bitrateKbps = 0;
|
||||
let windowMs: number | undefined;
|
||||
if (previous?.bytes !== undefined) {
|
||||
const dt = (now - previous.timestamp) / 1000;
|
||||
const db = currentBytes - previous.bytes;
|
||||
if (dt > 0 && db >= 0) bitrateKbps = (db * 8) / 1000 / dt;
|
||||
if (dt > 0 && db >= 0) {
|
||||
bitrateKbps = (db * 8) / 1000 / dt;
|
||||
windowMs = Math.round(now - previous.timestamp);
|
||||
}
|
||||
}
|
||||
rtpCounters.set(reportId, {
|
||||
...previous,
|
||||
bytes: currentBytes,
|
||||
timestamp: now,
|
||||
});
|
||||
return bitrateKbps;
|
||||
return {bitrateKbps, windowMs};
|
||||
}
|
||||
|
||||
function hasUsableLossDeltas(lostDelta: number, receivedDelta: number): boolean {
|
||||
@@ -344,6 +362,7 @@ function hasUsableJitterBuffer(report: RTCStatsEntry): boolean {
|
||||
|
||||
function buildOutboundTrackExtras(report: RTCStatsEntry): Partial<PerTrackStats> {
|
||||
return {
|
||||
active: report.active,
|
||||
retransmittedPacketsSent: report.retransmittedPacketsSent,
|
||||
retransmittedBytesSent: report.retransmittedBytesSent,
|
||||
keyFramesEncoded: report.keyFramesEncoded,
|
||||
@@ -379,11 +398,12 @@ function buildPerTrackStat(args: {
|
||||
now: number;
|
||||
rtpCounters: Map<string, VoiceStatsRtpCounter>;
|
||||
reportsById: Map<string, RTCStatsEntry>;
|
||||
midToSenderTrackId: Map<string, string>;
|
||||
}): PerTrackStats {
|
||||
const {report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById} = args;
|
||||
const {report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById, midToSenderTrackId} = args;
|
||||
const id = `${sourceId}:${report.type}:${report.id}`;
|
||||
const currentBytes = isOutbound ? report.bytesSent : report.bytesReceived;
|
||||
const bitrateKbps = getBitrateKbps(id, currentBytes, now, rtpCounters);
|
||||
const bitrate = getBitrateSample(id, currentBytes, now, rtpCounters);
|
||||
const codec = report.codecId ? reportsById.get(report.codecId) : undefined;
|
||||
const mediaSource = report.mediaSourceId ? reportsById.get(report.mediaSourceId) : undefined;
|
||||
const packetsLossPercent = isInbound ? getPacketLossPercent(id, report, now, rtpCounters) : undefined;
|
||||
@@ -402,11 +422,15 @@ function buildPerTrackStat(args: {
|
||||
ssrc: report.ssrc,
|
||||
rid: report.rid,
|
||||
mid: report.mid,
|
||||
trackIdentifier: report.trackIdentifier ?? mediaSource?.trackIdentifier,
|
||||
trackIdentifier:
|
||||
report.trackIdentifier ??
|
||||
mediaSource?.trackIdentifier ??
|
||||
(isOutbound && report.mid ? midToSenderTrackId.get(report.mid) : undefined),
|
||||
mediaSourceId: report.mediaSourceId,
|
||||
codec: codec?.mimeType,
|
||||
payloadType: codec?.payloadType,
|
||||
bitrateKbps: Math.round(bitrateKbps),
|
||||
bitrateKbps: Math.round(bitrate.bitrateKbps),
|
||||
bitrateWindowMs: bitrate.windowMs,
|
||||
packetsLost: report.packetsLost,
|
||||
packetsLossPercent: packetsLossPercent !== undefined ? Math.round(packetsLossPercent * 10) / 10 : undefined,
|
||||
jitterMs: typeof report.jitter === 'number' ? Math.round(report.jitter * 1000 * 10) / 10 : undefined,
|
||||
@@ -485,6 +509,11 @@ async function collectFromStatsSource(
|
||||
transport: TransportInfo | null;
|
||||
}> {
|
||||
const reports = await source.getStats();
|
||||
const midToSenderTrackId = new Map<string, string>();
|
||||
for (const transceiver of source.getTransceivers?.() ?? []) {
|
||||
const senderTrackId = transceiver.sender?.track?.id;
|
||||
if (transceiver.mid && senderTrackId) midToSenderTrackId.set(transceiver.mid, senderTrackId);
|
||||
}
|
||||
const reportsById = new Map<string, RTCStatsEntry>();
|
||||
for (const report of reports.values()) {
|
||||
const statsEntry = report as RTCStatsEntry;
|
||||
@@ -511,7 +540,9 @@ async function collectFromStatsSource(
|
||||
if (!isOutbound && !isInbound) continue;
|
||||
const id = `${sourceId}:${report.type}:${report.id}`;
|
||||
activeCounterIds.add(id);
|
||||
tracks.push(buildPerTrackStat({report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById}));
|
||||
tracks.push(
|
||||
buildPerTrackStat({report, sourceId, isOutbound, isInbound, now, rtpCounters, reportsById, midToSenderTrackId}),
|
||||
);
|
||||
}
|
||||
if (!activePair && transportReport?.selectedCandidatePairId) {
|
||||
const selectedPair = reportsById.get(transportReport.selectedCandidatePairId);
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
// @vitest-environment happy-dom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {findStalledVideoEncoder} from '@app/features/voice/engine/voice_screen_share_manager/shared';
|
||||
import {describe, expect, test, vi} from 'vitest';
|
||||
|
||||
vi.mock('@app/features/voice/utils/VideoQualityEntitlement', () => ({hasHigherVideoQuality: () => false}));
|
||||
|
||||
function makeStats(entries: ReadonlyArray<Record<string, unknown>>): RTCStatsReport {
|
||||
return new Map(entries.map((entry) => [entry.id as string, entry])) as unknown as RTCStatsReport;
|
||||
}
|
||||
|
||||
const CODEC_REPORT = {type: 'codec', id: 'codec-h264', mimeType: 'video/H264'};
|
||||
const SOURCE_REPORT = {type: 'media-source', id: 'source-1', kind: 'video', frames: 240, framesPerSecond: 30};
|
||||
|
||||
function outboundReport(overrides: Record<string, unknown>): Record<string, unknown> {
|
||||
return {
|
||||
type: 'outbound-rtp',
|
||||
id: 'outbound-1',
|
||||
kind: 'video',
|
||||
codecId: 'codec-h264',
|
||||
mediaSourceId: 'source-1',
|
||||
framesEncoded: 0,
|
||||
framesSent: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('findStalledVideoEncoder', () => {
|
||||
test('reports a stall when the encoding is active and the source is producing frames', () => {
|
||||
const stats = makeStats([CODEC_REPORT, SOURCE_REPORT, outboundReport({active: true})]);
|
||||
expect(findStalledVideoEncoder(stats, 'h264')).toEqual({
|
||||
codec: 'h264',
|
||||
framesEncoded: 0,
|
||||
framesSent: 0,
|
||||
sourceFrames: 240,
|
||||
sourceFramesPerSecond: 30,
|
||||
});
|
||||
});
|
||||
|
||||
test('ignores an encoding the SFU deactivated even though the source is producing frames', () => {
|
||||
const stats = makeStats([CODEC_REPORT, SOURCE_REPORT, outboundReport({active: false})]);
|
||||
expect(findStalledVideoEncoder(stats, 'h264')).toBeNull();
|
||||
});
|
||||
|
||||
test('still reports a stall when active is absent, as on Firefox', () => {
|
||||
const stats = makeStats([CODEC_REPORT, SOURCE_REPORT, outboundReport({})]);
|
||||
expect(findStalledVideoEncoder(stats, 'h264')?.codec).toBe('h264');
|
||||
});
|
||||
|
||||
test('skips a deactivated layer and reports the still-active sibling instead', () => {
|
||||
const stats = makeStats([
|
||||
CODEC_REPORT,
|
||||
SOURCE_REPORT,
|
||||
{type: 'media-source', id: 'source-low', kind: 'video', frames: 90, framesPerSecond: 15},
|
||||
outboundReport({id: 'outbound-low', mediaSourceId: 'source-low', active: false}),
|
||||
outboundReport({id: 'outbound-high', active: true}),
|
||||
]);
|
||||
expect(findStalledVideoEncoder(stats, 'h264')?.sourceFrames).toBe(240);
|
||||
});
|
||||
});
|
||||
@@ -469,6 +469,7 @@ interface OutboundVideoStatsEntry {
|
||||
mediaType?: string;
|
||||
codecId?: string;
|
||||
mediaSourceId?: string;
|
||||
active?: boolean;
|
||||
framesEncoded?: number;
|
||||
framesSent?: number;
|
||||
encoderImplementation?: string;
|
||||
@@ -592,6 +593,7 @@ export function findStalledVideoEncoder(stats: RTCStatsReport, codec?: VideoCode
|
||||
}
|
||||
for (const report of reports) {
|
||||
if (getStatsKind(report, reportsById) !== 'video') continue;
|
||||
if (report.active === false) continue;
|
||||
const mimeType = report.codecId ? reportsById.get(report.codecId)?.mimeType : undefined;
|
||||
if (report.codecId && !codecMatchesTarget(mimeType, codec)) continue;
|
||||
const resolvedCodec = codec ?? getVideoCodecFromMimeType(mimeType);
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {getNativeAudioCaptureDiagnosticState} from '@app/features/voice/utils/NativeAudioCaptureBridge';
|
||||
import {recordBridgeFrame, startBridgeStats} from '@app/features/voice/utils/native_audio_capture_bridge/bridgeStats';
|
||||
import type {
|
||||
NativeAudioBridgeEndedCapture,
|
||||
NativeAudioBridgeStats,
|
||||
} from '@app/features/voice/utils/native_audio_capture_bridge/shared';
|
||||
import {describe, expect, it} from 'vitest';
|
||||
|
||||
describe('getNativeAudioCaptureDiagnosticState', () => {
|
||||
it('keeps the six fields real dumps already carry and appends the retained histories', () => {
|
||||
expect(Object.keys(getNativeAudioCaptureDiagnosticState())).toEqual([
|
||||
'armedCapture',
|
||||
'activeBridge',
|
||||
'supersededBridge',
|
||||
'lastStartedCapture',
|
||||
'lastArmFailure',
|
||||
'bridgeStats',
|
||||
'endedBridgeCaptures',
|
||||
'lifecycleFaults',
|
||||
]);
|
||||
});
|
||||
|
||||
it('emits the superseded capture next to the live one instead of only the replacement', () => {
|
||||
startBridgeStats('generator', 'native-audio:diag-failed', {prebufferTargetUs: 60_000, frameDurationUs: 10_000});
|
||||
recordBridgeFrame('native-audio:diag-failed', {timestampUs: 0, durationUs: 10_000, peak: 0.31, rms: 0.08});
|
||||
startBridgeStats('generator', 'native-audio:diag-live', {prebufferTargetUs: 60_000, frameDurationUs: 10_000});
|
||||
|
||||
const state = getNativeAudioCaptureDiagnosticState();
|
||||
|
||||
expect((state.bridgeStats as NativeAudioBridgeStats).captureId).toBe('native-audio:diag-live');
|
||||
const ended = state.endedBridgeCaptures as Array<NativeAudioBridgeEndedCapture>;
|
||||
const retained = ended.find((capture) => capture.captureId === 'native-audio:diag-failed');
|
||||
expect(retained?.framesReceived).toBe(1);
|
||||
expect(retained?.nonSilentFrameCount).toBe(1);
|
||||
expect(retained?.endReason).toBe('superseded');
|
||||
});
|
||||
|
||||
it('exposes a native-audio fault history that outlives the lifecycle unbind', () => {
|
||||
expect(Array.isArray(getNativeAudioCaptureDiagnosticState().lifecycleFaults)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -4,7 +4,10 @@ import {Logger} from '@app/features/platform/utils/AppLogger';
|
||||
import {getElectronAPI} from '@app/features/ui/utils/NativeUtils';
|
||||
import type {VoiceEngineV2AppSourceLifecycleBridge} from '@app/features/voice/engine/v2/VoiceEngineV2AppSourceLifecycleBridge';
|
||||
import {getNativeAudioErrorDetail} from '@app/features/voice/utils/NativeAudioFailureUtils';
|
||||
import {getBridgeStats as getNativeAudioBridgeStats} from '@app/features/voice/utils/native_audio_capture_bridge/bridgeStats';
|
||||
import {
|
||||
getEndedBridgeCaptures,
|
||||
getBridgeStats as getNativeAudioBridgeStats,
|
||||
} from '@app/features/voice/utils/native_audio_capture_bridge/bridgeStats';
|
||||
import {createGeneratorBridge} from '@app/features/voice/utils/native_audio_capture_bridge/createGeneratorBridge';
|
||||
import {createScriptProcessorBridge} from '@app/features/voice/utils/native_audio_capture_bridge/createScriptProcessorBridge';
|
||||
import {
|
||||
@@ -67,8 +70,18 @@ interface NativeAudioStartedCaptureDiagnostic {
|
||||
includeSelfWindowAudio?: boolean;
|
||||
}
|
||||
|
||||
interface NativeAudioLifecycleFaultDiagnostic {
|
||||
captureId: string;
|
||||
sourceId: string;
|
||||
message: string;
|
||||
atMs: number;
|
||||
}
|
||||
|
||||
const MAX_RETAINED_LIFECYCLE_FAULTS = 8;
|
||||
|
||||
let lastStartedCapture: NativeAudioStartedCaptureDiagnostic | null = null;
|
||||
let sourceLifecycleBridge: VoiceEngineV2AppSourceLifecycleBridge | null = null;
|
||||
let lifecycleFaults: Array<NativeAudioLifecycleFaultDiagnostic> = [];
|
||||
const lifecycleBoundCaptureIds = new Set<string>();
|
||||
|
||||
export {getNativeAudioBridgeStats};
|
||||
@@ -93,9 +106,19 @@ function bindNativeAudioCaptureLifecycle(captureId: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
function recordLifecycleFault(captureId: string, message: string): void {
|
||||
lifecycleFaults.push({captureId, sourceId: `native-audio-tap:${captureId}`, message, atMs: Date.now()});
|
||||
if (lifecycleFaults.length > MAX_RETAINED_LIFECYCLE_FAULTS) {
|
||||
lifecycleFaults = lifecycleFaults.slice(-MAX_RETAINED_LIFECYCLE_FAULTS);
|
||||
}
|
||||
}
|
||||
|
||||
function unbindNativeAudioCaptureLifecycle(captureId: string, faulted: boolean): void {
|
||||
if (!lifecycleBoundCaptureIds.has(captureId)) return;
|
||||
const bridge = sourceLifecycleBridge;
|
||||
if (faulted) {
|
||||
recordLifecycleFault(captureId, 'native-audio-tap-track-ended');
|
||||
}
|
||||
if (bridge) {
|
||||
if (faulted) {
|
||||
bridge.reportLifecycle({captureId, kind: 'error', message: 'native-audio-tap-track-ended'});
|
||||
@@ -167,6 +190,8 @@ export function getNativeAudioCaptureDiagnosticState(): Record<string, unknown>
|
||||
lastStartedCapture: started,
|
||||
lastArmFailure: getLastNativeAudioArmFailure(),
|
||||
bridgeStats: getNativeAudioBridgeStats(),
|
||||
endedBridgeCaptures: getEndedBridgeCaptures(),
|
||||
lifecycleFaults: lifecycleFaults.map((fault) => ({...fault})),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1499,5 +1524,6 @@ export function resetNativeAudioCaptureBridgeForTests(): void {
|
||||
activeBridge = null;
|
||||
supersededBridge = null;
|
||||
sourceLifecycleBridge = null;
|
||||
lifecycleFaults = [];
|
||||
lifecycleBoundCaptureIds.clear();
|
||||
}
|
||||
|
||||
@@ -15,7 +15,6 @@ import MediaEngine from '@app/features/voice/engine/MediaEngineFacade';
|
||||
import ScreenShareCodecNegotiation, {
|
||||
getScreenShareCodecPreferenceOrder,
|
||||
} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
|
||||
import {getScreenShareAudioPumpDiagnostics} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump';
|
||||
import {getNativeEngineAudioTrackPumpStats} from '@app/features/voice/engine/voice_screen_share_manager/NativeEngineAudioTrackPump';
|
||||
import {getPublishedScreenShareMaxBitrateBps} from '@app/features/voice/engine/voice_screen_share_manager/shared';
|
||||
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
|
||||
@@ -36,6 +35,7 @@ import {getScreenShareBitrateBps, resolveStreamingModeSettings} from '@app/featu
|
||||
import {hasHigherVideoQuality} from '@app/features/voice/utils/VideoQualityEntitlement';
|
||||
import {
|
||||
buildVoiceStatsForNerdsPresentation,
|
||||
collectScreenShareAudioPublicationDiagnostics,
|
||||
type StatsForNerdsData,
|
||||
} from '@app/features/voice/utils/VoiceStatsForNerdsPresenter';
|
||||
import type {NativeAudioApplication, VirtmicNode} from '@app/types/electron.d';
|
||||
@@ -519,8 +519,8 @@ export function collectStatsForNerdsSnapshot(): StatsForNerdsData {
|
||||
openH264Enabled: VoiceSettings.getOpenH264Enabled(),
|
||||
},
|
||||
screenShareAudioCapture: {
|
||||
pump: getScreenShareAudioPumpDiagnostics(),
|
||||
nativeCapture: getNativeAudioCaptureDiagnosticState(),
|
||||
publications: collectScreenShareAudioPublicationDiagnostics(localParticipant),
|
||||
},
|
||||
appInfo: {
|
||||
appVersion: Config.PUBLIC_BUILD_VERSION ?? 'dev',
|
||||
|
||||
@@ -45,9 +45,11 @@ const CODEC_PROBES: ReadonlyArray<CodecProbe> = [
|
||||
},
|
||||
];
|
||||
|
||||
const SCREEN_SHARE_DECODE_FAILURES_MAX = 1;
|
||||
|
||||
let cachedExclusions: Array<VideoCodec> | null = null;
|
||||
let pendingProbe: Promise<Array<VideoCodec>> | null = null;
|
||||
const runtimeExcludedCodecs = new Set<VideoCodec>();
|
||||
const screenShareDecodeFailures = new Set<VideoCodec>();
|
||||
|
||||
interface VideoDecoderLike {
|
||||
isConfigSupported(config: WebCodecsConfig): Promise<{supported: boolean}>;
|
||||
@@ -80,15 +82,6 @@ function isBaselineWebRtcDecodeCodec(codec: VideoCodec): boolean {
|
||||
return codec === 'h264' || codec === 'vp8';
|
||||
}
|
||||
|
||||
function mergeRuntimeExclusions(exclusions: Array<VideoCodec> | null): Array<VideoCodec> | null {
|
||||
if (runtimeExcludedCodecs.size === 0) return exclusions;
|
||||
const merged = new Set(exclusions ?? []);
|
||||
for (const codec of runtimeExcludedCodecs) {
|
||||
merged.add(codec);
|
||||
}
|
||||
return [...merged];
|
||||
}
|
||||
|
||||
function getPlatformPolicyExclusions(): Array<VideoCodec> | null {
|
||||
if ((globalThis as Record<string, unknown>).RTCRtpReceiver === undefined) return null;
|
||||
const excluded: Array<VideoCodec> = [];
|
||||
@@ -145,17 +138,17 @@ async function probeAllCodecs(): Promise<Array<VideoCodec>> {
|
||||
}
|
||||
|
||||
export function loadVideoDecoderExclusions(): Promise<Array<VideoCodec>> {
|
||||
if (cachedExclusions) return Promise.resolve(mergeRuntimeExclusions(cachedExclusions) ?? []);
|
||||
if (cachedExclusions) return Promise.resolve(cachedExclusions);
|
||||
if (pendingProbe) return pendingProbe;
|
||||
pendingProbe = probeAllCodecs()
|
||||
.then((result) => {
|
||||
cachedExclusions = result;
|
||||
return mergeRuntimeExclusions(result) ?? [];
|
||||
return result;
|
||||
})
|
||||
.catch((error) => {
|
||||
logger.warn('Video decoder probe failed entirely, not excluding any codecs', {error});
|
||||
cachedExclusions = [];
|
||||
return mergeRuntimeExclusions([]) ?? [];
|
||||
return [];
|
||||
})
|
||||
.finally(() => {
|
||||
pendingProbe = null;
|
||||
@@ -164,21 +157,29 @@ export function loadVideoDecoderExclusions(): Promise<Array<VideoCodec>> {
|
||||
}
|
||||
|
||||
export function getVideoDecoderExclusionsSync(): Array<VideoCodec> | null {
|
||||
return mergeRuntimeExclusions(cachedExclusions ?? getPlatformPolicyExclusions());
|
||||
return cachedExclusions ?? getPlatformPolicyExclusions();
|
||||
}
|
||||
|
||||
export function markVideoDecoderRuntimeFailure(codec: VideoCodec, reason: string): boolean {
|
||||
if (isBaselineWebRtcDecodeCodec(codec)) return false;
|
||||
if (runtimeExcludedCodecs.has(codec)) return false;
|
||||
runtimeExcludedCodecs.add(codec);
|
||||
logger.warn('Excluding codec from subscriber decode after runtime failure', {codec, reason});
|
||||
export function markScreenShareDecodeFailure(codec: VideoCodec, reason: string): boolean {
|
||||
if (screenShareDecodeFailures.has(codec)) return false;
|
||||
if (screenShareDecodeFailures.size >= SCREEN_SHARE_DECODE_FAILURES_MAX) return false;
|
||||
screenShareDecodeFailures.add(codec);
|
||||
logger.warn('Withdrawing decode advertisement for a codec after a screen share decode stall', {codec, reason});
|
||||
return true;
|
||||
}
|
||||
|
||||
export function getScreenShareDecodeFailures(): ReadonlySet<VideoCodec> {
|
||||
return screenShareDecodeFailures;
|
||||
}
|
||||
|
||||
export function clearScreenShareDecodeFailures(): void {
|
||||
screenShareDecodeFailures.clear();
|
||||
}
|
||||
|
||||
export function resetVideoDecoderExclusions(): void {
|
||||
cachedExclusions = null;
|
||||
pendingProbe = null;
|
||||
runtimeExcludedCodecs.clear();
|
||||
screenShareDecodeFailures.clear();
|
||||
}
|
||||
|
||||
if (typeof window !== 'undefined') {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import {
|
||||
buildVoiceStatsForNerdsPresentation,
|
||||
collectScreenShareAudioPublicationDiagnostics,
|
||||
type ParticipantPublicationLookup,
|
||||
} from '@app/features/voice/utils/VoiceStatsForNerdsPresenter';
|
||||
import type {
|
||||
@@ -86,6 +87,64 @@ function nativeStats(overrides: Partial<VoiceEngineV2Stats>): VoiceEngineV2Stats
|
||||
return {rttMs: null, outbound: [], inbound: [], ...overrides};
|
||||
}
|
||||
|
||||
function chromiumCameraSimulcastTracks(): Array<VoiceEngineV2PerTrackStats> {
|
||||
return [
|
||||
{
|
||||
direction: 'send',
|
||||
kind: 'video',
|
||||
ssrc: 1001,
|
||||
rid: 'q',
|
||||
active: false,
|
||||
mid: '2',
|
||||
trackIdentifier: 'cam-track',
|
||||
mediaSourceId: 'SV1',
|
||||
bitrateKbps: 0,
|
||||
bitrateWindowMs: 2001,
|
||||
framesEncoded: 1,
|
||||
targetBitrateKbps: 0,
|
||||
},
|
||||
{
|
||||
direction: 'send',
|
||||
kind: 'video',
|
||||
ssrc: 1002,
|
||||
rid: 'h',
|
||||
active: false,
|
||||
mid: '2',
|
||||
trackIdentifier: 'cam-track',
|
||||
mediaSourceId: 'SV1',
|
||||
bitrateKbps: 0,
|
||||
bitrateWindowMs: 2001,
|
||||
framesEncoded: 1,
|
||||
targetBitrateKbps: 0,
|
||||
},
|
||||
{
|
||||
direction: 'send',
|
||||
kind: 'video',
|
||||
ssrc: 1003,
|
||||
rid: 'f',
|
||||
active: true,
|
||||
mid: '2',
|
||||
trackIdentifier: 'cam-track',
|
||||
mediaSourceId: 'SV1',
|
||||
bitrateKbps: 0,
|
||||
bitrateWindowMs: 2001,
|
||||
framesEncoded: 1,
|
||||
frameWidth: 1280,
|
||||
frameHeight: 720,
|
||||
targetBitrateKbps: 1133,
|
||||
},
|
||||
{
|
||||
direction: 'send',
|
||||
kind: 'video',
|
||||
ssrc: 2001,
|
||||
mid: '3',
|
||||
trackIdentifier: 'screen-track',
|
||||
mediaSourceId: 'SV2',
|
||||
bitrateKbps: 3500,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
describe('buildVoiceStatsForNerdsPresentation', () => {
|
||||
it('classifies browser per-track stats once for overlay and copy consumers', () => {
|
||||
const localParticipant = participant({
|
||||
@@ -180,6 +239,203 @@ describe('buildVoiceStatsForNerdsPresentation', () => {
|
||||
expect(presentation.remoteAudio).toBeNull();
|
||||
});
|
||||
|
||||
it('classifies a firefox screen share whose outbound stats omit the track identifier', () => {
|
||||
const localParticipant = participant({
|
||||
microphone: 'b0d2f1a7-6c3e-4f8a-9b21-5d7c4e0a1f36',
|
||||
screen_share: '2f4c8de1-9a07-4b53-8c6d-1e5b7a02d94f',
|
||||
});
|
||||
const remoteParticipant = participant({microphone: '8a106956-d202-4916-8e19-0a1672c64b73'});
|
||||
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
|
||||
{
|
||||
direction: 'recv',
|
||||
kind: 'audio',
|
||||
mid: '2',
|
||||
trackIdentifier: '{8a106956-d202-4916-8e19-0a1672c64b73}',
|
||||
bitrateKbps: 46,
|
||||
},
|
||||
{direction: 'send', kind: 'audio', mid: '1', bitrateKbps: 30},
|
||||
{direction: 'send', kind: 'video', mid: '3', bitrateKbps: 47, framesPerSecond: 58, frameWidth: 1920},
|
||||
];
|
||||
|
||||
const presentation = buildVoiceStatsForNerdsPresentation({
|
||||
connectionId: 'connection-a',
|
||||
connectionQuality: 'excellent',
|
||||
currentLatency: 37,
|
||||
averageLatency: 41,
|
||||
stats: voiceStats,
|
||||
perTrackStats,
|
||||
statsTimeSeries: timeSeries,
|
||||
nativeStats: null,
|
||||
publisherTransport: null,
|
||||
subscriberTransport: null,
|
||||
localParticipant,
|
||||
remoteParticipants: [remoteParticipant],
|
||||
});
|
||||
|
||||
expect(presentation.localScreenShare?.mid).toBe('3');
|
||||
expect(presentation.localVideo).toBeNull();
|
||||
});
|
||||
|
||||
it('classifies a firefox inbound screen share reported with a braced track identifier', () => {
|
||||
const remoteParticipant = participant({screen_share: '34171e0d-174d-4d56-bb71-76b4fba330c3'});
|
||||
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
|
||||
{
|
||||
direction: 'recv',
|
||||
kind: 'video',
|
||||
mid: '3',
|
||||
trackIdentifier: '{34171e0d-174d-4d56-bb71-76b4fba330c3}',
|
||||
bitrateKbps: 3585,
|
||||
},
|
||||
];
|
||||
|
||||
const presentation = buildVoiceStatsForNerdsPresentation({
|
||||
connectionId: 'connection-a',
|
||||
connectionQuality: 'excellent',
|
||||
currentLatency: 37,
|
||||
averageLatency: 41,
|
||||
stats: voiceStats,
|
||||
perTrackStats,
|
||||
statsTimeSeries: timeSeries,
|
||||
nativeStats: null,
|
||||
publisherTransport: null,
|
||||
subscriberTransport: null,
|
||||
localParticipant: null,
|
||||
remoteParticipants: [remoteParticipant],
|
||||
});
|
||||
|
||||
expect(presentation.remoteScreenShare?.mid).toBe('3');
|
||||
expect(presentation.remoteVideo).toBeNull();
|
||||
});
|
||||
|
||||
it('exposes every camera simulcast layer next to the projected localVideo row', () => {
|
||||
const localParticipant = participant({camera: 'cam-track', screen_share: 'screen-track'});
|
||||
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = chromiumCameraSimulcastTracks();
|
||||
|
||||
const presentation = buildVoiceStatsForNerdsPresentation({
|
||||
connectionId: 'connection-a',
|
||||
connectionQuality: 'excellent',
|
||||
currentLatency: 37,
|
||||
averageLatency: 41,
|
||||
stats: voiceStats,
|
||||
perTrackStats,
|
||||
statsTimeSeries: timeSeries,
|
||||
nativeStats: null,
|
||||
publisherTransport: null,
|
||||
subscriberTransport: null,
|
||||
localParticipant,
|
||||
remoteParticipants: null,
|
||||
});
|
||||
|
||||
expect(presentation.localVideo?.rid).toBe('q');
|
||||
expect(presentation.localVideoLayers).toHaveLength(3);
|
||||
expect(presentation.localVideoLayers.map((layer) => layer.rid)).toEqual(['q', 'h', 'f']);
|
||||
expect(presentation.localVideoLayers.map((layer) => layer.active)).toEqual([false, false, true]);
|
||||
expect(presentation.localVideoLayers.map((layer) => layer.bitrateWindowMs)).toEqual([2001, 2001, 2001]);
|
||||
});
|
||||
|
||||
it('keeps the screen-share publication out of the camera layer view', () => {
|
||||
const localParticipant = participant({camera: 'cam-track', screen_share: 'screen-track'});
|
||||
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = chromiumCameraSimulcastTracks();
|
||||
|
||||
const presentation = buildVoiceStatsForNerdsPresentation({
|
||||
connectionId: 'connection-a',
|
||||
connectionQuality: 'excellent',
|
||||
currentLatency: 37,
|
||||
averageLatency: 41,
|
||||
stats: voiceStats,
|
||||
perTrackStats,
|
||||
statsTimeSeries: timeSeries,
|
||||
nativeStats: null,
|
||||
publisherTransport: null,
|
||||
subscriberTransport: null,
|
||||
localParticipant,
|
||||
remoteParticipants: null,
|
||||
});
|
||||
|
||||
expect(presentation.localScreenShare?.trackIdentifier).toBe('screen-track');
|
||||
expect(presentation.localVideoLayers).not.toContain(perTrackStats[3]);
|
||||
});
|
||||
|
||||
it('reports a single camera layer when firefox omits rid and media source ids', () => {
|
||||
const localParticipant = participant({camera: 'cam-track'});
|
||||
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
|
||||
{direction: 'send', kind: 'audio', mid: '1', bitrateKbps: 30},
|
||||
{direction: 'send', kind: 'video', mid: '3', bitrateKbps: 47, framesPerSecond: 58, frameWidth: 1920},
|
||||
];
|
||||
|
||||
const presentation = buildVoiceStatsForNerdsPresentation({
|
||||
connectionId: 'connection-a',
|
||||
connectionQuality: 'excellent',
|
||||
currentLatency: 37,
|
||||
averageLatency: 41,
|
||||
stats: voiceStats,
|
||||
perTrackStats,
|
||||
statsTimeSeries: timeSeries,
|
||||
nativeStats: null,
|
||||
publisherTransport: null,
|
||||
subscriberTransport: null,
|
||||
localParticipant,
|
||||
remoteParticipants: null,
|
||||
});
|
||||
|
||||
expect(presentation.localVideo?.mid).toBe('3');
|
||||
expect(presentation.localVideoLayers).toEqual([perTrackStats[1]]);
|
||||
});
|
||||
|
||||
it('reports no camera layers rather than a false single layer when firefox simulcast is unidentifiable', () => {
|
||||
const localParticipant = participant({camera: 'cam-track'});
|
||||
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
|
||||
{direction: 'send', kind: 'audio', mid: '1', bitrateKbps: 30},
|
||||
{direction: 'send', kind: 'video', mid: '3', rid: 'q', bitrateKbps: 0},
|
||||
{direction: 'send', kind: 'video', mid: '3', rid: 'h', bitrateKbps: 0},
|
||||
{direction: 'send', kind: 'video', mid: '3', rid: 'f', bitrateKbps: 1133},
|
||||
];
|
||||
|
||||
const presentation = buildVoiceStatsForNerdsPresentation({
|
||||
connectionId: 'connection-a',
|
||||
connectionQuality: 'excellent',
|
||||
currentLatency: 37,
|
||||
averageLatency: 41,
|
||||
stats: voiceStats,
|
||||
perTrackStats,
|
||||
statsTimeSeries: timeSeries,
|
||||
nativeStats: null,
|
||||
publisherTransport: null,
|
||||
subscriberTransport: null,
|
||||
localParticipant,
|
||||
remoteParticipants: null,
|
||||
});
|
||||
|
||||
expect(presentation.localVideo).toBeNull();
|
||||
expect(presentation.localVideoLayers).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps the chromium simulcast projection byte-identical', () => {
|
||||
const localParticipant = participant({camera: 'cam-track', screen_share: 'screen-track'});
|
||||
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = chromiumCameraSimulcastTracks();
|
||||
|
||||
const presentation = buildVoiceStatsForNerdsPresentation({
|
||||
connectionId: 'connection-a',
|
||||
connectionQuality: 'excellent',
|
||||
currentLatency: 37,
|
||||
averageLatency: 41,
|
||||
stats: voiceStats,
|
||||
perTrackStats,
|
||||
statsTimeSeries: timeSeries,
|
||||
nativeStats: null,
|
||||
publisherTransport: null,
|
||||
subscriberTransport: null,
|
||||
localParticipant,
|
||||
remoteParticipants: null,
|
||||
});
|
||||
|
||||
expect(presentation.localVideo).toBe(perTrackStats[0]);
|
||||
expect(presentation.localScreenShare).toBe(perTrackStats[3]);
|
||||
expect(presentation.localAudio).toBeNull();
|
||||
expect(presentation.remoteVideo).toBeNull();
|
||||
expect(presentation.network.videoSendBitrateKbps).toBe(1200);
|
||||
});
|
||||
|
||||
it('uses the canonical v2 native stats projection when native stats are available', () => {
|
||||
const perTrackStats: Array<VoiceEngineV2PerTrackStats> = [
|
||||
{direction: 'send', kind: 'video', trackIdentifier: 'browser-camera', bitrateKbps: 500},
|
||||
@@ -309,3 +565,87 @@ describe('buildVoiceStatsForNerdsPresentation', () => {
|
||||
expect(presentation.network.rttMs).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('collectScreenShareAudioPublicationDiagnostics', () => {
|
||||
it('reports the mute and upstream state of every local screen-share audio publication', () => {
|
||||
const displayPublication = {
|
||||
trackSid: 'TR_display',
|
||||
source: 'screen_share_audio',
|
||||
isMuted: false,
|
||||
audioTrack: {
|
||||
isUpstreamPaused: true,
|
||||
mediaStreamTrack: {
|
||||
id: 'display-audio',
|
||||
readyState: 'live',
|
||||
muted: true,
|
||||
enabled: true,
|
||||
} as MediaStreamTrack,
|
||||
},
|
||||
};
|
||||
const devicePublication = {
|
||||
trackSid: 'TR_device',
|
||||
source: 'screen_share_audio',
|
||||
isMuted: false,
|
||||
audioTrack: {
|
||||
isUpstreamPaused: false,
|
||||
mediaStreamTrack: {
|
||||
id: 'device-audio',
|
||||
readyState: 'live',
|
||||
muted: false,
|
||||
enabled: true,
|
||||
} as MediaStreamTrack,
|
||||
},
|
||||
};
|
||||
const micPublication = {
|
||||
trackSid: 'TR_mic',
|
||||
source: 'microphone',
|
||||
isMuted: false,
|
||||
audioTrack: {
|
||||
isUpstreamPaused: false,
|
||||
mediaStreamTrack: {
|
||||
id: 'mic-audio',
|
||||
readyState: 'live',
|
||||
muted: false,
|
||||
enabled: true,
|
||||
} as MediaStreamTrack,
|
||||
},
|
||||
};
|
||||
const localParticipant: ParticipantPublicationLookup = {
|
||||
getTrackPublication: (source) => (String(source) === 'screen_share_audio' ? displayPublication : undefined),
|
||||
audioTrackPublications: new Map([
|
||||
['TR_display', displayPublication],
|
||||
['TR_device', devicePublication],
|
||||
['TR_mic', micPublication],
|
||||
]),
|
||||
};
|
||||
|
||||
const diagnostics = collectScreenShareAudioPublicationDiagnostics(localParticipant);
|
||||
|
||||
expect(diagnostics).toEqual([
|
||||
{
|
||||
trackSid: 'TR_display',
|
||||
source: 'screen_share_audio',
|
||||
isMuted: false,
|
||||
isUpstreamPaused: true,
|
||||
mediaStreamTrackId: 'display-audio',
|
||||
mediaStreamTrackReadyState: 'live',
|
||||
mediaStreamTrackMuted: true,
|
||||
mediaStreamTrackEnabled: true,
|
||||
},
|
||||
{
|
||||
trackSid: 'TR_device',
|
||||
source: 'screen_share_audio',
|
||||
isMuted: false,
|
||||
isUpstreamPaused: false,
|
||||
mediaStreamTrackId: 'device-audio',
|
||||
mediaStreamTrackReadyState: 'live',
|
||||
mediaStreamTrackMuted: false,
|
||||
mediaStreamTrackEnabled: true,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('reports nothing when there is no local participant', () => {
|
||||
expect(collectScreenShareAudioPublicationDiagnostics(null)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -6,7 +6,6 @@ import {
|
||||
type VoiceTrackPublicationSourceLike,
|
||||
VoiceTrackSource,
|
||||
} from '@app/features/voice/engine/VoiceTrackSource';
|
||||
import type {ScreenShareAudioPumpDiagnostics} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump';
|
||||
import {
|
||||
classifyVoiceEngineV2TrackStats,
|
||||
selectVoiceEngineV2StatsPresentationProjection,
|
||||
@@ -21,6 +20,17 @@ import {
|
||||
} from '@fluxer/voice_engine_v2';
|
||||
import type {Track} from 'livekit-client';
|
||||
|
||||
export interface ScreenShareAudioPublicationDiagnostic {
|
||||
trackSid: string | null;
|
||||
source: string | null;
|
||||
isMuted: boolean | null;
|
||||
isUpstreamPaused: boolean | null;
|
||||
mediaStreamTrackId: string | null;
|
||||
mediaStreamTrackReadyState: string | null;
|
||||
mediaStreamTrackMuted: boolean | null;
|
||||
mediaStreamTrackEnabled: boolean | null;
|
||||
}
|
||||
|
||||
export interface StatsForNerdsData {
|
||||
session: {
|
||||
connectionId: string;
|
||||
@@ -44,6 +54,7 @@ export interface StatsForNerdsData {
|
||||
subscriberTransport: VoiceEngineV2TransportInfo | null;
|
||||
};
|
||||
localVideo: VoiceEngineV2PerTrackStats | null;
|
||||
localVideoLayers: Array<VoiceEngineV2PerTrackStats>;
|
||||
localAudio: VoiceEngineV2PerTrackStats | null;
|
||||
localScreenShare: VoiceEngineV2PerTrackStats | null;
|
||||
localScreenShareAudio: VoiceEngineV2PerTrackStats | null;
|
||||
@@ -85,8 +96,8 @@ export interface StatsForNerdsData {
|
||||
openH264Enabled: boolean;
|
||||
};
|
||||
screenShareAudioCapture: {
|
||||
pump: ScreenShareAudioPumpDiagnostics;
|
||||
nativeCapture: Record<string, unknown>;
|
||||
publications: Array<ScreenShareAudioPublicationDiagnostic>;
|
||||
};
|
||||
appInfo: {
|
||||
appVersion: string;
|
||||
@@ -119,6 +130,7 @@ export interface VoiceStatsForNerdsPresentation {
|
||||
session: StatsForNerdsData['session'];
|
||||
network: StatsForNerdsData['network'];
|
||||
localVideo: VoiceEngineV2PerTrackStats | null;
|
||||
localVideoLayers: Array<VoiceEngineV2PerTrackStats>;
|
||||
localAudio: VoiceEngineV2PerTrackStats | null;
|
||||
localScreenShare: VoiceEngineV2PerTrackStats | null;
|
||||
localScreenShareAudio: VoiceEngineV2PerTrackStats | null;
|
||||
@@ -131,9 +143,12 @@ export interface VoiceStatsForNerdsPresentation {
|
||||
|
||||
interface PublicationTrackLike {
|
||||
mediaStreamTrack?: MediaStreamTrack;
|
||||
isUpstreamPaused?: boolean;
|
||||
}
|
||||
|
||||
interface TrackPublicationLike extends VoiceTrackPublicationSourceLike {
|
||||
trackSid?: string;
|
||||
isMuted?: boolean;
|
||||
audioTrack?: PublicationTrackLike | null;
|
||||
videoTrack?: PublicationTrackLike | null;
|
||||
track?: PublicationTrackLike | null;
|
||||
@@ -234,6 +249,38 @@ function collectParticipantScreenShareAudioTrackIdentifiers(
|
||||
return Array.from(identifiers);
|
||||
}
|
||||
|
||||
function summarizeScreenShareAudioPublication(
|
||||
publication: TrackPublicationLike,
|
||||
): ScreenShareAudioPublicationDiagnostic {
|
||||
const track = publication.audioTrack ?? publication.track ?? null;
|
||||
const mediaStreamTrack = track?.mediaStreamTrack ?? null;
|
||||
return {
|
||||
trackSid: publication.trackSid ?? null,
|
||||
source: typeof publication.source === 'string' ? publication.source : null,
|
||||
isMuted: publication.isMuted ?? null,
|
||||
isUpstreamPaused: track?.isUpstreamPaused ?? null,
|
||||
mediaStreamTrackId: mediaStreamTrack?.id ?? null,
|
||||
mediaStreamTrackReadyState: mediaStreamTrack?.readyState ?? null,
|
||||
mediaStreamTrackMuted: mediaStreamTrack?.muted ?? null,
|
||||
mediaStreamTrackEnabled: mediaStreamTrack?.enabled ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export function collectScreenShareAudioPublicationDiagnostics(
|
||||
participant: ParticipantPublicationLookup | null | undefined,
|
||||
): Array<ScreenShareAudioPublicationDiagnostic> {
|
||||
const publications = new Set<TrackPublicationLike>();
|
||||
const directPublication = participant?.getTrackPublication(SCREEN_SHARE_AUDIO_SOURCE);
|
||||
if (directPublication) {
|
||||
publications.add(directPublication);
|
||||
}
|
||||
for (const publication of participant?.audioTrackPublications?.values() ?? []) {
|
||||
if (!isScreenShareAudioPublicationLike(publication)) continue;
|
||||
publications.add(publication);
|
||||
}
|
||||
return Array.from(publications, summarizeScreenShareAudioPublication);
|
||||
}
|
||||
|
||||
function collectRemoteScreenShareAudioTrackIdentifiers(
|
||||
remoteParticipants: Iterable<ParticipantPublicationLookup> | null | undefined,
|
||||
): Array<string> {
|
||||
@@ -284,6 +331,25 @@ function getClassifiedTrack(
|
||||
return perTrackStats[index] ?? null;
|
||||
}
|
||||
|
||||
function getPublicationGroupKey(track: VoiceEngineV2PerTrackStats): string | null {
|
||||
if (track.mediaSourceId !== undefined) return `source:${track.mediaSourceId}`;
|
||||
if (track.trackIdentifier !== undefined) return `track:${track.trackIdentifier}`;
|
||||
if (track.mid !== undefined) return `mid:${track.mid}`;
|
||||
return null;
|
||||
}
|
||||
|
||||
function selectLocalVideoLayers(
|
||||
perTrackStats: ReadonlyArray<VoiceEngineV2PerTrackStats>,
|
||||
localVideo: VoiceEngineV2PerTrackStats | null,
|
||||
): Array<VoiceEngineV2PerTrackStats> {
|
||||
if (!localVideo) return [];
|
||||
const groupKey = getPublicationGroupKey(localVideo);
|
||||
if (groupKey === null) return [localVideo];
|
||||
return perTrackStats.filter(
|
||||
(track) => track.direction === 'send' && track.kind === 'video' && getPublicationGroupKey(track) === groupKey,
|
||||
);
|
||||
}
|
||||
|
||||
function selectNativeStatsProjection(
|
||||
nativeStats: VoiceEngineV2Stats | null,
|
||||
): VoiceEngineV2StatsPresentationProjection | null {
|
||||
@@ -366,6 +432,7 @@ export function buildVoiceStatsForNerdsPresentation(
|
||||
subscriberTransport,
|
||||
},
|
||||
localVideo: nativeProjection ? nativeProjection.localVideo : localVideo,
|
||||
localVideoLayers: selectLocalVideoLayers(perTrackStats, localVideo),
|
||||
localAudio: nativeProjection ? nativeProjection.localAudio : localAudio,
|
||||
localScreenShare: nativeProjection ? nativeProjection.localScreenShare : localScreenShare,
|
||||
localScreenShareAudio: nativeProjection ? nativeProjection.localScreenShareAudio : localScreenShareAudio,
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {describe, expect, it, vi} from 'vitest';
|
||||
import {
|
||||
endBridgeStats,
|
||||
getBridgeStats,
|
||||
getEndedBridgeCaptures,
|
||||
recordBridgeFrame,
|
||||
startBridgeStats,
|
||||
} from './bridgeStats';
|
||||
|
||||
function recordFrames(captureId: string, count: number, peak: number, rms: number): void {
|
||||
for (let index = 0; index < count; index++) {
|
||||
recordBridgeFrame(captureId, {timestampUs: index * 10_000, durationUs: 10_000, peak, rms});
|
||||
}
|
||||
}
|
||||
|
||||
function findEndedCapture(captureId: string) {
|
||||
return getEndedBridgeCaptures().find((capture) => capture.captureId === captureId);
|
||||
}
|
||||
|
||||
describe('bridgeStats', () => {
|
||||
it('retains the counters of a capture that a mid-share restart supersedes', () => {
|
||||
startBridgeStats('generator', 'native-audio:failed', {prebufferTargetUs: 60_000, frameDurationUs: 10_000});
|
||||
recordFrames('native-audio:failed', 1124, 0.42, 0.11);
|
||||
|
||||
startBridgeStats('generator', 'native-audio:replacement', {prebufferTargetUs: 60_000, frameDurationUs: 10_000});
|
||||
|
||||
expect(getBridgeStats().captureId).toBe('native-audio:replacement');
|
||||
expect(getBridgeStats().framesReceived).toBe(0);
|
||||
const retained = findEndedCapture('native-audio:failed');
|
||||
expect(retained?.bridgeMode).toBe('generator');
|
||||
expect(retained?.framesReceived).toBe(1124);
|
||||
expect(retained?.nonSilentFrameCount).toBe(1124);
|
||||
expect(retained?.lastFramePeak).toBe(0.42);
|
||||
expect(retained?.endReason).toBe('superseded');
|
||||
});
|
||||
|
||||
it('records the end reason of a capture the newer capture already superseded', () => {
|
||||
startBridgeStats('generator', 'native-audio:ends-late');
|
||||
recordFrames('native-audio:ends-late', 3, 0.2, 0.05);
|
||||
startBridgeStats('generator', 'native-audio:ends-late-replacement');
|
||||
|
||||
endBridgeStats('native-audio:ends-late', 'ended', 'source-disappeared');
|
||||
|
||||
const retained = findEndedCapture('native-audio:ends-late');
|
||||
expect(retained?.endReason).toBe('ended');
|
||||
expect(retained?.endDetail).toBe('source-disappeared');
|
||||
expect(retained?.endedAt).not.toBeNull();
|
||||
expect(getBridgeStats().captureId).toBe('native-audio:ends-late-replacement');
|
||||
expect(getBridgeStats().active).toBe(true);
|
||||
});
|
||||
|
||||
it('still ends the active capture in place', () => {
|
||||
startBridgeStats('script-processor', 'native-audio:active-end');
|
||||
|
||||
endBridgeStats('native-audio:active-end', 'cleanup', 'caller-stopped');
|
||||
|
||||
const stats = getBridgeStats();
|
||||
expect(stats.active).toBe(false);
|
||||
expect(stats.endReason).toBe('cleanup');
|
||||
expect(stats.endDetail).toBe('caller-stopped');
|
||||
expect(stats.endedAt).not.toBeNull();
|
||||
});
|
||||
|
||||
it('keeps the first end reason of the active capture', () => {
|
||||
startBridgeStats('generator', 'native-audio:double-end');
|
||||
endBridgeStats('native-audio:double-end', 'ended', 'source-disappeared');
|
||||
|
||||
endBridgeStats('native-audio:double-end', 'cleanup', 'caller-stopped');
|
||||
|
||||
expect(getBridgeStats().endReason).toBe('ended');
|
||||
expect(getBridgeStats().endDetail).toBe('source-disappeared');
|
||||
});
|
||||
|
||||
it('flags a capture whose frames keep arriving while every one of them is silence', () => {
|
||||
const startMs = Date.UTC(2026, 8, 8, 12, 0, 0);
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
vi.setSystemTime(startMs);
|
||||
startBridgeStats('generator', 'native-audio:all-silent', {frameDurationUs: 10_000});
|
||||
for (let index = 0; index < 400; index++) {
|
||||
vi.setSystemTime(startMs + index * 100);
|
||||
recordBridgeFrame('native-audio:all-silent', {
|
||||
timestampUs: index * 10_000,
|
||||
durationUs: 10_000,
|
||||
peak: 0,
|
||||
rms: 0,
|
||||
});
|
||||
}
|
||||
|
||||
const stats = getBridgeStats();
|
||||
expect(stats.framesReceived).toBe(400);
|
||||
expect(stats.nonSilentFrameCount).toBe(0);
|
||||
expect(stats.lastNonSilentFrameAt).toBeNull();
|
||||
expect(stats.silentFrameStreak).toBe(400);
|
||||
expect(stats.maxSilentRunMs).toBe(39_900);
|
||||
expect(stats.sustainedSilenceWarned).toBe(true);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('does not flag a capture that keeps producing audible frames', () => {
|
||||
const startMs = Date.UTC(2026, 8, 8, 13, 0, 0);
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
vi.setSystemTime(startMs);
|
||||
startBridgeStats('generator', 'native-audio:audible', {frameDurationUs: 10_000});
|
||||
for (let index = 0; index < 400; index++) {
|
||||
vi.setSystemTime(startMs + index * 100);
|
||||
const audible = index % 4 === 0;
|
||||
recordBridgeFrame('native-audio:audible', {
|
||||
timestampUs: index * 10_000,
|
||||
durationUs: 10_000,
|
||||
peak: audible ? 0.3 : 0,
|
||||
rms: audible ? 0.1 : 0,
|
||||
});
|
||||
}
|
||||
|
||||
const stats = getBridgeStats();
|
||||
expect(stats.nonSilentFrameCount).toBe(100);
|
||||
expect(stats.lastNonSilentFrameAt).toBe(startMs + 396 * 100);
|
||||
expect(stats.silentFrameStreak).toBe(3);
|
||||
expect(stats.maxSilentRunMs).toBe(300);
|
||||
expect(stats.sustainedSilenceWarned).toBe(false);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('bounds the retained capture history', () => {
|
||||
for (let index = 0; index < 12; index++) {
|
||||
startBridgeStats('generator', `native-audio:bounded-${index}`);
|
||||
}
|
||||
|
||||
expect(getEndedBridgeCaptures().length).toBeLessThanOrEqual(8);
|
||||
expect(findEndedCapture('native-audio:bounded-11')).toBeUndefined();
|
||||
expect(findEndedCapture('native-audio:bounded-10')).toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -1,22 +1,54 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {Logger} from '@app/features/platform/utils/AppLogger';
|
||||
import {
|
||||
initialBridgeStats,
|
||||
type NativeAudioBridgeEndedCapture,
|
||||
type NativeAudioBridgeFrameMetrics,
|
||||
type NativeAudioBridgeStats,
|
||||
} from '@app/features/voice/utils/native_audio_capture_bridge/shared';
|
||||
|
||||
const logger = new Logger('NativeAudioBridgeStats');
|
||||
|
||||
const MAX_ENDED_BRIDGE_CAPTURES = 8;
|
||||
const SUPERSEDED_END_REASON = 'superseded';
|
||||
const SUSTAINED_SILENCE_WARN_MS = 30_000;
|
||||
|
||||
let bridgeStats: NativeAudioBridgeStats = {...initialBridgeStats};
|
||||
let endedBridgeCaptures: Array<NativeAudioBridgeEndedCapture> = [];
|
||||
|
||||
export function getBridgeStats(): NativeAudioBridgeStats {
|
||||
return {...bridgeStats};
|
||||
}
|
||||
|
||||
export function getEndedBridgeCaptures(): Array<NativeAudioBridgeEndedCapture> {
|
||||
return endedBridgeCaptures.map((capture) => ({...capture}));
|
||||
}
|
||||
|
||||
function retireBridgeStats(): void {
|
||||
if (bridgeStats.captureId === null) return;
|
||||
endedBridgeCaptures.push({
|
||||
captureId: bridgeStats.captureId,
|
||||
bridgeMode: bridgeStats.bridgeMode,
|
||||
startedAt: bridgeStats.startedAt,
|
||||
endedAt: bridgeStats.endedAt ?? Date.now(),
|
||||
endReason: bridgeStats.endReason ?? SUPERSEDED_END_REASON,
|
||||
endDetail: bridgeStats.endDetail,
|
||||
framesReceived: bridgeStats.framesReceived,
|
||||
nonSilentFrameCount: bridgeStats.nonSilentFrameCount,
|
||||
lastFramePeak: bridgeStats.lastFramePeak,
|
||||
});
|
||||
if (endedBridgeCaptures.length > MAX_ENDED_BRIDGE_CAPTURES) {
|
||||
endedBridgeCaptures = endedBridgeCaptures.slice(-MAX_ENDED_BRIDGE_CAPTURES);
|
||||
}
|
||||
}
|
||||
|
||||
export function startBridgeStats(
|
||||
mode: 'generator' | 'script-processor',
|
||||
captureId: string,
|
||||
options: {prebufferTargetUs?: number; frameDurationUs?: number} = {},
|
||||
): void {
|
||||
retireBridgeStats();
|
||||
bridgeStats = {
|
||||
...initialBridgeStats,
|
||||
active: true,
|
||||
@@ -57,6 +89,24 @@ export function recordBridgeFrame(captureId: string, metrics: NativeAudioBridgeF
|
||||
bridgeStats.maxFrameRms = Math.max(bridgeStats.maxFrameRms, bridgeStats.lastFrameRms);
|
||||
if (peak >= 0.0005 || rms >= 0.0001) {
|
||||
bridgeStats.nonSilentFrameCount += 1;
|
||||
bridgeStats.lastNonSilentFrameAt = now;
|
||||
bridgeStats.silentFrameStreak = 0;
|
||||
} else {
|
||||
bridgeStats.silentFrameStreak += 1;
|
||||
const silentSinceMs = bridgeStats.lastNonSilentFrameAt ?? bridgeStats.startedAt ?? now;
|
||||
const silentRunMs = Math.max(0, now - silentSinceMs);
|
||||
bridgeStats.maxSilentRunMs = Math.max(bridgeStats.maxSilentRunMs, silentRunMs);
|
||||
if (!bridgeStats.sustainedSilenceWarned && silentRunMs >= SUSTAINED_SILENCE_WARN_MS) {
|
||||
bridgeStats.sustainedSilenceWarned = true;
|
||||
logger.warn('Native audio capture is delivering frames that contain only silence', {
|
||||
captureId: bridgeStats.captureId,
|
||||
bridgeMode: bridgeStats.bridgeMode,
|
||||
framesReceived: bridgeStats.framesReceived,
|
||||
nonSilentFrameCount: bridgeStats.nonSilentFrameCount,
|
||||
silentFrameStreak: bridgeStats.silentFrameStreak,
|
||||
silentRunMs,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -95,9 +145,16 @@ export function recordBridgeQueue(captureId: string, pendingFrames: number, buff
|
||||
}
|
||||
|
||||
export function endBridgeStats(captureId: string, reason: string | null, detail: string | null): void {
|
||||
if (!bridgeStats.active || bridgeStats.captureId !== captureId) return;
|
||||
bridgeStats.active = false;
|
||||
bridgeStats.endReason = reason;
|
||||
bridgeStats.endDetail = detail;
|
||||
bridgeStats.endedAt = Date.now();
|
||||
if (bridgeStats.active && bridgeStats.captureId === captureId) {
|
||||
bridgeStats.active = false;
|
||||
bridgeStats.endReason = reason;
|
||||
bridgeStats.endDetail = detail;
|
||||
bridgeStats.endedAt = Date.now();
|
||||
return;
|
||||
}
|
||||
const ended = endedBridgeCaptures.find((capture) => capture.captureId === captureId);
|
||||
if (!ended || ended.endReason !== SUPERSEDED_END_REASON) return;
|
||||
ended.endReason = reason;
|
||||
ended.endDetail = detail;
|
||||
ended.endedAt = Date.now();
|
||||
}
|
||||
|
||||
@@ -122,6 +122,10 @@ export interface NativeAudioBridgeStats {
|
||||
maxFramePeak: number;
|
||||
maxFrameRms: number;
|
||||
nonSilentFrameCount: number;
|
||||
lastNonSilentFrameAt: number | null;
|
||||
silentFrameStreak: number;
|
||||
maxSilentRunMs: number;
|
||||
sustainedSilenceWarned: boolean;
|
||||
prebufferTargetMs: number | null;
|
||||
frameDurationMs: number | null;
|
||||
endReason: string | null;
|
||||
@@ -129,6 +133,18 @@ export interface NativeAudioBridgeStats {
|
||||
endedAt: number | null;
|
||||
}
|
||||
|
||||
export interface NativeAudioBridgeEndedCapture {
|
||||
captureId: string;
|
||||
bridgeMode: 'generator' | 'script-processor' | null;
|
||||
startedAt: number | null;
|
||||
endedAt: number | null;
|
||||
endReason: string | null;
|
||||
endDetail: string | null;
|
||||
framesReceived: number;
|
||||
nonSilentFrameCount: number;
|
||||
lastFramePeak: number | null;
|
||||
}
|
||||
|
||||
export interface NativeAudioBridgeFrameMetrics {
|
||||
timestampUs?: number;
|
||||
durationUs?: number;
|
||||
@@ -156,6 +172,10 @@ export const initialBridgeStats: NativeAudioBridgeStats = {
|
||||
maxFramePeak: 0,
|
||||
maxFrameRms: 0,
|
||||
nonSilentFrameCount: 0,
|
||||
lastNonSilentFrameAt: null,
|
||||
silentFrameStreak: 0,
|
||||
maxSilentRunMs: 0,
|
||||
sustainedSilenceWarned: false,
|
||||
prebufferTargetMs: null,
|
||||
frameDurationMs: null,
|
||||
endReason: null,
|
||||
|
||||
@@ -119,5 +119,177 @@
|
||||
"remoteScreenShareTrackIndex": 3,
|
||||
"remoteScreenShareAudioTrackIndex": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "classifies a firefox screen share whose outbound stats omit the track identifier",
|
||||
"input": {
|
||||
"tracks": [
|
||||
{
|
||||
"direction": "recv",
|
||||
"kind": "audio",
|
||||
"trackIdentifier": "{8a106956-d202-4916-8e19-0a1672c64b73}",
|
||||
"bitrateKbps": 46
|
||||
},
|
||||
{"direction": "send", "kind": "audio", "bitrateKbps": 30},
|
||||
{"direction": "send", "kind": "video", "bitrateKbps": 47}
|
||||
],
|
||||
"publications": {
|
||||
"localCameraTrackId": null,
|
||||
"localMicrophoneTrackId": "b0d2f1a7-6c3e-4f8a-9b21-5d7c4e0a1f36",
|
||||
"localScreenShareTrackId": "2f4c8de1-9a07-4b53-8c6d-1e5b7a02d94f",
|
||||
"localScreenShareAudioTrackId": null,
|
||||
"remoteMicrophoneTrackIds": ["8a106956-d202-4916-8e19-0a1672c64b73"],
|
||||
"remoteScreenShareTrackIds": [],
|
||||
"remoteScreenShareAudioTrackIds": []
|
||||
}
|
||||
},
|
||||
"expected": {
|
||||
"localVideoTrackIndex": null,
|
||||
"localAudioTrackIndex": 1,
|
||||
"localScreenShareTrackIndex": 2,
|
||||
"localScreenShareAudioTrackIndex": null,
|
||||
"remoteVideoTrackIndex": null,
|
||||
"remoteAudioTrackIndex": 0,
|
||||
"remoteScreenShareTrackIndex": null,
|
||||
"remoteScreenShareAudioTrackIndex": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "classifies a firefox inbound screen share reported with a braced track identifier",
|
||||
"input": {
|
||||
"tracks": [
|
||||
{
|
||||
"direction": "recv",
|
||||
"kind": "audio",
|
||||
"trackIdentifier": "{9010d313-264b-4c51-9f13-9a6aefafce83}",
|
||||
"bitrateKbps": 29
|
||||
},
|
||||
{
|
||||
"direction": "recv",
|
||||
"kind": "video",
|
||||
"trackIdentifier": "{34171e0d-174d-4d56-bb71-76b4fba330c3}",
|
||||
"bitrateKbps": 3585
|
||||
},
|
||||
{"direction": "send", "kind": "audio", "bitrateKbps": 44}
|
||||
],
|
||||
"publications": {
|
||||
"localCameraTrackId": null,
|
||||
"localMicrophoneTrackId": "7c5a1b98-3d24-4e6f-8a10-9b2c3d4e5f60",
|
||||
"localScreenShareTrackId": null,
|
||||
"localScreenShareAudioTrackId": null,
|
||||
"remoteMicrophoneTrackIds": ["9010d313-264b-4c51-9f13-9a6aefafce83"],
|
||||
"remoteScreenShareTrackIds": ["34171e0d-174d-4d56-bb71-76b4fba330c3"],
|
||||
"remoteScreenShareAudioTrackIds": []
|
||||
}
|
||||
},
|
||||
"expected": {
|
||||
"localVideoTrackIndex": null,
|
||||
"localAudioTrackIndex": 2,
|
||||
"localScreenShareTrackIndex": null,
|
||||
"localScreenShareAudioTrackIndex": null,
|
||||
"remoteVideoTrackIndex": null,
|
||||
"remoteAudioTrackIndex": 0,
|
||||
"remoteScreenShareTrackIndex": 1,
|
||||
"remoteScreenShareAudioTrackIndex": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "leaves both local video roles unresolved when unidentified sends could be camera or screen share",
|
||||
"input": {
|
||||
"tracks": [
|
||||
{"direction": "send", "kind": "video", "bitrateKbps": 1600},
|
||||
{"direction": "send", "kind": "video", "bitrateKbps": 500}
|
||||
],
|
||||
"publications": {
|
||||
"localCameraTrackId": "c1e7d3b0-88f4-45a9-9c62-0ab5d1e2f374",
|
||||
"localMicrophoneTrackId": null,
|
||||
"localScreenShareTrackId": "2f4c8de1-9a07-4b53-8c6d-1e5b7a02d94f",
|
||||
"localScreenShareAudioTrackId": null,
|
||||
"remoteMicrophoneTrackIds": [],
|
||||
"remoteScreenShareTrackIds": [],
|
||||
"remoteScreenShareAudioTrackIds": []
|
||||
}
|
||||
},
|
||||
"expected": {
|
||||
"localVideoTrackIndex": null,
|
||||
"localAudioTrackIndex": null,
|
||||
"localScreenShareTrackIndex": null,
|
||||
"localScreenShareAudioTrackIndex": null,
|
||||
"remoteVideoTrackIndex": null,
|
||||
"remoteAudioTrackIndex": null,
|
||||
"remoteScreenShareTrackIndex": null,
|
||||
"remoteScreenShareAudioTrackIndex": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "keeps chromium outbound identifier matching for a screen share published without a camera",
|
||||
"input": {
|
||||
"tracks": [
|
||||
{
|
||||
"direction": "send",
|
||||
"kind": "video",
|
||||
"trackIdentifier": "97079fec-a3af-4bc0-a8e6-ad142dc05c83",
|
||||
"bitrateKbps": 1600
|
||||
},
|
||||
{"direction": "send", "kind": "audio", "bitrateKbps": 30}
|
||||
],
|
||||
"publications": {
|
||||
"localCameraTrackId": null,
|
||||
"localMicrophoneTrackId": null,
|
||||
"localScreenShareTrackId": "97079fec-a3af-4bc0-a8e6-ad142dc05c83",
|
||||
"localScreenShareAudioTrackId": null,
|
||||
"remoteMicrophoneTrackIds": [],
|
||||
"remoteScreenShareTrackIds": [],
|
||||
"remoteScreenShareAudioTrackIds": []
|
||||
}
|
||||
},
|
||||
"expected": {
|
||||
"localVideoTrackIndex": null,
|
||||
"localAudioTrackIndex": 1,
|
||||
"localScreenShareTrackIndex": 0,
|
||||
"localScreenShareAudioTrackIndex": null,
|
||||
"remoteVideoTrackIndex": null,
|
||||
"remoteAudioTrackIndex": null,
|
||||
"remoteScreenShareTrackIndex": null,
|
||||
"remoteScreenShareAudioTrackIndex": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "keeps chromium identified sends eligible for the camera role while a screen share is published",
|
||||
"input": {
|
||||
"tracks": [
|
||||
{
|
||||
"direction": "send",
|
||||
"kind": "video",
|
||||
"trackIdentifier": "97079fec-a3af-4bc0-a8e6-ad142dc05c83",
|
||||
"bitrateKbps": 1600
|
||||
},
|
||||
{
|
||||
"direction": "send",
|
||||
"kind": "video",
|
||||
"trackIdentifier": "1d0a4c62-5b3e-4f71-88d9-6c2a7e50b431",
|
||||
"bitrateKbps": 400
|
||||
}
|
||||
],
|
||||
"publications": {
|
||||
"localCameraTrackId": null,
|
||||
"localMicrophoneTrackId": null,
|
||||
"localScreenShareTrackId": "97079fec-a3af-4bc0-a8e6-ad142dc05c83",
|
||||
"localScreenShareAudioTrackId": null,
|
||||
"remoteMicrophoneTrackIds": [],
|
||||
"remoteScreenShareTrackIds": [],
|
||||
"remoteScreenShareAudioTrackIds": []
|
||||
}
|
||||
},
|
||||
"expected": {
|
||||
"localVideoTrackIndex": 1,
|
||||
"localAudioTrackIndex": null,
|
||||
"localScreenShareTrackIndex": 0,
|
||||
"localScreenShareAudioTrackIndex": null,
|
||||
"remoteVideoTrackIndex": null,
|
||||
"remoteAudioTrackIndex": null,
|
||||
"remoteScreenShareTrackIndex": null,
|
||||
"remoteScreenShareAudioTrackIndex": null
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -513,24 +513,55 @@ function normalizedId(id: string | null | undefined): string | null {
|
||||
return trimmed ? trimmed : null;
|
||||
}
|
||||
|
||||
function canonicalId(id: string | null | undefined): string | null {
|
||||
const normalized = normalizedId(id);
|
||||
if (normalized === null) return null;
|
||||
const unwrapped =
|
||||
normalized.startsWith('{') && normalized.endsWith('}') ? normalized.slice(1, -1).trim() : normalized;
|
||||
return unwrapped ? unwrapped.toLowerCase() : null;
|
||||
}
|
||||
|
||||
function idList(ids: ReadonlyArray<string>): Array<string> {
|
||||
return ids.map((id) => normalizedId(id)).filter((id): id is string => id !== null);
|
||||
return ids.map((id) => canonicalId(id)).filter((id): id is string => id !== null);
|
||||
}
|
||||
|
||||
function matchesId(candidate: IndexedVoiceEngineV2StatsTrack, id: string | null): boolean {
|
||||
const trackId = normalizedId(candidate.track.trackIdentifier);
|
||||
return trackId !== null && id !== null && trackId === id;
|
||||
const trackId = canonicalId(candidate.track.trackIdentifier);
|
||||
const publicationId = canonicalId(id);
|
||||
return trackId !== null && publicationId !== null && trackId === publicationId;
|
||||
}
|
||||
|
||||
function idInList(candidate: IndexedVoiceEngineV2StatsTrack, ids: ReadonlyArray<string>): boolean {
|
||||
const trackId = normalizedId(candidate.track.trackIdentifier);
|
||||
const trackId = canonicalId(candidate.track.trackIdentifier);
|
||||
return trackId !== null && ids.includes(trackId);
|
||||
}
|
||||
|
||||
function hasId(candidate: IndexedVoiceEngineV2StatsTrack): boolean {
|
||||
return canonicalId(candidate.track.trackIdentifier) !== null;
|
||||
}
|
||||
|
||||
function hasRid(candidate: IndexedVoiceEngineV2StatsTrack): boolean {
|
||||
return normalizedId(candidate.track.rid) !== null;
|
||||
}
|
||||
|
||||
function couldBeLocalScreenShare(
|
||||
candidate: IndexedVoiceEngineV2StatsTrack,
|
||||
localScreenShareTrackId: string | null,
|
||||
): boolean {
|
||||
if (localScreenShareTrackId === null) return false;
|
||||
return matchesId(candidate, localScreenShareTrackId) || !hasId(candidate);
|
||||
}
|
||||
|
||||
function unidentifiedLocalScreenShareIndex(
|
||||
sentVideoTracks: ReadonlyArray<IndexedVoiceEngineV2StatsTrack>,
|
||||
localCameraTrackId: string | null,
|
||||
localScreenShareTrackId: string | null,
|
||||
): number | null {
|
||||
if (localScreenShareTrackId === null || localCameraTrackId !== null || sentVideoTracks.length !== 1) return null;
|
||||
const candidate = sentVideoTracks[0];
|
||||
return candidate && !hasId(candidate) ? candidate.index : null;
|
||||
}
|
||||
|
||||
function filterById(
|
||||
candidates: ReadonlyArray<IndexedVoiceEngineV2StatsTrack>,
|
||||
id: string | null,
|
||||
@@ -565,19 +596,25 @@ export function classifyVoiceEngineV2TrackStats(
|
||||
const remoteScreenShareTracks = receivedVideoTracks.filter((track) => idInList(track, remoteScreenShareTrackIds));
|
||||
const remoteScreenShareTrackIndex = pickActiveTrackIndex(remoteScreenShareTracks);
|
||||
const fallbackRemoteVideoTracks = receivedVideoTracks.filter((track) => !idInList(track, remoteScreenShareTrackIds));
|
||||
const localScreenShareFallbackIndex = unidentifiedLocalScreenShareIndex(
|
||||
sentVideoTracks,
|
||||
localCameraTrackId,
|
||||
localScreenShareTrackId,
|
||||
);
|
||||
|
||||
return {
|
||||
localVideoTrackIndex:
|
||||
pickActiveTrackIndex(filterById(sentVideoTracks, localCameraTrackId)) ??
|
||||
pickActiveTrackIndex(
|
||||
sentVideoTracks.filter((track) => !hasRid(track) && !matchesId(track, localScreenShareTrackId)),
|
||||
sentVideoTracks.filter((track) => !hasRid(track) && !couldBeLocalScreenShare(track, localScreenShareTrackId)),
|
||||
),
|
||||
localAudioTrackIndex:
|
||||
pickActiveTrackIndex(filterById(sentAudioTracks, localMicrophoneTrackId)) ??
|
||||
pickActiveTrackIndex(sentAudioTracks.filter((track) => !matchesId(track, localScreenShareAudioTrackId))),
|
||||
localScreenShareTrackIndex:
|
||||
pickActiveTrackIndex(filterById(sentVideoTracks, localScreenShareTrackId)) ??
|
||||
pickActiveTrackIndex(sentVideoTracks.filter((track) => hasRid(track) && !matchesId(track, localCameraTrackId))),
|
||||
pickActiveTrackIndex(sentVideoTracks.filter((track) => hasRid(track) && !matchesId(track, localCameraTrackId))) ??
|
||||
localScreenShareFallbackIndex,
|
||||
localScreenShareAudioTrackIndex: pickActiveTrackIndex(filterById(sentAudioTracks, localScreenShareAudioTrackId)),
|
||||
remoteVideoTrackIndex:
|
||||
pickActiveTrackIndex(fallbackRemoteVideoTracks) ??
|
||||
|
||||
@@ -499,12 +499,14 @@ export interface VoiceEngineV2PerTrackStats {
|
||||
kind: 'audio' | 'video' | 'unknown';
|
||||
ssrc?: number;
|
||||
rid?: string;
|
||||
active?: boolean;
|
||||
mid?: string;
|
||||
trackIdentifier?: string;
|
||||
mediaSourceId?: string;
|
||||
codec?: string;
|
||||
payloadType?: number;
|
||||
bitrateKbps: number;
|
||||
bitrateWindowMs?: number;
|
||||
packetsLost?: number;
|
||||
packetsLossPercent?: number;
|
||||
jitterMs?: number;
|
||||
|
||||
Reference in New Issue
Block a user