fix(voice): record why a screen share stopped (#2664)

This commit is contained in:
Hampus authored and GitHub committed 2026-09-11 12:59:51 +02:00
1 parent 5da256df12
commit 3a862f1484
11 files changed
+594 -36

No files matched your search

@@ -0,0 +1,81 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
type VoiceEngineV2AppScreenShareControllerGateway,
VoiceEngineV2AppScreenShareControllerRouting,
} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareControllerRouting';
import type {VoiceEngineV2AppScreenShareExecutionAdapter} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter';
import {
getRecentScreenShares,
recordScreenShareStarted,
resetRecentScreenSharesForTests,
} from '@app/features/voice/utils/ScreenShareLifecycleLog';
import {beforeEach, describe, expect, it, vi} from 'vitest';
vi.mock('@app/features/voice/engine/voice_screen_share_manager/shared', () => ({
logger: {debug: () => undefined, info: () => undefined, warn: () => undefined, error: () => undefined},
}));
vi.mock('@app/features/voice/state/LocalVoiceState', () => ({
default: {getSelfStream: () => true},
}));
const setEnabled = vi.fn(async () => undefined);
function createAdapter(): VoiceEngineV2AppScreenShareExecutionAdapter {
return {liveKitFlows: {setEnabled}} as unknown as VoiceEngineV2AppScreenShareExecutionAdapter;
}
function createGateway(
overrides: Partial<VoiceEngineV2AppScreenShareControllerGateway> = {},
): VoiceEngineV2AppScreenShareControllerGateway {
return {
isScreenCommandRoutable: () => true,
hasScreenPublication: () => true,
hasScreenDesired: () => false,
clearScreenDesired: () => undefined,
executingScreenOperationId: () => null,
isScreenOperationPending: () => true,
publishScreen: async () => undefined,
unpublishScreen: async () => undefined,
...overrides,
};
}
describe('routed screen share stop attribution', () => {
beforeEach(() => {
resetRecentScreenSharesForTests();
setEnabled.mockClear();
});
it('keeps a stop the user asked for as a user stop when the controller carries it out', async () => {
const routing = new VoiceEngineV2AppScreenShareControllerRouting(createAdapter());
let executingOperationId: number | null = null;
routing.setGateway(
createGateway({
executingScreenOperationId: () => executingOperationId,
unpublishScreen: async (onPlanned) => {
onPlanned([7]);
executingOperationId = 7;
await routing.unpublishViaLiveKitFlows(null);
},
}),
);
recordScreenShareStarted();
await routing.setEnabled(null, false, {sendUpdate: true, playSound: true});
expect(setEnabled).toHaveBeenCalledTimes(1);
expect(getRecentScreenShares()[0].stopTrigger).toBe('user');
});
it('records an unpublish the app never asked for as a gateway echo', async () => {
const routing = new VoiceEngineV2AppScreenShareControllerRouting(createAdapter());
routing.setGateway(createGateway());
recordScreenShareStarted();
await routing.unpublishViaLiveKitFlows(null);
expect(getRecentScreenShares()[0].stopTrigger).toBe('gateway-echo');
});
});
@@ -4,6 +4,7 @@ import assert from 'node:assert/strict';
import type {VoiceEngineV2AppScreenShareExecutionAdapter} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter'; import type {VoiceEngineV2AppScreenShareExecutionAdapter} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter';
import {logger} from '@app/features/voice/engine/voice_screen_share_manager/shared'; import {logger} from '@app/features/voice/engine/voice_screen_share_manager/shared';
import LocalVoiceState from '@app/features/voice/state/LocalVoiceState'; import LocalVoiceState from '@app/features/voice/state/LocalVoiceState';
import {recordScreenShareStopped} from '@app/features/voice/utils/ScreenShareLifecycleLog';
import type {VoiceEngineV2ScreenOptions} from '@fluxer/voice_engine_v2'; import type {VoiceEngineV2ScreenOptions} from '@fluxer/voice_engine_v2';
import type {Room, ScreenShareCaptureOptions, TrackPublishOptions, VideoCodec} from 'livekit-client'; import type {Room, ScreenShareCaptureOptions, TrackPublishOptions, VideoCodec} from 'livekit-client';
@@ -445,6 +446,7 @@ export class VoiceEngineV2AppScreenShareControllerRouting {
async unpublishViaLiveKitFlows(roomFromPort: Room | null): Promise<void> { async unpublishViaLiveKitFlows(roomFromPort: Room | null): Promise<void> {
const request = this.takeStopRequest(this.executingScreenOperationId()); const request = this.takeStopRequest(this.executingScreenOperationId());
recordScreenShareStopped(request !== null ? 'user' : 'gateway-echo');
const room = request !== null ? request.room : roomFromPort; const room = request !== null ? request.room : roomFromPort;
try { try {
await this.adapter.liveKitFlows.setEnabled(room, false, { await this.adapter.liveKitFlows.setEnabled(room, false, {
@@ -78,6 +78,15 @@ import {
disarmNativeAudio, disarmNativeAudio,
reconfigureLinuxNativeAudioRouting, reconfigureLinuxNativeAudioRouting,
} from '@app/features/voice/utils/NativeAudioCaptureBridge'; } from '@app/features/voice/utils/NativeAudioCaptureBridge';
import {
recordScreenShareEncoderVerification,
recordScreenShareEndedModal,
recordScreenShareRequestedCodec,
recordScreenShareStartError,
recordScreenShareStarted,
recordScreenShareStopped,
type ScreenShareEndedModal,
} from '@app/features/voice/utils/ScreenShareLifecycleLog';
import {SCREEN_SHARE_DEGRADATION_PREFERENCE} from '@app/features/voice/utils/ScreenShareOptions'; import {SCREEN_SHARE_DEGRADATION_PREFERENCE} from '@app/features/voice/utils/ScreenShareOptions';
import {ScreenShareRollbackIncompleteError} from '@app/features/voice/utils/ScreenShareRollbackIncompleteError'; import {ScreenShareRollbackIncompleteError} from '@app/features/voice/utils/ScreenShareRollbackIncompleteError';
import {handleScreenShareError} from '@app/features/voice/utils/ScreenShareUtils'; import {handleScreenShareError} from '@app/features/voice/utils/ScreenShareUtils';
@@ -400,7 +409,8 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
this.transitionScreenShareLifecycleInternal({type: 'share.encoderVerification.cleared'}); this.transitionScreenShareLifecycleInternal({type: 'share.encoderVerification.cleared'});
} }
showScreenShareEndedModalInternal(description: string): void { showScreenShareEndedModalInternal(description: string, modal: ScreenShareEndedModal): void {
recordScreenShareEndedModal(modal);
ModalCommands.pushWithKey( ModalCommands.pushWithKey(
ModalCommands.modal(() => ( ModalCommands.modal(() => (
<GenericErrorModal <GenericErrorModal
@@ -491,9 +501,11 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
i18n._( i18n._(
cause === 'stalled' ? SCREEN_SHARE_ENCODER_FAILED_DESCRIPTOR : SCREEN_SHARE_CODEC_POLICY_FAILED_DESCRIPTOR, cause === 'stalled' ? SCREEN_SHARE_ENCODER_FAILED_DESCRIPTOR : SCREEN_SHARE_CODEC_POLICY_FAILED_DESCRIPTOR,
), ),
cause === 'stalled' ? 'encoder-failed' : 'codec-policy-failed',
); );
if (this.endedScreenShareStopInFlight) return; if (this.endedScreenShareStopInFlight) return;
if (!this.isScreenShareTrackPublishedInternal(participant, track)) return; if (!this.isScreenShareTrackPublishedInternal(participant, track)) return;
recordScreenShareStopped('codec-republish-failed');
this.transitionScreenShareLifecycleInternal({type: 'share.endedStop.start'}); this.transitionScreenShareLifecycleInternal({type: 'share.endedStop.start'});
const stopPromise = this.setScreenShareEnabled(room, false, {sendUpdate: true, playSound: true}) const stopPromise = this.setScreenShareEnabled(room, false, {sendUpdate: true, playSound: true})
.catch((error) => { .catch((error) => {
@@ -516,6 +528,7 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
): void { ): void {
this.cancelEncoderVerificationInternal(); this.cancelEncoderVerificationInternal();
if (!codec) return; if (!codec) return;
recordScreenShareRequestedCodec(codec);
const publication = preferredTrack ? undefined : participant.getTrackPublication(Track.Source.ScreenShare); const publication = preferredTrack ? undefined : participant.getTrackPublication(Track.Source.ScreenShare);
const track = preferredTrack ?? (publication?.videoTrack as LocalVideoTrack | undefined); const track = preferredTrack ?? (publication?.videoTrack as LocalVideoTrack | undefined);
const sender = track?.sender; const sender = track?.sender;
@@ -530,8 +543,10 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
const action = resolveScreenShareEncoderVerificationAction(failure); const action = resolveScreenShareEncoderVerificationAction(failure);
switch (action.kind) { switch (action.kind) {
case 'ignore-repeated-stall': case 'ignore-repeated-stall':
recordScreenShareEncoderVerification('ignore-repeated-stall', null);
return; return;
case 'recover-stalled': case 'recover-stalled':
recordScreenShareEncoderVerification('recover-stalled', null);
logger.warn('Screen share encoder verification failed', { logger.warn('Screen share encoder verification failed', {
codec: action.codec, codec: action.codec,
failureReason: 'screen-share-encode-stalled', failureReason: 'screen-share-encode-stalled',
@@ -546,12 +561,14 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
); );
return; return;
case 'accept-negotiated': case 'accept-negotiated':
recordScreenShareEncoderVerification('accept-negotiated', action.negotiated);
logger.info('Screen share publisher negotiated a different codec inside the publish policy', { logger.info('Screen share publisher negotiated a different codec inside the publish policy', {
requested: action.requested, requested: action.requested,
negotiated: action.negotiated, negotiated: action.negotiated,
}); });
return; return;
case 'correct-negotiated': case 'correct-negotiated':
recordScreenShareEncoderVerification('correct-negotiated', action.negotiated);
logger.warn('Screen share is sending a codec outside the publish policy', { logger.warn('Screen share is sending a codec outside the publish policy', {
requested: action.requested, requested: action.requested,
negotiated: action.negotiated, negotiated: action.negotiated,
@@ -599,7 +616,8 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
trigger, trigger,
readyState: mediaStreamTrack.readyState, readyState: mediaStreamTrack.readyState,
}); });
this.showScreenShareEndedModalInternal(i18n._(SCREEN_SHARE_SOURCE_STOPPED_DESCRIPTOR)); this.showScreenShareEndedModalInternal(i18n._(SCREEN_SHARE_SOURCE_STOPPED_DESCRIPTOR), 'source-stopped');
recordScreenShareStopped('media-track-ended');
this.transitionScreenShareLifecycleInternal({type: 'share.endedStop.start'}); this.transitionScreenShareLifecycleInternal({type: 'share.endedStop.start'});
const stopPromise = this.setScreenShareEnabled(room, false, {sendUpdate: true, playSound: true}) const stopPromise = this.setScreenShareEnabled(room, false, {sendUpdate: true, playSound: true})
.catch((error) => { .catch((error) => {
@@ -663,6 +681,7 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
} }
handleLocalScreenShareTrackUnpublished(room: Room, playSound: boolean, publication?: LocalTrackPublication): void { handleLocalScreenShareTrackUnpublished(room: Room, playSound: boolean, publication?: LocalTrackPublication): void {
recordScreenShareStopped(this.isScreenSharePending ? 'user' : 'server-unpublish');
this.clearScreenShareKeepAliveSinkInternal(); this.clearScreenShareKeepAliveSinkInternal();
this.cleanupActiveScreenShareEndListenerInternal(); this.cleanupActiveScreenShareEndListenerInternal();
const participant = room.localParticipant; const participant = room.localParticipant;
@@ -866,7 +885,17 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
publishOptions?: TrackPublishOptions, publishOptions?: TrackPublishOptions,
): Promise<void> { ): Promise<void> {
assert.equal(typeof enabled, 'boolean'); assert.equal(typeof enabled, 'boolean');
await this.controllerRouting.setEnabled(room, enabled, options, publishOptions); if (enabled) {
recordScreenShareStarted();
}
try {
await this.controllerRouting.setEnabled(room, enabled, options, publishOptions);
} catch (error) {
if (enabled) {
recordScreenShareStartError(error);
}
throw error;
}
if (!enabled && options?.preserveStreamAudioPreferences !== true) { if (!enabled && options?.preserveStreamAudioPreferences !== true) {
VoiceSettingsCommands.update({ VoiceSettingsCommands.update({
shareAppAudio: true, shareAppAudio: true,
@@ -908,7 +937,13 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
options?: DeviceScreenShareCaptureOptions, options?: DeviceScreenShareCaptureOptions,
publishOptions?: TrackPublishOptions, publishOptions?: TrackPublishOptions,
): Promise<void> { ): Promise<void> {
await this.liveKitFlows.startDeviceScreenShare(room, options, publishOptions); recordScreenShareStarted();
try {
await this.liveKitFlows.startDeviceScreenShare(room, options, publishOptions);
} catch (error) {
recordScreenShareStartError(error);
throw error;
}
} }
async replaceActiveDisplayScreenShare( async replaceActiveDisplayScreenShare(
@@ -1,12 +1,138 @@
// SPDX-License-Identifier: AGPL-3.0-or-later // SPDX-License-Identifier: AGPL-3.0-or-later
import {getNativeAudioCaptureDiagnosticState} from '@app/features/voice/utils/NativeAudioCaptureBridge'; import type {VoiceEngineV2AppSourceLifecycleBridge} from '@app/features/voice/engine/v2/VoiceEngineV2AppSourceLifecycleBridge';
import {
armNativeSystemAudioForNextCapture,
captureNativeAudioTrackForLinuxRouting,
getNativeAudioCaptureDiagnosticState,
resetNativeAudioCaptureBridgeForTests,
setNativeAudioCaptureBridgeLifecycleBridge,
} from '@app/features/voice/utils/NativeAudioCaptureBridge';
import {recordBridgeFrame, startBridgeStats} from '@app/features/voice/utils/native_audio_capture_bridge/bridgeStats'; import {recordBridgeFrame, startBridgeStats} from '@app/features/voice/utils/native_audio_capture_bridge/bridgeStats';
import type { import type {
NativeAudioBridgeEndedCapture, NativeAudioBridgeEndedCapture,
NativeAudioBridgeStats, NativeAudioBridgeStats,
} from '@app/features/voice/utils/native_audio_capture_bridge/shared'; } from '@app/features/voice/utils/native_audio_capture_bridge/shared';
import {describe, expect, it} from 'vitest'; import {beforeEach, describe, expect, it, vi} from 'vitest';
const electronApi = {current: null as Record<string, unknown> | null};
vi.mock('@app/features/ui/utils/NativeUtils', () => ({
getElectronAPI: () => electronApi.current,
}));
vi.mock('@app/features/voice/utils/SelfWindowScreenShareAudioMix', () => ({
mixTrackWithSelfWindowScreenShareAudio: async (primaryTrack: MediaStreamTrack) => ({
track: primaryTrack,
cleanup: async () => undefined,
}),
}));
const CAPTURE_ID = 'native-audio:stop-cause';
class FakeMediaStreamTrack extends EventTarget {
readyState: 'live' | 'ended' = 'live';
constructor(readonly kind: 'audio' | 'video') {
super();
}
stop(): void {
this.readyState = 'ended';
}
}
class FakeGeneratorTrack extends FakeMediaStreamTrack {
readonly writable = new WritableStream();
constructor(_options: {kind: 'audio'}) {
super('audio');
}
}
class FakeMediaStream {
private readonly tracks: Array<FakeMediaStreamTrack>;
constructor(tracks: Array<FakeMediaStreamTrack> = []) {
this.tracks = [...tracks];
}
getTracks(): Array<FakeMediaStreamTrack> {
return [...this.tracks];
}
getVideoTracks(): Array<FakeMediaStreamTrack> {
return this.tracks.filter((track) => track.kind === 'video');
}
getAudioTracks(): Array<FakeMediaStreamTrack> {
return this.tracks.filter((track) => track.kind === 'audio');
}
addTrack(track: FakeMediaStreamTrack): void {
this.tracks.push(track);
}
removeTrack(track: FakeMediaStreamTrack): void {
const index = this.tracks.indexOf(track);
if (index >= 0) this.tracks.splice(index, 1);
}
}
const stopNativeCapture = vi.fn(async () => undefined);
const reportLifecycle = vi.fn(() => true);
function createNativeAudioApi() {
return {
getAvailability: async () => ({
available: true,
backend: 'windows-wasapi-loopback',
capabilities: {process: true, system: true, systemExcludesSelf: true},
}),
start: async () => ({captureId: CAPTURE_ID, sampleRate: 48000, channels: 2}),
stop: stopNativeCapture,
onFrame: () => () => undefined,
onEnd: () => () => undefined,
};
}
function installLifecycleBridge(): void {
setNativeAudioCaptureBridgeLifecycleBridge({
bind: () => true,
unbind: () => undefined,
reportLifecycle,
} as unknown as VoiceEngineV2AppSourceLifecycleBridge);
}
async function flushCleanup(): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, 0));
await new Promise((resolve) => setTimeout(resolve, 0));
}
async function startDisplayShareWithNativeAudio(): Promise<{
videoTrack: FakeMediaStreamTrack;
audioTrack: FakeMediaStreamTrack;
}> {
const videoTrack = new FakeMediaStreamTrack('video');
const capturedStream = new FakeMediaStream([videoTrack]);
Object.defineProperty(globalThis, 'navigator', {
configurable: true,
value: {mediaDevices: {getDisplayMedia: async () => capturedStream}},
});
expect(await armNativeSystemAudioForNextCapture()).toBe(true);
const stream = (await navigator.mediaDevices.getDisplayMedia({video: true})) as unknown as FakeMediaStream;
const audioTrack = stream.getAudioTracks()[0];
expect(audioTrack).toBeInstanceOf(FakeGeneratorTrack);
return {videoTrack, audioTrack};
}
function currentBridgeStats(): NativeAudioBridgeStats {
return getNativeAudioCaptureDiagnosticState().bridgeStats as NativeAudioBridgeStats;
}
function currentLifecycleFaults(): Array<{message: string}> {
return getNativeAudioCaptureDiagnosticState().lifecycleFaults as Array<{message: string}>;
}
describe('getNativeAudioCaptureDiagnosticState', () => { describe('getNativeAudioCaptureDiagnosticState', () => {
it('keeps the six fields real dumps already carry and appends the retained histories', () => { it('keeps the six fields real dumps already carry and appends the retained histories', () => {
@@ -41,3 +167,93 @@ describe('getNativeAudioCaptureDiagnosticState', () => {
expect(Array.isArray(getNativeAudioCaptureDiagnosticState().lifecycleFaults)).toBe(true); expect(Array.isArray(getNativeAudioCaptureDiagnosticState().lifecycleFaults)).toBe(true);
}); });
}); });
describe('native audio capture teardown cause', () => {
beforeEach(() => {
resetNativeAudioCaptureBridgeForTests();
Object.defineProperty(globalThis, 'window', {
configurable: true,
value: {MediaStreamTrackGenerator: FakeGeneratorTrack, AudioData: class {}},
});
Object.defineProperty(globalThis, 'MediaStream', {configurable: true, value: FakeMediaStream});
electronApi.current = {platform: 'win32', nativeAudio: createNativeAudioApi()};
stopNativeCapture.mockClear();
reportLifecycle.mockClear();
installLifecycleBridge();
});
it('records caller-stopped with no fault when the share is stopped by hand', async () => {
const {audioTrack} = await startDisplayShareWithNativeAudio();
audioTrack.stop();
await flushCleanup();
expect(currentLifecycleFaults()).toEqual([]);
expect(reportLifecycle).not.toHaveBeenCalled();
expect(currentBridgeStats().endDetail).toBe('caller-stopped');
expect(stopNativeCapture).toHaveBeenCalledTimes(1);
});
it('names the audio track when the capture track ends on its own', async () => {
const {audioTrack} = await startDisplayShareWithNativeAudio();
audioTrack.dispatchEvent(new Event('ended'));
await flushCleanup();
expect(currentLifecycleFaults().map((fault) => fault.message)).toEqual(['native-audio-tap-audio-track-ended']);
expect(reportLifecycle).toHaveBeenCalledWith({
captureId: CAPTURE_ID,
kind: 'error',
message: 'native-audio-tap-audio-track-ended',
});
expect(currentBridgeStats().endDetail).toBe('audio-track-ended');
});
it('names the video track when the shared screen stops sending frames', async () => {
const {videoTrack} = await startDisplayShareWithNativeAudio();
videoTrack.dispatchEvent(new Event('ended'));
await flushCleanup();
expect(currentLifecycleFaults().map((fault) => fault.message)).toEqual(['native-audio-tap-video-track-ended']);
expect(currentBridgeStats().endDetail).toBe('video-track-ended');
});
it('tears down once and keeps the track-ended cause when a stop follows it', async () => {
const {audioTrack} = await startDisplayShareWithNativeAudio();
audioTrack.dispatchEvent(new Event('ended'));
audioTrack.stop();
await flushCleanup();
expect(currentLifecycleFaults()).toHaveLength(1);
expect(currentBridgeStats().endDetail).toBe('audio-track-ended');
expect(stopNativeCapture).toHaveBeenCalledTimes(1);
});
it('names the audio track when a capture mixed with self-window audio ends on its own', async () => {
electronApi.current = {platform: 'linux', nativeAudio: createNativeAudioApi()};
const track = (await captureNativeAudioTrackForLinuxRouting(
{include: []},
{includeSelfWindowAudio: true},
)) as unknown as FakeMediaStreamTrack | null;
expect(track).toBeInstanceOf(FakeGeneratorTrack);
track?.dispatchEvent(new Event('ended'));
await flushCleanup();
expect(currentBridgeStats().endDetail).toBe('audio-track-ended');
});
it('tears down once and keeps caller-stopped when the track ends after the stop', async () => {
const {audioTrack} = await startDisplayShareWithNativeAudio();
audioTrack.stop();
audioTrack.dispatchEvent(new Event('ended'));
await flushCleanup();
expect(currentLifecycleFaults()).toEqual([]);
expect(currentBridgeStats().endDetail).toBe('caller-stopped');
expect(stopNativeCapture).toHaveBeenCalledTimes(1);
});
});
@@ -113,15 +113,15 @@ function recordLifecycleFault(captureId: string, message: string): void {
} }
} }
function unbindNativeAudioCaptureLifecycle(captureId: string, faulted: boolean): void { function unbindNativeAudioCaptureLifecycle(captureId: string, fault: string | null): void {
if (!lifecycleBoundCaptureIds.has(captureId)) return; if (!lifecycleBoundCaptureIds.has(captureId)) return;
const bridge = sourceLifecycleBridge; const bridge = sourceLifecycleBridge;
if (faulted) { if (fault) {
recordLifecycleFault(captureId, 'native-audio-tap-track-ended'); recordLifecycleFault(captureId, fault);
} }
if (bridge) { if (bridge) {
if (faulted) { if (fault) {
bridge.reportLifecycle({captureId, kind: 'error', message: 'native-audio-tap-track-ended'}); bridge.reportLifecycle({captureId, kind: 'error', message: fault});
} }
bridge.unbind(captureId); bridge.unbind(captureId);
} }
@@ -195,8 +195,13 @@ export function getNativeAudioCaptureDiagnosticState(): Record<string, unknown>
}; };
} }
function cleanupBridgeAsync(bridge: ActiveNativeAudioBridge, stopRemote: boolean, logContext: string): void { function cleanupBridgeAsync(
void bridge.cleanup(stopRemote).catch((error) => { bridge: ActiveNativeAudioBridge,
stopRemote: boolean,
logContext: string,
endDetail?: string,
): void {
void bridge.cleanup(stopRemote, endDetail).catch((error) => {
logger.warn(`Failed to clean up native audio bridge after ${logContext}`, { logger.warn(`Failed to clean up native audio bridge after ${logContext}`, {
captureId: bridge.captureId, captureId: bridge.captureId,
error, error,
@@ -204,18 +209,23 @@ function cleanupBridgeAsync(bridge: ActiveNativeAudioBridge, stopRemote: boolean
}); });
} }
function cleanupManagedBridgeById(captureId: string, stopRemote: boolean, logContext: string): boolean { function cleanupManagedBridgeById(
captureId: string,
stopRemote: boolean,
logContext: string,
endDetail?: string,
): boolean {
if (activeBridge?.captureId === captureId) { if (activeBridge?.captureId === captureId) {
const bridge = activeBridge; const bridge = activeBridge;
activeBridge = supersededBridge; activeBridge = supersededBridge;
supersededBridge = null; supersededBridge = null;
cleanupBridgeAsync(bridge, stopRemote, logContext); cleanupBridgeAsync(bridge, stopRemote, logContext, endDetail);
return true; return true;
} }
if (supersededBridge?.captureId === captureId) { if (supersededBridge?.captureId === captureId) {
const bridge = supersededBridge; const bridge = supersededBridge;
supersededBridge = null; supersededBridge = null;
cleanupBridgeAsync(bridge, stopRemote, logContext); cleanupBridgeAsync(bridge, stopRemote, logContext, endDetail);
return true; return true;
} }
return false; return false;
@@ -242,12 +252,13 @@ function attachNativeAudioCleanup(
captureId: string, captureId: string,
handle: NativeAudioBridgeHandle, handle: NativeAudioBridgeHandle,
): ActiveNativeAudioBridge['cleanup'] { ): ActiveNativeAudioBridge['cleanup'] {
const tracks = [...stream.getVideoTracks(), handle.track]; const videoTracks = stream.getVideoTracks();
const tracks = [...videoTracks, handle.track];
const cleanupListeners: Array<() => void> = []; const cleanupListeners: Array<() => void> = [];
const restoreStops: Array<() => void> = []; const restoreStops: Array<() => void> = [];
let cleanedUp = false; let cleanedUp = false;
bindNativeAudioCaptureLifecycle(captureId); bindNativeAudioCaptureLifecycle(captureId);
const cleanup = async (stopRemote: boolean = true): Promise<void> => { const cleanup = async (stopRemote: boolean = true, endDetail?: string): Promise<void> => {
if (cleanedUp) return; if (cleanedUp) return;
cleanedUp = true; cleanedUp = true;
for (const removeListener of cleanupListeners.splice(0)) { for (const removeListener of cleanupListeners.splice(0)) {
@@ -259,20 +270,22 @@ function attachNativeAudioCleanup(
if (activeBridge?.captureId === captureId) { if (activeBridge?.captureId === captureId) {
activeBridge = null; activeBridge = null;
} }
unbindNativeAudioCaptureLifecycle(captureId, false); unbindNativeAudioCaptureLifecycle(captureId, null);
await handle.cleanup(stopRemote); await handle.cleanup(stopRemote, endDetail);
}; };
const requestCleanup = (): void => { const requestCleanup = (fault: string | null, endDetail: string, logContext: string): void => {
unbindNativeAudioCaptureLifecycle(captureId, true); unbindNativeAudioCaptureLifecycle(captureId, fault);
if (!cleanupManagedBridgeById(captureId, true, 'track ended')) { if (!cleanupManagedBridgeById(captureId, true, logContext, endDetail)) {
cleanupBridgeAsync({captureId, cleanup}, true, 'track ended'); cleanupBridgeAsync({captureId, cleanup}, true, logContext, endDetail);
} }
}; };
for (const track of tracks) { for (const track of tracks) {
const onEnded = (): void => requestCleanup(); const trackKind = track === handle.track ? 'audio' : 'video';
const onEnded = (): void =>
requestCleanup(`native-audio-tap-${trackKind}-track-ended`, `${trackKind}-track-ended`, 'track ended');
track.addEventListener('ended', onEnded); track.addEventListener('ended', onEnded);
cleanupListeners.push(() => track.removeEventListener('ended', onEnded)); cleanupListeners.push(() => track.removeEventListener('ended', onEnded));
restoreStops.push(patchTrackStopForCleanup(track, requestCleanup)); restoreStops.push(patchTrackStopForCleanup(track, () => requestCleanup(null, 'caller-stopped', 'caller stop')));
} }
return cleanup; return cleanup;
} }
@@ -303,11 +316,11 @@ async function createNativeAudioBridgeWithSelfWindowAudio(captureId: string): Pr
throw error; throw error;
} }
let cleanedUp = false; let cleanedUp = false;
const cleanup = async (stopRemote: boolean = true): Promise<void> => { const cleanup = async (stopRemote: boolean = true, endDetail?: string): Promise<void> => {
if (cleanedUp) return; if (cleanedUp) return;
cleanedUp = true; cleanedUp = true;
await mixedTrack.cleanup(); await mixedTrack.cleanup();
await nativeHandle.cleanup(stopRemote); await nativeHandle.cleanup(stopRemote, endDetail);
}; };
return { return {
track: mixedTrack.track, track: mixedTrack.track,
@@ -421,7 +434,7 @@ function createDirectNativeAudioFramePump(
cleanedUp = true; cleanedUp = true;
unsubscribeFrame(); unsubscribeFrame();
unsubscribeEnd(); unsubscribeEnd();
unbindNativeAudioCaptureLifecycle(captureId, true); unbindNativeAudioCaptureLifecycle(captureId, 'native-audio-tap-track-ended');
onEnd?.(message); onEnd?.(message);
}); });
const cleanup = async (stopRemote: boolean = true): Promise<void> => { const cleanup = async (stopRemote: boolean = true): Promise<void> => {
@@ -429,7 +442,7 @@ function createDirectNativeAudioFramePump(
cleanedUp = true; cleanedUp = true;
unsubscribeFrame(); unsubscribeFrame();
unsubscribeEnd(); unsubscribeEnd();
unbindNativeAudioCaptureLifecycle(captureId, false); unbindNativeAudioCaptureLifecycle(captureId, null);
if (stopRemote) { if (stopRemote) {
await nativeAudioApi.stop(captureId).catch((error) => { await nativeAudioApi.stop(captureId).catch((error) => {
logger.warn('Failed to stop native screen-share audio frame pump', {captureId, error}); logger.warn('Failed to stop native screen-share audio frame pump', {captureId, error});
@@ -0,0 +1,98 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import ActiveScreenShareSource from '@app/features/voice/state/ActiveScreenShareSource';
import {
getRecentScreenShares,
recordScreenShareEncoderVerification,
recordScreenShareEndedModal,
recordScreenShareRequestedCodec,
recordScreenShareStartError,
recordScreenShareStarted,
recordScreenShareStopped,
resetRecentScreenSharesForTests,
} from '@app/features/voice/utils/ScreenShareLifecycleLog';
import {beforeEach, describe, expect, it} from 'vitest';
describe('ScreenShareLifecycleLog', () => {
beforeEach(() => {
resetRecentScreenSharesForTests();
ActiveScreenShareSource.clear();
});
it('keeps the first trigger that stopped a share instead of the later unpublish', () => {
recordScreenShareStarted();
recordScreenShareStopped('media-track-ended');
recordScreenShareStopped('server-unpublish');
const [entry] = getRecentScreenShares();
expect(entry.stopTrigger).toBe('media-track-ended');
expect(entry.stoppedAt).not.toBeNull();
});
it('tells a user stop apart from a share that stopped on its own', () => {
recordScreenShareStarted();
recordScreenShareStopped('user');
recordScreenShareStarted();
recordScreenShareStopped('gateway-echo');
expect(getRecentScreenShares().map((entry) => entry.stopTrigger)).toEqual(['user', 'gateway-echo']);
});
it('records the codec verification and the modal a stopped share showed', () => {
recordScreenShareStarted();
recordScreenShareRequestedCodec('h264');
recordScreenShareEncoderVerification('correct-negotiated', ['vp9']);
recordScreenShareEndedModal('codec-policy-failed');
recordScreenShareStopped('codec-republish-failed');
const [entry] = getRecentScreenShares();
expect(entry.requestedCodec).toBe('h264');
expect(entry.negotiatedCodecs).toEqual(['vp9']);
expect(entry.encoderVerification).toBe('correct-negotiated');
expect(entry.modalShown).toBe('codec-policy-failed');
expect(entry.stopTrigger).toBe('codec-republish-failed');
});
it('freezes the published source when the share stops', () => {
ActiveScreenShareSource.setPublishedSource('app', 'window:7');
recordScreenShareStarted();
recordScreenShareStopped('user');
ActiveScreenShareSource.clear();
expect(getRecentScreenShares()[0].sourceKind).toBe('app');
});
it('reports the live published source while a share is still running', () => {
recordScreenShareStarted();
ActiveScreenShareSource.setPublishedSource('display', 'screen:1');
expect(getRecentScreenShares()[0].sourceKind).toBe('display');
});
it('records a start failure without marking it as a stop', () => {
recordScreenShareStarted();
recordScreenShareStartError(new Error('portal refused'));
const [entry] = getRecentScreenShares();
expect(entry.startError).toBe('Error: portal refused');
expect(entry.stopTrigger).toBeNull();
expect(entry.stoppedAt).toBeNull();
});
it('leaves a closed share untouched when a later trigger arrives', () => {
recordScreenShareStarted();
recordScreenShareStartError(new Error('portal refused'));
recordScreenShareStopped('user');
expect(getRecentScreenShares()[0].stopTrigger).toBeNull();
});
it('keeps only the eight most recent shares', () => {
for (let index = 0; index < 11; index += 1) {
recordScreenShareStarted();
recordScreenShareStopped('user');
}
expect(getRecentScreenShares()).toHaveLength(8);
});
});
@@ -0,0 +1,109 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import ActiveScreenShareSource, {
type PublishedScreenShareSource,
} from '@app/features/voice/state/ActiveScreenShareSource';
import type {VideoCodec} from 'livekit-client';
const MAX_RETAINED_SCREEN_SHARES = 8;
export type ScreenShareStopTrigger =
| 'user'
| 'media-track-ended'
| 'server-unpublish'
| 'gateway-echo'
| 'codec-republish-failed';
export type ScreenShareEndedModal = 'source-stopped' | 'encoder-failed' | 'codec-policy-failed';
export type ScreenShareEncoderVerification =
| 'recover-stalled'
| 'ignore-repeated-stall'
| 'accept-negotiated'
| 'correct-negotiated';
export interface ScreenShareLifecycleEntry {
startedAt: number;
sourceKind: PublishedScreenShareSource | null;
requestedCodec: VideoCodec | null;
negotiatedCodecs: ReadonlyArray<VideoCodec> | null;
encoderVerification: ScreenShareEncoderVerification | null;
startError: string | null;
stoppedAt: number | null;
stopTrigger: ScreenShareStopTrigger | null;
modalShown: ScreenShareEndedModal | null;
}
let entries: Array<ScreenShareLifecycleEntry> = [];
function getOpenEntry(): ScreenShareLifecycleEntry | null {
const entry = entries.at(-1);
if (!entry || entry.stoppedAt != null || entry.startError != null) return null;
return entry;
}
export function recordScreenShareStarted(): void {
entries.push({
startedAt: Date.now(),
sourceKind: null,
requestedCodec: null,
negotiatedCodecs: null,
encoderVerification: null,
startError: null,
stoppedAt: null,
stopTrigger: null,
modalShown: null,
});
if (entries.length > MAX_RETAINED_SCREEN_SHARES) {
entries = entries.slice(-MAX_RETAINED_SCREEN_SHARES);
}
}
export function recordScreenShareStartError(error: unknown): void {
const entry = getOpenEntry();
if (!entry) return;
entry.sourceKind = ActiveScreenShareSource.getPublishedSource();
entry.startError = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
}
export function recordScreenShareRequestedCodec(codec: VideoCodec): void {
const entry = getOpenEntry();
if (!entry) return;
entry.requestedCodec = codec;
}
export function recordScreenShareEncoderVerification(
verification: ScreenShareEncoderVerification,
negotiatedCodecs: ReadonlyArray<VideoCodec> | null,
): void {
const entry = getOpenEntry();
if (!entry) return;
entry.encoderVerification = verification;
entry.negotiatedCodecs = negotiatedCodecs;
}
export function recordScreenShareEndedModal(modal: ScreenShareEndedModal): void {
const entry = getOpenEntry();
if (!entry) return;
entry.modalShown = modal;
}
export function recordScreenShareStopped(trigger: ScreenShareStopTrigger): void {
const entry = getOpenEntry();
if (!entry) return;
entry.sourceKind = ActiveScreenShareSource.getPublishedSource();
entry.stoppedAt = Date.now();
entry.stopTrigger = trigger;
}
export function getRecentScreenShares(): Array<ScreenShareLifecycleEntry> {
const open = getOpenEntry();
return entries.map((entry) => ({
...entry,
sourceKind: entry === open ? ActiveScreenShareSource.getPublishedSource() : entry.sourceKind,
}));
}
export function resetRecentScreenSharesForTests(): void {
entries = [];
}
@@ -31,6 +31,7 @@ import {
getNativeAudioCaptureDiagnosticState, getNativeAudioCaptureDiagnosticState,
} from '@app/features/voice/utils/NativeAudioCaptureBridge'; } from '@app/features/voice/utils/NativeAudioCaptureBridge';
import {getDisplayShareEnvironment} from '@app/features/voice/utils/ScreenShareEnvironment'; import {getDisplayShareEnvironment} from '@app/features/voice/utils/ScreenShareEnvironment';
import {getRecentScreenShares} from '@app/features/voice/utils/ScreenShareLifecycleLog';
import {getScreenShareBitrateBps, resolveStreamingModeSettings} from '@app/features/voice/utils/ScreenShareOptions'; import {getScreenShareBitrateBps, resolveStreamingModeSettings} from '@app/features/voice/utils/ScreenShareOptions';
import {hasHigherVideoQuality} from '@app/features/voice/utils/VideoQualityEntitlement'; import {hasHigherVideoQuality} from '@app/features/voice/utils/VideoQualityEntitlement';
import { import {
@@ -584,6 +585,7 @@ export async function buildStatsForNerdsCopyPayload(data: StatsForNerdsData): Pr
mediaDevices, mediaDevices,
voiceSettings, voiceSettings,
voiceSession: summarizeRoom(), voiceSession: summarizeRoom(),
recentScreenShares: getRecentScreenShares(),
desktop, desktop,
}; };
} }
@@ -168,10 +168,10 @@ export async function createGeneratorBridge(
pumpPendingFrames(); pumpPendingFrames();
} }
}; };
const cleanup = async (stopRemote: boolean = true): Promise<void> => { const cleanup = async (stopRemote: boolean = true, endDetail?: string): Promise<void> => {
if (cleanedUp) return; if (cleanedUp) return;
cleanedUp = true; cleanedUp = true;
endBridgeStats(captureId, 'cleanup', stopRemote ? 'caller-stopped' : 'remote-ended'); endBridgeStats(captureId, 'cleanup', endDetail ?? (stopRemote ? 'caller-stopped' : 'remote-ended'));
unsubscribeFrame(); unsubscribeFrame();
unsubscribeEnd(); unsubscribeEnd();
clearPrebufferTimer(); clearPrebufferTimer();
@@ -128,10 +128,10 @@ export async function createScriptProcessorBridge(
right.fill(0); right.fill(0);
pendingChunks.drainInto(left, right); pendingChunks.drainInto(left, right);
}; };
const cleanup = async (stopRemote: boolean = true): Promise<void> => { const cleanup = async (stopRemote: boolean = true, endDetail?: string): Promise<void> => {
if (cleanedUp) return; if (cleanedUp) return;
cleanedUp = true; cleanedUp = true;
endBridgeStats(captureId, 'cleanup', stopRemote ? 'caller-stopped' : 'remote-ended'); endBridgeStats(captureId, 'cleanup', endDetail ?? (stopRemote ? 'caller-stopped' : 'remote-ended'));
unsubscribeFrame(); unsubscribeFrame();
unsubscribeEnd(); unsubscribeEnd();
processor.onaudioprocess = null; processor.onaudioprocess = null;
@@ -32,14 +32,16 @@ export type ArmedNativeAudioCapture =
kind: 'self-window-web-audio'; kind: 'self-window-web-audio';
}; };
export type NativeAudioBridgeCleanup = (stopRemote?: boolean, endDetail?: string) => Promise<void>;
export interface NativeAudioBridgeHandle { export interface NativeAudioBridgeHandle {
track: MediaStreamTrack; track: MediaStreamTrack;
cleanup: (stopRemote?: boolean) => Promise<void>; cleanup: NativeAudioBridgeCleanup;
} }
export interface ActiveNativeAudioBridge { export interface ActiveNativeAudioBridge {
captureId: string; captureId: string;
cleanup: (stopRemote?: boolean) => Promise<void>; cleanup: NativeAudioBridgeCleanup;
linuxRule?: NativeAudioRoutingRule; linuxRule?: NativeAudioRoutingRule;
includeSelfWindowAudio?: boolean; includeSelfWindowAudio?: boolean;
} }