From 3a862f14842dc4b9897becf77858ca5ad237f578 Mon Sep 17 00:00:00 2001 From: Hampus Date: Fri, 11 Sep 2026 12:59:51 +0200 Subject: [PATCH] fix(voice): record why a screen share stopped (#2664) --- ...eV2AppScreenShareControllerRouting.test.ts | 81 +++++++ ...EngineV2AppScreenShareControllerRouting.ts | 2 + ...EngineV2AppScreenShareExecutionAdapter.tsx | 43 +++- .../utils/NativeAudioCaptureBridge.test.ts | 220 +++++++++++++++++- .../voice/utils/NativeAudioCaptureBridge.ts | 61 +++-- .../utils/ScreenShareLifecycleLog.test.ts | 98 ++++++++ .../voice/utils/ScreenShareLifecycleLog.ts | 109 +++++++++ .../features/voice/utils/StatsForNerdsCopy.ts | 2 + .../createGeneratorBridge.ts | 4 +- .../createScriptProcessorBridge.ts | 4 +- .../native_audio_capture_bridge/shared.ts | 6 +- 11 files changed, 594 insertions(+), 36 deletions(-) create mode 100644 fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareControllerRouting.test.ts create mode 100644 fluxer_app/src/features/voice/utils/ScreenShareLifecycleLog.test.ts create mode 100644 fluxer_app/src/features/voice/utils/ScreenShareLifecycleLog.ts diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareControllerRouting.test.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareControllerRouting.test.ts new file mode 100644 index 000000000..bf73d88c2 --- /dev/null +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareControllerRouting.test.ts @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import { + type VoiceEngineV2AppScreenShareControllerGateway, + VoiceEngineV2AppScreenShareControllerRouting, +} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareControllerRouting'; +import type {VoiceEngineV2AppScreenShareExecutionAdapter} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter'; +import { + getRecentScreenShares, + recordScreenShareStarted, + resetRecentScreenSharesForTests, +} from '@app/features/voice/utils/ScreenShareLifecycleLog'; +import {beforeEach, describe, expect, it, vi} from 'vitest'; + +vi.mock('@app/features/voice/engine/voice_screen_share_manager/shared', () => ({ + logger: {debug: () => undefined, info: () => undefined, warn: () => undefined, error: () => undefined}, +})); + +vi.mock('@app/features/voice/state/LocalVoiceState', () => ({ + default: {getSelfStream: () => true}, +})); + +const setEnabled = vi.fn(async () => undefined); + +function createAdapter(): VoiceEngineV2AppScreenShareExecutionAdapter { + return {liveKitFlows: {setEnabled}} as unknown as VoiceEngineV2AppScreenShareExecutionAdapter; +} + +function createGateway( + overrides: Partial = {}, +): VoiceEngineV2AppScreenShareControllerGateway { + return { + isScreenCommandRoutable: () => true, + hasScreenPublication: () => true, + hasScreenDesired: () => false, + clearScreenDesired: () => undefined, + executingScreenOperationId: () => null, + isScreenOperationPending: () => true, + publishScreen: async () => undefined, + unpublishScreen: async () => undefined, + ...overrides, + }; +} + +describe('routed screen share stop attribution', () => { + beforeEach(() => { + resetRecentScreenSharesForTests(); + setEnabled.mockClear(); + }); + + it('keeps a stop the user asked for as a user stop when the controller carries it out', async () => { + const routing = new VoiceEngineV2AppScreenShareControllerRouting(createAdapter()); + let executingOperationId: number | null = null; + routing.setGateway( + createGateway({ + executingScreenOperationId: () => executingOperationId, + unpublishScreen: async (onPlanned) => { + onPlanned([7]); + executingOperationId = 7; + await routing.unpublishViaLiveKitFlows(null); + }, + }), + ); + recordScreenShareStarted(); + + await routing.setEnabled(null, false, {sendUpdate: true, playSound: true}); + + expect(setEnabled).toHaveBeenCalledTimes(1); + expect(getRecentScreenShares()[0].stopTrigger).toBe('user'); + }); + + it('records an unpublish the app never asked for as a gateway echo', async () => { + const routing = new VoiceEngineV2AppScreenShareControllerRouting(createAdapter()); + routing.setGateway(createGateway()); + recordScreenShareStarted(); + + await routing.unpublishViaLiveKitFlows(null); + + expect(getRecentScreenShares()[0].stopTrigger).toBe('gateway-echo'); + }); +}); diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareControllerRouting.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareControllerRouting.ts index 96467b9a8..92ad20e4b 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareControllerRouting.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareControllerRouting.ts @@ -4,6 +4,7 @@ import assert from 'node:assert/strict'; import type {VoiceEngineV2AppScreenShareExecutionAdapter} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter'; import {logger} from '@app/features/voice/engine/voice_screen_share_manager/shared'; import LocalVoiceState from '@app/features/voice/state/LocalVoiceState'; +import {recordScreenShareStopped} from '@app/features/voice/utils/ScreenShareLifecycleLog'; import type {VoiceEngineV2ScreenOptions} from '@fluxer/voice_engine_v2'; import type {Room, ScreenShareCaptureOptions, TrackPublishOptions, VideoCodec} from 'livekit-client'; @@ -445,6 +446,7 @@ export class VoiceEngineV2AppScreenShareControllerRouting { async unpublishViaLiveKitFlows(roomFromPort: Room | null): Promise { const request = this.takeStopRequest(this.executingScreenOperationId()); + recordScreenShareStopped(request !== null ? 'user' : 'gateway-echo'); const room = request !== null ? request.room : roomFromPort; try { await this.adapter.liveKitFlows.setEnabled(room, false, { diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx index 5feb0d577..e1667e7c0 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx @@ -78,6 +78,15 @@ import { disarmNativeAudio, reconfigureLinuxNativeAudioRouting, } from '@app/features/voice/utils/NativeAudioCaptureBridge'; +import { + recordScreenShareEncoderVerification, + recordScreenShareEndedModal, + recordScreenShareRequestedCodec, + recordScreenShareStartError, + recordScreenShareStarted, + recordScreenShareStopped, + type ScreenShareEndedModal, +} from '@app/features/voice/utils/ScreenShareLifecycleLog'; import {SCREEN_SHARE_DEGRADATION_PREFERENCE} from '@app/features/voice/utils/ScreenShareOptions'; import {ScreenShareRollbackIncompleteError} from '@app/features/voice/utils/ScreenShareRollbackIncompleteError'; import {handleScreenShareError} from '@app/features/voice/utils/ScreenShareUtils'; @@ -400,7 +409,8 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { this.transitionScreenShareLifecycleInternal({type: 'share.encoderVerification.cleared'}); } - showScreenShareEndedModalInternal(description: string): void { + showScreenShareEndedModalInternal(description: string, modal: ScreenShareEndedModal): void { + recordScreenShareEndedModal(modal); ModalCommands.pushWithKey( ModalCommands.modal(() => ( { @@ -516,6 +528,7 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { ): void { this.cancelEncoderVerificationInternal(); if (!codec) return; + recordScreenShareRequestedCodec(codec); const publication = preferredTrack ? undefined : participant.getTrackPublication(Track.Source.ScreenShare); const track = preferredTrack ?? (publication?.videoTrack as LocalVideoTrack | undefined); const sender = track?.sender; @@ -530,8 +543,10 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { const action = resolveScreenShareEncoderVerificationAction(failure); switch (action.kind) { case 'ignore-repeated-stall': + recordScreenShareEncoderVerification('ignore-repeated-stall', null); return; case 'recover-stalled': + recordScreenShareEncoderVerification('recover-stalled', null); logger.warn('Screen share encoder verification failed', { codec: action.codec, failureReason: 'screen-share-encode-stalled', @@ -546,12 +561,14 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { ); return; case 'accept-negotiated': + recordScreenShareEncoderVerification('accept-negotiated', action.negotiated); logger.info('Screen share publisher negotiated a different codec inside the publish policy', { requested: action.requested, negotiated: action.negotiated, }); return; case 'correct-negotiated': + recordScreenShareEncoderVerification('correct-negotiated', action.negotiated); logger.warn('Screen share is sending a codec outside the publish policy', { requested: action.requested, negotiated: action.negotiated, @@ -599,7 +616,8 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { trigger, readyState: mediaStreamTrack.readyState, }); - this.showScreenShareEndedModalInternal(i18n._(SCREEN_SHARE_SOURCE_STOPPED_DESCRIPTOR)); + this.showScreenShareEndedModalInternal(i18n._(SCREEN_SHARE_SOURCE_STOPPED_DESCRIPTOR), 'source-stopped'); + recordScreenShareStopped('media-track-ended'); this.transitionScreenShareLifecycleInternal({type: 'share.endedStop.start'}); const stopPromise = this.setScreenShareEnabled(room, false, {sendUpdate: true, playSound: true}) .catch((error) => { @@ -663,6 +681,7 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { } handleLocalScreenShareTrackUnpublished(room: Room, playSound: boolean, publication?: LocalTrackPublication): void { + recordScreenShareStopped(this.isScreenSharePending ? 'user' : 'server-unpublish'); this.clearScreenShareKeepAliveSinkInternal(); this.cleanupActiveScreenShareEndListenerInternal(); const participant = room.localParticipant; @@ -866,7 +885,17 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { publishOptions?: TrackPublishOptions, ): Promise { assert.equal(typeof enabled, 'boolean'); - await this.controllerRouting.setEnabled(room, enabled, options, publishOptions); + if (enabled) { + recordScreenShareStarted(); + } + try { + await this.controllerRouting.setEnabled(room, enabled, options, publishOptions); + } catch (error) { + if (enabled) { + recordScreenShareStartError(error); + } + throw error; + } if (!enabled && options?.preserveStreamAudioPreferences !== true) { VoiceSettingsCommands.update({ shareAppAudio: true, @@ -908,7 +937,13 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { options?: DeviceScreenShareCaptureOptions, publishOptions?: TrackPublishOptions, ): Promise { - await this.liveKitFlows.startDeviceScreenShare(room, options, publishOptions); + recordScreenShareStarted(); + try { + await this.liveKitFlows.startDeviceScreenShare(room, options, publishOptions); + } catch (error) { + recordScreenShareStartError(error); + throw error; + } } async replaceActiveDisplayScreenShare( diff --git a/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.test.ts b/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.test.ts index 1e0db0bf5..e53da140e 100644 --- a/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.test.ts +++ b/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.test.ts @@ -1,12 +1,138 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import {getNativeAudioCaptureDiagnosticState} from '@app/features/voice/utils/NativeAudioCaptureBridge'; +import type {VoiceEngineV2AppSourceLifecycleBridge} from '@app/features/voice/engine/v2/VoiceEngineV2AppSourceLifecycleBridge'; +import { + armNativeSystemAudioForNextCapture, + captureNativeAudioTrackForLinuxRouting, + getNativeAudioCaptureDiagnosticState, + resetNativeAudioCaptureBridgeForTests, + setNativeAudioCaptureBridgeLifecycleBridge, +} from '@app/features/voice/utils/NativeAudioCaptureBridge'; import {recordBridgeFrame, startBridgeStats} from '@app/features/voice/utils/native_audio_capture_bridge/bridgeStats'; import type { NativeAudioBridgeEndedCapture, NativeAudioBridgeStats, } from '@app/features/voice/utils/native_audio_capture_bridge/shared'; -import {describe, expect, it} from 'vitest'; +import {beforeEach, describe, expect, it, vi} from 'vitest'; + +const electronApi = {current: null as Record | null}; + +vi.mock('@app/features/ui/utils/NativeUtils', () => ({ + getElectronAPI: () => electronApi.current, +})); + +vi.mock('@app/features/voice/utils/SelfWindowScreenShareAudioMix', () => ({ + mixTrackWithSelfWindowScreenShareAudio: async (primaryTrack: MediaStreamTrack) => ({ + track: primaryTrack, + cleanup: async () => undefined, + }), +})); + +const CAPTURE_ID = 'native-audio:stop-cause'; + +class FakeMediaStreamTrack extends EventTarget { + readyState: 'live' | 'ended' = 'live'; + + constructor(readonly kind: 'audio' | 'video') { + super(); + } + + stop(): void { + this.readyState = 'ended'; + } +} + +class FakeGeneratorTrack extends FakeMediaStreamTrack { + readonly writable = new WritableStream(); + + constructor(_options: {kind: 'audio'}) { + super('audio'); + } +} + +class FakeMediaStream { + private readonly tracks: Array; + + constructor(tracks: Array = []) { + this.tracks = [...tracks]; + } + + getTracks(): Array { + return [...this.tracks]; + } + + getVideoTracks(): Array { + return this.tracks.filter((track) => track.kind === 'video'); + } + + getAudioTracks(): Array { + return this.tracks.filter((track) => track.kind === 'audio'); + } + + addTrack(track: FakeMediaStreamTrack): void { + this.tracks.push(track); + } + + removeTrack(track: FakeMediaStreamTrack): void { + const index = this.tracks.indexOf(track); + if (index >= 0) this.tracks.splice(index, 1); + } +} + +const stopNativeCapture = vi.fn(async () => undefined); +const reportLifecycle = vi.fn(() => true); + +function createNativeAudioApi() { + return { + getAvailability: async () => ({ + available: true, + backend: 'windows-wasapi-loopback', + capabilities: {process: true, system: true, systemExcludesSelf: true}, + }), + start: async () => ({captureId: CAPTURE_ID, sampleRate: 48000, channels: 2}), + stop: stopNativeCapture, + onFrame: () => () => undefined, + onEnd: () => () => undefined, + }; +} + +function installLifecycleBridge(): void { + setNativeAudioCaptureBridgeLifecycleBridge({ + bind: () => true, + unbind: () => undefined, + reportLifecycle, + } as unknown as VoiceEngineV2AppSourceLifecycleBridge); +} + +async function flushCleanup(): Promise { + await new Promise((resolve) => setTimeout(resolve, 0)); + await new Promise((resolve) => setTimeout(resolve, 0)); +} + +async function startDisplayShareWithNativeAudio(): Promise<{ + videoTrack: FakeMediaStreamTrack; + audioTrack: FakeMediaStreamTrack; +}> { + const videoTrack = new FakeMediaStreamTrack('video'); + const capturedStream = new FakeMediaStream([videoTrack]); + Object.defineProperty(globalThis, 'navigator', { + configurable: true, + value: {mediaDevices: {getDisplayMedia: async () => capturedStream}}, + }); + expect(await armNativeSystemAudioForNextCapture()).toBe(true); + const stream = (await navigator.mediaDevices.getDisplayMedia({video: true})) as unknown as FakeMediaStream; + const audioTrack = stream.getAudioTracks()[0]; + expect(audioTrack).toBeInstanceOf(FakeGeneratorTrack); + return {videoTrack, audioTrack}; +} + +function currentBridgeStats(): NativeAudioBridgeStats { + return getNativeAudioCaptureDiagnosticState().bridgeStats as NativeAudioBridgeStats; +} + +function currentLifecycleFaults(): Array<{message: string}> { + return getNativeAudioCaptureDiagnosticState().lifecycleFaults as Array<{message: string}>; +} describe('getNativeAudioCaptureDiagnosticState', () => { it('keeps the six fields real dumps already carry and appends the retained histories', () => { @@ -41,3 +167,93 @@ describe('getNativeAudioCaptureDiagnosticState', () => { expect(Array.isArray(getNativeAudioCaptureDiagnosticState().lifecycleFaults)).toBe(true); }); }); + +describe('native audio capture teardown cause', () => { + beforeEach(() => { + resetNativeAudioCaptureBridgeForTests(); + Object.defineProperty(globalThis, 'window', { + configurable: true, + value: {MediaStreamTrackGenerator: FakeGeneratorTrack, AudioData: class {}}, + }); + Object.defineProperty(globalThis, 'MediaStream', {configurable: true, value: FakeMediaStream}); + electronApi.current = {platform: 'win32', nativeAudio: createNativeAudioApi()}; + stopNativeCapture.mockClear(); + reportLifecycle.mockClear(); + installLifecycleBridge(); + }); + + it('records caller-stopped with no fault when the share is stopped by hand', async () => { + const {audioTrack} = await startDisplayShareWithNativeAudio(); + + audioTrack.stop(); + await flushCleanup(); + + expect(currentLifecycleFaults()).toEqual([]); + expect(reportLifecycle).not.toHaveBeenCalled(); + expect(currentBridgeStats().endDetail).toBe('caller-stopped'); + expect(stopNativeCapture).toHaveBeenCalledTimes(1); + }); + + it('names the audio track when the capture track ends on its own', async () => { + const {audioTrack} = await startDisplayShareWithNativeAudio(); + + audioTrack.dispatchEvent(new Event('ended')); + await flushCleanup(); + + expect(currentLifecycleFaults().map((fault) => fault.message)).toEqual(['native-audio-tap-audio-track-ended']); + expect(reportLifecycle).toHaveBeenCalledWith({ + captureId: CAPTURE_ID, + kind: 'error', + message: 'native-audio-tap-audio-track-ended', + }); + expect(currentBridgeStats().endDetail).toBe('audio-track-ended'); + }); + + it('names the video track when the shared screen stops sending frames', async () => { + const {videoTrack} = await startDisplayShareWithNativeAudio(); + + videoTrack.dispatchEvent(new Event('ended')); + await flushCleanup(); + + expect(currentLifecycleFaults().map((fault) => fault.message)).toEqual(['native-audio-tap-video-track-ended']); + expect(currentBridgeStats().endDetail).toBe('video-track-ended'); + }); + + it('tears down once and keeps the track-ended cause when a stop follows it', async () => { + const {audioTrack} = await startDisplayShareWithNativeAudio(); + + audioTrack.dispatchEvent(new Event('ended')); + audioTrack.stop(); + await flushCleanup(); + + expect(currentLifecycleFaults()).toHaveLength(1); + expect(currentBridgeStats().endDetail).toBe('audio-track-ended'); + expect(stopNativeCapture).toHaveBeenCalledTimes(1); + }); + + it('names the audio track when a capture mixed with self-window audio ends on its own', async () => { + electronApi.current = {platform: 'linux', nativeAudio: createNativeAudioApi()}; + const track = (await captureNativeAudioTrackForLinuxRouting( + {include: []}, + {includeSelfWindowAudio: true}, + )) as unknown as FakeMediaStreamTrack | null; + expect(track).toBeInstanceOf(FakeGeneratorTrack); + + track?.dispatchEvent(new Event('ended')); + await flushCleanup(); + + expect(currentBridgeStats().endDetail).toBe('audio-track-ended'); + }); + + it('tears down once and keeps caller-stopped when the track ends after the stop', async () => { + const {audioTrack} = await startDisplayShareWithNativeAudio(); + + audioTrack.stop(); + audioTrack.dispatchEvent(new Event('ended')); + await flushCleanup(); + + expect(currentLifecycleFaults()).toEqual([]); + expect(currentBridgeStats().endDetail).toBe('caller-stopped'); + expect(stopNativeCapture).toHaveBeenCalledTimes(1); + }); +}); diff --git a/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.ts b/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.ts index 836807bb5..a9e939e5e 100644 --- a/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.ts +++ b/fluxer_app/src/features/voice/utils/NativeAudioCaptureBridge.ts @@ -113,15 +113,15 @@ function recordLifecycleFault(captureId: string, message: string): void { } } -function unbindNativeAudioCaptureLifecycle(captureId: string, faulted: boolean): void { +function unbindNativeAudioCaptureLifecycle(captureId: string, fault: string | null): void { if (!lifecycleBoundCaptureIds.has(captureId)) return; const bridge = sourceLifecycleBridge; - if (faulted) { - recordLifecycleFault(captureId, 'native-audio-tap-track-ended'); + if (fault) { + recordLifecycleFault(captureId, fault); } if (bridge) { - if (faulted) { - bridge.reportLifecycle({captureId, kind: 'error', message: 'native-audio-tap-track-ended'}); + if (fault) { + bridge.reportLifecycle({captureId, kind: 'error', message: fault}); } bridge.unbind(captureId); } @@ -195,8 +195,13 @@ export function getNativeAudioCaptureDiagnosticState(): Record }; } -function cleanupBridgeAsync(bridge: ActiveNativeAudioBridge, stopRemote: boolean, logContext: string): void { - void bridge.cleanup(stopRemote).catch((error) => { +function cleanupBridgeAsync( + bridge: ActiveNativeAudioBridge, + stopRemote: boolean, + logContext: string, + endDetail?: string, +): void { + void bridge.cleanup(stopRemote, endDetail).catch((error) => { logger.warn(`Failed to clean up native audio bridge after ${logContext}`, { captureId: bridge.captureId, error, @@ -204,18 +209,23 @@ function cleanupBridgeAsync(bridge: ActiveNativeAudioBridge, stopRemote: boolean }); } -function cleanupManagedBridgeById(captureId: string, stopRemote: boolean, logContext: string): boolean { +function cleanupManagedBridgeById( + captureId: string, + stopRemote: boolean, + logContext: string, + endDetail?: string, +): boolean { if (activeBridge?.captureId === captureId) { const bridge = activeBridge; activeBridge = supersededBridge; supersededBridge = null; - cleanupBridgeAsync(bridge, stopRemote, logContext); + cleanupBridgeAsync(bridge, stopRemote, logContext, endDetail); return true; } if (supersededBridge?.captureId === captureId) { const bridge = supersededBridge; supersededBridge = null; - cleanupBridgeAsync(bridge, stopRemote, logContext); + cleanupBridgeAsync(bridge, stopRemote, logContext, endDetail); return true; } return false; @@ -242,12 +252,13 @@ function attachNativeAudioCleanup( captureId: string, handle: NativeAudioBridgeHandle, ): ActiveNativeAudioBridge['cleanup'] { - const tracks = [...stream.getVideoTracks(), handle.track]; + const videoTracks = stream.getVideoTracks(); + const tracks = [...videoTracks, handle.track]; const cleanupListeners: Array<() => void> = []; const restoreStops: Array<() => void> = []; let cleanedUp = false; bindNativeAudioCaptureLifecycle(captureId); - const cleanup = async (stopRemote: boolean = true): Promise => { + const cleanup = async (stopRemote: boolean = true, endDetail?: string): Promise => { if (cleanedUp) return; cleanedUp = true; for (const removeListener of cleanupListeners.splice(0)) { @@ -259,20 +270,22 @@ function attachNativeAudioCleanup( if (activeBridge?.captureId === captureId) { activeBridge = null; } - unbindNativeAudioCaptureLifecycle(captureId, false); - await handle.cleanup(stopRemote); + unbindNativeAudioCaptureLifecycle(captureId, null); + await handle.cleanup(stopRemote, endDetail); }; - const requestCleanup = (): void => { - unbindNativeAudioCaptureLifecycle(captureId, true); - if (!cleanupManagedBridgeById(captureId, true, 'track ended')) { - cleanupBridgeAsync({captureId, cleanup}, true, 'track ended'); + const requestCleanup = (fault: string | null, endDetail: string, logContext: string): void => { + unbindNativeAudioCaptureLifecycle(captureId, fault); + if (!cleanupManagedBridgeById(captureId, true, logContext, endDetail)) { + cleanupBridgeAsync({captureId, cleanup}, true, logContext, endDetail); } }; for (const track of tracks) { - const onEnded = (): void => requestCleanup(); + const trackKind = track === handle.track ? 'audio' : 'video'; + const onEnded = (): void => + requestCleanup(`native-audio-tap-${trackKind}-track-ended`, `${trackKind}-track-ended`, 'track ended'); track.addEventListener('ended', onEnded); cleanupListeners.push(() => track.removeEventListener('ended', onEnded)); - restoreStops.push(patchTrackStopForCleanup(track, requestCleanup)); + restoreStops.push(patchTrackStopForCleanup(track, () => requestCleanup(null, 'caller-stopped', 'caller stop'))); } return cleanup; } @@ -303,11 +316,11 @@ async function createNativeAudioBridgeWithSelfWindowAudio(captureId: string): Pr throw error; } let cleanedUp = false; - const cleanup = async (stopRemote: boolean = true): Promise => { + const cleanup = async (stopRemote: boolean = true, endDetail?: string): Promise => { if (cleanedUp) return; cleanedUp = true; await mixedTrack.cleanup(); - await nativeHandle.cleanup(stopRemote); + await nativeHandle.cleanup(stopRemote, endDetail); }; return { track: mixedTrack.track, @@ -421,7 +434,7 @@ function createDirectNativeAudioFramePump( cleanedUp = true; unsubscribeFrame(); unsubscribeEnd(); - unbindNativeAudioCaptureLifecycle(captureId, true); + unbindNativeAudioCaptureLifecycle(captureId, 'native-audio-tap-track-ended'); onEnd?.(message); }); const cleanup = async (stopRemote: boolean = true): Promise => { @@ -429,7 +442,7 @@ function createDirectNativeAudioFramePump( cleanedUp = true; unsubscribeFrame(); unsubscribeEnd(); - unbindNativeAudioCaptureLifecycle(captureId, false); + unbindNativeAudioCaptureLifecycle(captureId, null); if (stopRemote) { await nativeAudioApi.stop(captureId).catch((error) => { logger.warn('Failed to stop native screen-share audio frame pump', {captureId, error}); diff --git a/fluxer_app/src/features/voice/utils/ScreenShareLifecycleLog.test.ts b/fluxer_app/src/features/voice/utils/ScreenShareLifecycleLog.test.ts new file mode 100644 index 000000000..86234064d --- /dev/null +++ b/fluxer_app/src/features/voice/utils/ScreenShareLifecycleLog.test.ts @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import ActiveScreenShareSource from '@app/features/voice/state/ActiveScreenShareSource'; +import { + getRecentScreenShares, + recordScreenShareEncoderVerification, + recordScreenShareEndedModal, + recordScreenShareRequestedCodec, + recordScreenShareStartError, + recordScreenShareStarted, + recordScreenShareStopped, + resetRecentScreenSharesForTests, +} from '@app/features/voice/utils/ScreenShareLifecycleLog'; +import {beforeEach, describe, expect, it} from 'vitest'; + +describe('ScreenShareLifecycleLog', () => { + beforeEach(() => { + resetRecentScreenSharesForTests(); + ActiveScreenShareSource.clear(); + }); + + it('keeps the first trigger that stopped a share instead of the later unpublish', () => { + recordScreenShareStarted(); + recordScreenShareStopped('media-track-ended'); + recordScreenShareStopped('server-unpublish'); + + const [entry] = getRecentScreenShares(); + expect(entry.stopTrigger).toBe('media-track-ended'); + expect(entry.stoppedAt).not.toBeNull(); + }); + + it('tells a user stop apart from a share that stopped on its own', () => { + recordScreenShareStarted(); + recordScreenShareStopped('user'); + recordScreenShareStarted(); + recordScreenShareStopped('gateway-echo'); + + expect(getRecentScreenShares().map((entry) => entry.stopTrigger)).toEqual(['user', 'gateway-echo']); + }); + + it('records the codec verification and the modal a stopped share showed', () => { + recordScreenShareStarted(); + recordScreenShareRequestedCodec('h264'); + recordScreenShareEncoderVerification('correct-negotiated', ['vp9']); + recordScreenShareEndedModal('codec-policy-failed'); + recordScreenShareStopped('codec-republish-failed'); + + const [entry] = getRecentScreenShares(); + expect(entry.requestedCodec).toBe('h264'); + expect(entry.negotiatedCodecs).toEqual(['vp9']); + expect(entry.encoderVerification).toBe('correct-negotiated'); + expect(entry.modalShown).toBe('codec-policy-failed'); + expect(entry.stopTrigger).toBe('codec-republish-failed'); + }); + + it('freezes the published source when the share stops', () => { + ActiveScreenShareSource.setPublishedSource('app', 'window:7'); + recordScreenShareStarted(); + recordScreenShareStopped('user'); + ActiveScreenShareSource.clear(); + + expect(getRecentScreenShares()[0].sourceKind).toBe('app'); + }); + + it('reports the live published source while a share is still running', () => { + recordScreenShareStarted(); + ActiveScreenShareSource.setPublishedSource('display', 'screen:1'); + + expect(getRecentScreenShares()[0].sourceKind).toBe('display'); + }); + + it('records a start failure without marking it as a stop', () => { + recordScreenShareStarted(); + recordScreenShareStartError(new Error('portal refused')); + + const [entry] = getRecentScreenShares(); + expect(entry.startError).toBe('Error: portal refused'); + expect(entry.stopTrigger).toBeNull(); + expect(entry.stoppedAt).toBeNull(); + }); + + it('leaves a closed share untouched when a later trigger arrives', () => { + recordScreenShareStarted(); + recordScreenShareStartError(new Error('portal refused')); + recordScreenShareStopped('user'); + + expect(getRecentScreenShares()[0].stopTrigger).toBeNull(); + }); + + it('keeps only the eight most recent shares', () => { + for (let index = 0; index < 11; index += 1) { + recordScreenShareStarted(); + recordScreenShareStopped('user'); + } + + expect(getRecentScreenShares()).toHaveLength(8); + }); +}); diff --git a/fluxer_app/src/features/voice/utils/ScreenShareLifecycleLog.ts b/fluxer_app/src/features/voice/utils/ScreenShareLifecycleLog.ts new file mode 100644 index 000000000..e0241118b --- /dev/null +++ b/fluxer_app/src/features/voice/utils/ScreenShareLifecycleLog.ts @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import ActiveScreenShareSource, { + type PublishedScreenShareSource, +} from '@app/features/voice/state/ActiveScreenShareSource'; +import type {VideoCodec} from 'livekit-client'; + +const MAX_RETAINED_SCREEN_SHARES = 8; + +export type ScreenShareStopTrigger = + | 'user' + | 'media-track-ended' + | 'server-unpublish' + | 'gateway-echo' + | 'codec-republish-failed'; + +export type ScreenShareEndedModal = 'source-stopped' | 'encoder-failed' | 'codec-policy-failed'; + +export type ScreenShareEncoderVerification = + | 'recover-stalled' + | 'ignore-repeated-stall' + | 'accept-negotiated' + | 'correct-negotiated'; + +export interface ScreenShareLifecycleEntry { + startedAt: number; + sourceKind: PublishedScreenShareSource | null; + requestedCodec: VideoCodec | null; + negotiatedCodecs: ReadonlyArray | null; + encoderVerification: ScreenShareEncoderVerification | null; + startError: string | null; + stoppedAt: number | null; + stopTrigger: ScreenShareStopTrigger | null; + modalShown: ScreenShareEndedModal | null; +} + +let entries: Array = []; + +function getOpenEntry(): ScreenShareLifecycleEntry | null { + const entry = entries.at(-1); + if (!entry || entry.stoppedAt != null || entry.startError != null) return null; + return entry; +} + +export function recordScreenShareStarted(): void { + entries.push({ + startedAt: Date.now(), + sourceKind: null, + requestedCodec: null, + negotiatedCodecs: null, + encoderVerification: null, + startError: null, + stoppedAt: null, + stopTrigger: null, + modalShown: null, + }); + if (entries.length > MAX_RETAINED_SCREEN_SHARES) { + entries = entries.slice(-MAX_RETAINED_SCREEN_SHARES); + } +} + +export function recordScreenShareStartError(error: unknown): void { + const entry = getOpenEntry(); + if (!entry) return; + entry.sourceKind = ActiveScreenShareSource.getPublishedSource(); + entry.startError = error instanceof Error ? `${error.name}: ${error.message}` : String(error); +} + +export function recordScreenShareRequestedCodec(codec: VideoCodec): void { + const entry = getOpenEntry(); + if (!entry) return; + entry.requestedCodec = codec; +} + +export function recordScreenShareEncoderVerification( + verification: ScreenShareEncoderVerification, + negotiatedCodecs: ReadonlyArray | null, +): void { + const entry = getOpenEntry(); + if (!entry) return; + entry.encoderVerification = verification; + entry.negotiatedCodecs = negotiatedCodecs; +} + +export function recordScreenShareEndedModal(modal: ScreenShareEndedModal): void { + const entry = getOpenEntry(); + if (!entry) return; + entry.modalShown = modal; +} + +export function recordScreenShareStopped(trigger: ScreenShareStopTrigger): void { + const entry = getOpenEntry(); + if (!entry) return; + entry.sourceKind = ActiveScreenShareSource.getPublishedSource(); + entry.stoppedAt = Date.now(); + entry.stopTrigger = trigger; +} + +export function getRecentScreenShares(): Array { + const open = getOpenEntry(); + return entries.map((entry) => ({ + ...entry, + sourceKind: entry === open ? ActiveScreenShareSource.getPublishedSource() : entry.sourceKind, + })); +} + +export function resetRecentScreenSharesForTests(): void { + entries = []; +} diff --git a/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts b/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts index 943dc7aa8..7a2273c1e 100644 --- a/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts +++ b/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts @@ -31,6 +31,7 @@ import { getNativeAudioCaptureDiagnosticState, } from '@app/features/voice/utils/NativeAudioCaptureBridge'; import {getDisplayShareEnvironment} from '@app/features/voice/utils/ScreenShareEnvironment'; +import {getRecentScreenShares} from '@app/features/voice/utils/ScreenShareLifecycleLog'; import {getScreenShareBitrateBps, resolveStreamingModeSettings} from '@app/features/voice/utils/ScreenShareOptions'; import {hasHigherVideoQuality} from '@app/features/voice/utils/VideoQualityEntitlement'; import { @@ -584,6 +585,7 @@ export async function buildStatsForNerdsCopyPayload(data: StatsForNerdsData): Pr mediaDevices, voiceSettings, voiceSession: summarizeRoom(), + recentScreenShares: getRecentScreenShares(), desktop, }; } diff --git a/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/createGeneratorBridge.ts b/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/createGeneratorBridge.ts index 7acc76eb2..30f661c2c 100644 --- a/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/createGeneratorBridge.ts +++ b/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/createGeneratorBridge.ts @@ -168,10 +168,10 @@ export async function createGeneratorBridge( pumpPendingFrames(); } }; - const cleanup = async (stopRemote: boolean = true): Promise => { + const cleanup = async (stopRemote: boolean = true, endDetail?: string): Promise => { if (cleanedUp) return; cleanedUp = true; - endBridgeStats(captureId, 'cleanup', stopRemote ? 'caller-stopped' : 'remote-ended'); + endBridgeStats(captureId, 'cleanup', endDetail ?? (stopRemote ? 'caller-stopped' : 'remote-ended')); unsubscribeFrame(); unsubscribeEnd(); clearPrebufferTimer(); diff --git a/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/createScriptProcessorBridge.ts b/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/createScriptProcessorBridge.ts index 835c7976d..dc141da99 100644 --- a/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/createScriptProcessorBridge.ts +++ b/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/createScriptProcessorBridge.ts @@ -128,10 +128,10 @@ export async function createScriptProcessorBridge( right.fill(0); pendingChunks.drainInto(left, right); }; - const cleanup = async (stopRemote: boolean = true): Promise => { + const cleanup = async (stopRemote: boolean = true, endDetail?: string): Promise => { if (cleanedUp) return; cleanedUp = true; - endBridgeStats(captureId, 'cleanup', stopRemote ? 'caller-stopped' : 'remote-ended'); + endBridgeStats(captureId, 'cleanup', endDetail ?? (stopRemote ? 'caller-stopped' : 'remote-ended')); unsubscribeFrame(); unsubscribeEnd(); processor.onaudioprocess = null; diff --git a/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/shared.ts b/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/shared.ts index 9bed519a9..435e3aa76 100644 --- a/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/shared.ts +++ b/fluxer_app/src/features/voice/utils/native_audio_capture_bridge/shared.ts @@ -32,14 +32,16 @@ export type ArmedNativeAudioCapture = kind: 'self-window-web-audio'; }; +export type NativeAudioBridgeCleanup = (stopRemote?: boolean, endDetail?: string) => Promise; + export interface NativeAudioBridgeHandle { track: MediaStreamTrack; - cleanup: (stopRemote?: boolean) => Promise; + cleanup: NativeAudioBridgeCleanup; } export interface ActiveNativeAudioBridge { captureId: string; - cleanup: (stopRemote?: boolean) => Promise; + cleanup: NativeAudioBridgeCleanup; linuxRule?: NativeAudioRoutingRule; includeSelfWindowAudio?: boolean; }