diff --git a/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts b/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts index 14ddbe8b1..56d03bfd1 100644 --- a/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts +++ b/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts @@ -58,6 +58,7 @@ import { saveCurrentVoiceSessionRestoreSnapshot, type VoiceSessionRestoreSyncHandle, } from '@app/features/voice/engine/media_engine_facade/VoiceSessionRestoreSync'; +import ScreenShareCodecNegotiation from '@app/features/voice/engine/ScreenShareCodecNegotiation'; import ScreenSharePublicationMigration from '@app/features/voice/engine/ScreenSharePublicationMigration'; import {Store, useStoreVersion} from '@app/features/voice/engine/Store'; import {shouldMoveToAfkOnTick} from '@app/features/voice/engine/VoiceAfkTracking'; @@ -388,6 +389,17 @@ class MediaEngineFacade extends Store { this.voiceEngineV2SourceLifecycleBridge = this.createSourceLifecycleBridge(); voiceEngineV2AppScreenShareExecutionAdapter.setControllerGateway(this.createScreenShareControllerGateway()); voiceEngineV2AppScreenShareExecutionAdapter.setSourceLifecycleBridge(this.voiceEngineV2SourceLifecycleBridge); + ScreenShareCodecNegotiation.setSelectionChangeListener((activeRoom, codec, reason) => { + void voiceEngineV2AppScreenShareExecutionAdapter + .republishActiveScreenShareForNegotiatedCodecInternal(activeRoom, codec) + .catch((error) => { + logger.warn('Failed to republish active screen share after a codec negotiation change', { + error, + codec, + reason, + }); + }); + }); voiceEngineV2AppMediaExecutionAdapter.setSourceLifecycleBridge(this.voiceEngineV2SourceLifecycleBridge); setNativeAudioCaptureBridgeLifecycleBridge(this.voiceEngineV2SourceLifecycleBridge); this.syncVoiceEngineV2AudioControlsFromAppState(); diff --git a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts index bcc58dd53..d8fb37c40 100644 --- a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts +++ b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts @@ -74,10 +74,45 @@ Object.defineProperty(globalThis, 'RTCRtpReceiver', { value: {getCapabilities: () => VIDEO_CAPABILITIES}, }); -const {buildLocalCodecAdvertisements, computeNegotiatedVideoCodec, getScreenShareCodecPreferenceOrder} = await import( - './ScreenShareCodecNegotiation' -); +const { + default: ScreenShareCodecNegotiation, + buildLocalCodecAdvertisements, + computeNegotiatedVideoCodec, + getScreenShareCodecPreferenceOrder, + SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC, +} = await import('./ScreenShareCodecNegotiation'); const {resetCachedCodecCapabilities} = await import('@app/features/voice/utils/CodecCapabilityDetector'); +const {RoomEvent} = await import('livekit-client'); + +class FakeRoom { + private handlers = new Map) => void>>(); + localParticipant = {identity: 'local', publishData: vi.fn(async () => undefined)}; + remoteParticipants = new Map(); + + on(event: string, handler: (...args: Array) => void): this { + if (!this.handlers.has(event)) this.handlers.set(event, new Set()); + this.handlers.get(event)?.add(handler); + return this; + } + + off(event: string, handler: (...args: Array) => void): this { + this.handlers.get(event)?.delete(handler); + return this; + } + + emit(event: string, ...args: Array): void { + for (const handler of this.handlers.get(event) ?? []) handler(...args); + } +} + +function sessionUpdatePayload(codecs: Array): Uint8Array { + return new TextEncoder().encode( + JSON.stringify({ + op: 14, + d: {video_codec: 'H264', media_session_id: 'peer-session', reason: 'connected', codecs}, + }), + ); +} function videoAdvertisement(name: 'AV1' | 'VP9' | 'H265', encode: boolean, decode: boolean): FluxerCodecAdvertisement { const payloadType = name === 'AV1' ? 101 : name === 'H265' ? 105 : 109; @@ -180,3 +215,89 @@ describe('screen-share codec negotiation with the HEVC opt-in off', () => { expect(computeNegotiatedVideoCodec(local, remote, 0, getScreenShareCodecPreferenceOrder()).codec).toBe('h265'); }); }); + +describe('reacting to a mid-session codec selection change', () => { + let room: FakeRoom; + + beforeEach(() => { + av1OptIn = false; + hevcOptIn = true; + preferredScreenShareCodec = 'auto'; + gpuReport = {av1: 'hardware', h265: 'hardware', h264: 'hardware', vp9: 'software', vp8: 'software'}; + resetCachedCodecCapabilities(); + room = new FakeRoom(); + ScreenShareCodecNegotiation.setSelectionChangeListener(null); + }); + + function flush(): Promise { + return new Promise((resolve) => setTimeout(resolve, 0)); + } + + async function establishBaseline(): Promise { + ScreenShareCodecNegotiation.bind(room as never); + room.emit(RoomEvent.ParticipantDisconnected, {identity: 'nobody-was-connected'}); + await flush(); + } + + it('picks the locally preferred codec while no remote participant is known', async () => { + await establishBaseline(); + expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('h265'); + ScreenShareCodecNegotiation.dispose(); + }); + + it('notifies the selection-change listener once a peer without HEVC decode is discovered', async () => { + await establishBaseline(); + expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('h265'); + const listener = vi.fn(); + ScreenShareCodecNegotiation.setSelectionChangeListener(listener); + room.remoteParticipants.set('peer-without-hevc', {identity: 'peer-without-hevc'}); + const payload = sessionUpdatePayload([videoAdvertisement('VP9', true, true)]); + room.emit( + RoomEvent.DataReceived, + payload, + {identity: 'peer-without-hevc'}, + undefined, + SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC, + ); + await flush(); + const negotiated = ScreenShareCodecNegotiation.getSelectedCodec(); + expect(negotiated).not.toBe('h265'); + expect(listener).toHaveBeenCalledTimes(1); + expect(listener).toHaveBeenCalledWith(room, negotiated, 'data'); + ScreenShareCodecNegotiation.dispose(); + }); + + it('does not notify the listener when the recomputed selection is unchanged', async () => { + await establishBaseline(); + room.remoteParticipants.set('peer-with-hevc', {identity: 'peer-with-hevc'}); + const firstPayload = sessionUpdatePayload([ + videoAdvertisement('H265', true, true), + videoAdvertisement('VP9', true, true), + ]); + room.emit( + RoomEvent.DataReceived, + firstPayload, + {identity: 'peer-with-hevc'}, + undefined, + SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC, + ); + await flush(); + expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('h265'); + const listener = vi.fn(); + ScreenShareCodecNegotiation.setSelectionChangeListener(listener); + const secondPayload = sessionUpdatePayload([ + videoAdvertisement('H265', true, true), + videoAdvertisement('VP9', true, true), + ]); + room.emit( + RoomEvent.DataReceived, + secondPayload, + {identity: 'peer-with-hevc'}, + undefined, + SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC, + ); + await flush(); + expect(listener).not.toHaveBeenCalled(); + ScreenShareCodecNegotiation.dispose(); + }); +}); diff --git a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts index 463cdebb6..3ed6fb826 100644 --- a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts +++ b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts @@ -73,7 +73,7 @@ const VIDEO_CODEC_PROTOCOL_TABLE: Record< type FluxerVideoCodecName = 'AV1' | 'H265' | 'H264' | 'VP9' | 'VP8'; type FluxerCodecName = 'opus' | FluxerVideoCodecName; type FluxerCodecType = 'audio' | 'video'; -type NegotiationReason = +export type NegotiationReason = | 'connected' | 'data' | 'participant-connected' @@ -499,11 +499,18 @@ class ScreenShareCodecNegotiation { private mediaSessionId = createId('media'); private negotiationSnapshot = createScreenShareCodecNegotiationSnapshot(); private bindingRevision = 0; + private onSelectionChanged: ((room: Room, codec: VideoCodec, reason: NegotiationReason) => void) | null = null; getSelectedCodec(): VideoCodec | null { return this.selectedCodec; } + setSelectionChangeListener( + listener: ((room: Room, codec: VideoCodec, reason: NegotiationReason) => void) | null, + ): void { + this.onSelectionChanged = listener; + } + selectScreenShareCodec(preference: CodecPreference = 'auto'): VideoCodec { const selected = this.selectedCodec; const selector = (codecPreference: CodecPreference): VideoCodec => @@ -716,6 +723,7 @@ class ScreenShareCodecNegotiation { logger.info('Selected screen share codec from XState capability intersection', selection); await this.publishSessionUpdate(room, selection); if (!this.isBindingCurrent(room, bindingRevision)) return null; + this.onSelectionChanged?.(room, selection.codec, selection.reason); return selection; } diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx index 4efba2c53..79354de8a 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx @@ -433,6 +433,23 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { this.stopScreenShareAfterEncoderFailure(room, participant, track, failedCodec, 'stalled'); } + async republishActiveScreenShareForNegotiatedCodecInternal(room: Room | null, codec: VideoCodec): Promise { + const participant = room?.localParticipant; + if (!room || !participant) return; + const publication = participant.getTrackPublication(Track.Source.ScreenShare); + const track = publication?.videoTrack as LocalVideoTrack | undefined; + if (!track || !this.isScreenShareTrackPublishedInternal(participant, track)) return; + const publishedOptions = ((publication as {options?: TrackPublishOptions}).options ?? {}) as TrackPublishOptions; + if (publishedOptions.videoCodec === codec) return; + const republished = await this.liveKitFlows.republishActiveShareWithCodec(room, track, codec); + if (!republished) { + logger.warn('Failed to republish active screen share after a codec negotiation change', { + codec, + previousCodec: publishedOptions.videoCodec, + }); + } + } + private async correctVerifiedScreenShareCodec( room: Room | null, participant: LocalParticipant,