fix(voice): republish screen share when its codec goes stale (#2426)

fluxer_app/src/features/voice/engine/MediaEngineFacade.ts

@@ -58,6 +58,7 @@ import {
 	saveCurrentVoiceSessionRestoreSnapshot,
 	type VoiceSessionRestoreSyncHandle,
 } from '@app/features/voice/engine/media_engine_facade/VoiceSessionRestoreSync';
+import ScreenShareCodecNegotiation from '@app/features/voice/engine/ScreenShareCodecNegotiation';
 import ScreenSharePublicationMigration from '@app/features/voice/engine/ScreenSharePublicationMigration';
 import {Store, useStoreVersion} from '@app/features/voice/engine/Store';
 import {shouldMoveToAfkOnTick} from '@app/features/voice/engine/VoiceAfkTracking';
@@ -388,6 +389,17 @@ class MediaEngineFacade extends Store {
 		this.voiceEngineV2SourceLifecycleBridge = this.createSourceLifecycleBridge();
 		voiceEngineV2AppScreenShareExecutionAdapter.setControllerGateway(this.createScreenShareControllerGateway());
 		voiceEngineV2AppScreenShareExecutionAdapter.setSourceLifecycleBridge(this.voiceEngineV2SourceLifecycleBridge);
+		ScreenShareCodecNegotiation.setSelectionChangeListener((activeRoom, codec, reason) => {
+			void voiceEngineV2AppScreenShareExecutionAdapter
+				.republishActiveScreenShareForNegotiatedCodecInternal(activeRoom, codec)
+				.catch((error) => {
+					logger.warn('Failed to republish active screen share after a codec negotiation change', {
+						error,
+						codec,
+						reason,
+					});
+				});
+		});
 		voiceEngineV2AppMediaExecutionAdapter.setSourceLifecycleBridge(this.voiceEngineV2SourceLifecycleBridge);
 		setNativeAudioCaptureBridgeLifecycleBridge(this.voiceEngineV2SourceLifecycleBridge);
 		this.syncVoiceEngineV2AudioControlsFromAppState();

fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts

@@ -74,10 +74,45 @@ Object.defineProperty(globalThis, 'RTCRtpReceiver', {
 	value: {getCapabilities: () => VIDEO_CAPABILITIES},
 });
 
-const {buildLocalCodecAdvertisements, computeNegotiatedVideoCodec, getScreenShareCodecPreferenceOrder} = await import(
-	'./ScreenShareCodecNegotiation'
-);
+const {
+	default: ScreenShareCodecNegotiation,
+	buildLocalCodecAdvertisements,
+	computeNegotiatedVideoCodec,
+	getScreenShareCodecPreferenceOrder,
+	SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC,
+} = await import('./ScreenShareCodecNegotiation');
 const {resetCachedCodecCapabilities} = await import('@app/features/voice/utils/CodecCapabilityDetector');
+const {RoomEvent} = await import('livekit-client');
+
+class FakeRoom {
+	private handlers = new Map<string, Set<(...args: Array<unknown>) => void>>();
+	localParticipant = {identity: 'local', publishData: vi.fn(async () => undefined)};
+	remoteParticipants = new Map<string, {identity: string}>();
+
+	on(event: string, handler: (...args: Array<unknown>) => void): this {
+		if (!this.handlers.has(event)) this.handlers.set(event, new Set());
+		this.handlers.get(event)?.add(handler);
+		return this;
+	}
+
+	off(event: string, handler: (...args: Array<unknown>) => void): this {
+		this.handlers.get(event)?.delete(handler);
+		return this;
+	}
+
+	emit(event: string, ...args: Array<unknown>): void {
+		for (const handler of this.handlers.get(event) ?? []) handler(...args);
+	}
+}
+
+function sessionUpdatePayload(codecs: Array<FluxerCodecAdvertisement>): Uint8Array {
+	return new TextEncoder().encode(
+		JSON.stringify({
+			op: 14,
+			d: {video_codec: 'H264', media_session_id: 'peer-session', reason: 'connected', codecs},
+		}),
+	);
+}
 
 function videoAdvertisement(name: 'AV1' | 'VP9' | 'H265', encode: boolean, decode: boolean): FluxerCodecAdvertisement {
 	const payloadType = name === 'AV1' ? 101 : name === 'H265' ? 105 : 109;
@@ -180,3 +215,89 @@ describe('screen-share codec negotiation with the HEVC opt-in off', () => {
 		expect(computeNegotiatedVideoCodec(local, remote, 0, getScreenShareCodecPreferenceOrder()).codec).toBe('h265');
 	});
 });
+
+describe('reacting to a mid-session codec selection change', () => {
+	let room: FakeRoom;
+
+	beforeEach(() => {
+		av1OptIn = false;
+		hevcOptIn = true;
+		preferredScreenShareCodec = 'auto';
+		gpuReport = {av1: 'hardware', h265: 'hardware', h264: 'hardware', vp9: 'software', vp8: 'software'};
+		resetCachedCodecCapabilities();
+		room = new FakeRoom();
+		ScreenShareCodecNegotiation.setSelectionChangeListener(null);
+	});
+
+	function flush(): Promise<void> {
+		return new Promise((resolve) => setTimeout(resolve, 0));
+	}
+
+	async function establishBaseline(): Promise<void> {
+		ScreenShareCodecNegotiation.bind(room as never);
+		room.emit(RoomEvent.ParticipantDisconnected, {identity: 'nobody-was-connected'});
+		await flush();
+	}
+
+	it('picks the locally preferred codec while no remote participant is known', async () => {
+		await establishBaseline();
+		expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('h265');
+		ScreenShareCodecNegotiation.dispose();
+	});
+
+	it('notifies the selection-change listener once a peer without HEVC decode is discovered', async () => {
+		await establishBaseline();
+		expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('h265');
+		const listener = vi.fn();
+		ScreenShareCodecNegotiation.setSelectionChangeListener(listener);
+		room.remoteParticipants.set('peer-without-hevc', {identity: 'peer-without-hevc'});
+		const payload = sessionUpdatePayload([videoAdvertisement('VP9', true, true)]);
+		room.emit(
+			RoomEvent.DataReceived,
+			payload,
+			{identity: 'peer-without-hevc'},
+			undefined,
+			SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC,
+		);
+		await flush();
+		const negotiated = ScreenShareCodecNegotiation.getSelectedCodec();
+		expect(negotiated).not.toBe('h265');
+		expect(listener).toHaveBeenCalledTimes(1);
+		expect(listener).toHaveBeenCalledWith(room, negotiated, 'data');
+		ScreenShareCodecNegotiation.dispose();
+	});
+
+	it('does not notify the listener when the recomputed selection is unchanged', async () => {
+		await establishBaseline();
+		room.remoteParticipants.set('peer-with-hevc', {identity: 'peer-with-hevc'});
+		const firstPayload = sessionUpdatePayload([
+			videoAdvertisement('H265', true, true),
+			videoAdvertisement('VP9', true, true),
+		]);
+		room.emit(
+			RoomEvent.DataReceived,
+			firstPayload,
+			{identity: 'peer-with-hevc'},
+			undefined,
+			SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC,
+		);
+		await flush();
+		expect(ScreenShareCodecNegotiation.getSelectedCodec()).toBe('h265');
+		const listener = vi.fn();
+		ScreenShareCodecNegotiation.setSelectionChangeListener(listener);
+		const secondPayload = sessionUpdatePayload([
+			videoAdvertisement('H265', true, true),
+			videoAdvertisement('VP9', true, true),
+		]);
+		room.emit(
+			RoomEvent.DataReceived,
+			secondPayload,
+			{identity: 'peer-with-hevc'},
+			undefined,
+			SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC,
+		);
+		await flush();
+		expect(listener).not.toHaveBeenCalled();
+		ScreenShareCodecNegotiation.dispose();
+	});
+});

fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts

@@ -73,7 +73,7 @@ const VIDEO_CODEC_PROTOCOL_TABLE: Record<
 type FluxerVideoCodecName = 'AV1' | 'H265' | 'H264' | 'VP9' | 'VP8';
 type FluxerCodecName = 'opus' | FluxerVideoCodecName;
 type FluxerCodecType = 'audio' | 'video';
-type NegotiationReason =
+export type NegotiationReason =
 	| 'connected'
 	| 'data'
 	| 'participant-connected'
@@ -499,11 +499,18 @@ class ScreenShareCodecNegotiation {
 	private mediaSessionId = createId('media');
 	private negotiationSnapshot = createScreenShareCodecNegotiationSnapshot();
 	private bindingRevision = 0;
+	private onSelectionChanged: ((room: Room, codec: VideoCodec, reason: NegotiationReason) => void) | null = null;
 
 	getSelectedCodec(): VideoCodec | null {
 		return this.selectedCodec;
 	}
 
+	setSelectionChangeListener(
+		listener: ((room: Room, codec: VideoCodec, reason: NegotiationReason) => void) | null,
+	): void {
+		this.onSelectionChanged = listener;
+	}
+
 	selectScreenShareCodec(preference: CodecPreference = 'auto'): VideoCodec {
 		const selected = this.selectedCodec;
 		const selector = (codecPreference: CodecPreference): VideoCodec =>
@@ -716,6 +723,7 @@ class ScreenShareCodecNegotiation {
 		logger.info('Selected screen share codec from XState capability intersection', selection);
 		await this.publishSessionUpdate(room, selection);
 		if (!this.isBindingCurrent(room, bindingRevision)) return null;
+		this.onSelectionChanged?.(room, selection.codec, selection.reason);
 		return selection;
 	}
 

fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx

@@ -433,6 +433,23 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
 		this.stopScreenShareAfterEncoderFailure(room, participant, track, failedCodec, 'stalled');
 	}
 
+	async republishActiveScreenShareForNegotiatedCodecInternal(room: Room | null, codec: VideoCodec): Promise<void> {
+		const participant = room?.localParticipant;
+		if (!room || !participant) return;
+		const publication = participant.getTrackPublication(Track.Source.ScreenShare);
+		const track = publication?.videoTrack as LocalVideoTrack | undefined;
+		if (!track || !this.isScreenShareTrackPublishedInternal(participant, track)) return;
+		const publishedOptions = ((publication as {options?: TrackPublishOptions}).options ?? {}) as TrackPublishOptions;
+		if (publishedOptions.videoCodec === codec) return;
+		const republished = await this.liveKitFlows.republishActiveShareWithCodec(room, track, codec);
+		if (!republished) {
+			logger.warn('Failed to republish active screen share after a codec negotiation change', {
+				codec,
+				previousCodec: publishedOptions.videoCodec,
+			});
+		}
+	}
+
 	private async correctVerifiedScreenShareCodec(
 		room: Room | null,
 		participant: LocalParticipant,