fix(voice): preserve microphone channels during screen sharing (#3070)

fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts

@@ -8,6 +8,7 @@ import type {TrackBitrateInfo} from './PCTransport.ts';
 import {
 	applyVideoStartBitrate,
 	collectStereoMids,
+	conformBundledCodecFmtp,
 	ensureAudioNackAndStereo,
 	ensureOpusFmtp,
 	ensureVideoDDExtension,
@@ -282,15 +283,63 @@ describe('placeholderMidsFromTransceivers', () => {
 		return {mid, currentDirection, sender: {track}} as unknown as RTCRtpTransceiver;
 	}
 
-	it('keeps the trackless recvonly sections that still hold an m-line', () => {
+	it('keeps unused trackless sections that still hold an m-line', () => {
 		const mids = placeholderMidsFromTransceivers([
-			transceiver('3', null, 'recvonly'),
+			transceiver('3', null, 'inactive'),
+			transceiver('4', null, null),
 			transceiver('7', {} as MediaStreamTrack, 'sendonly'),
 		]);
-		expect(mids).toEqual(new Set(['3']));
+		expect(mids).toEqual(new Set(['3', '4']));
 	});
 
 	it('drops a transceiver that unpublish stopped so its recycled m-section is not fmtp-conformed', () => {
 		expect(placeholderMidsFromTransceivers([transceiver('7', null, 'stopped')])).toEqual(new Set());
 	});
+
+	it.each(['recvonly', 'sendrecv'] as const)(
+		'preserves a negotiated %s receiver without a local sender',
+		(direction) => {
+			expect(placeholderMidsFromTransceivers([transceiver('0', null, direction)])).toEqual(new Set());
+		},
+	);
+
+	it.each(['sendonly', 'sendrecv', undefined] as const)(
+		'preserves a receiver activated by a remote %s answer before currentDirection updates',
+		(direction) => {
+			const incoming = opusMedia('useinbandfec=1');
+			incoming.direction = direction;
+			expect(placeholderMidsFromTransceivers([transceiver('0', null, null)], [incoming])).toEqual(new Set());
+		},
+	);
+
+	it('keeps inactive and rejected remote sections eligible for placeholder conformance', () => {
+		const inactive = opusMedia('useinbandfec=1', '0');
+		inactive.direction = 'inactive';
+		const rejected = opusMedia('useinbandfec=1', '1');
+		rejected.direction = 'sendonly';
+		rejected.port = 0;
+		expect(
+			placeholderMidsFromTransceivers(
+				[transceiver('0', null, 'inactive'), transceiver('1', null, null)],
+				[inactive, rejected],
+			),
+		).toEqual(new Set(['0', '1']));
+	});
+
+	it('starting a stereo share preserves incoming microphone fmtp while conforming unused placeholders', () => {
+		const microphone = opusMedia('useinbandfec=1;maxaveragebitrate=64000', '0');
+		const placeholder = opusMedia('useinbandfec=1;usedtx=1', '1');
+		const screenShare = opusMedia('useinbandfec=1', '2');
+		ensureOpusFmtp(screenShare, 128000, true);
+		const microphoneConfig = opusConfig(microphone);
+		const mids = placeholderMidsFromTransceivers([
+			transceiver('0', null, 'recvonly'),
+			transceiver('1', null, 'inactive'),
+			transceiver('2', {} as MediaStreamTrack, 'sendonly'),
+		]);
+		conformBundledCodecFmtp([microphone, placeholder, screenShare], (media) => mids.has(String(media.mid)));
+		expect(opusConfig(microphone)).toBe(microphoneConfig);
+		expect(opusConfig(screenShare)).toContain('stereo=1');
+		expect(opusConfig(placeholder)).toBe(opusConfig(screenShare));
+	});
 });

fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts

@@ -257,7 +257,7 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter
 					}
 				}
 			});
-			const placeholderMids = this.getPlaceholderMids();
+			const placeholderMids = this.getPlaceholderMids(sdpParsed.media);
 			if (placeholderMids.size > 0) {
 				conformBundledCodecFmtp(sdpParsed.media, (media) => placeholderMids.has(getMidString(media.mid!)));
 			}
@@ -461,8 +461,11 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter
 		}
 	}
 
-	private getPlaceholderMids(): Set<string> {
-		return placeholderMidsFromTransceivers(this._pc?.getTransceivers() ?? []);
+	private getPlaceholderMids(remoteMedia?: ReadonlyArray<MediaDescription>): Set<string> {
+		return placeholderMidsFromTransceivers(
+			this._pc?.getTransceivers() ?? [],
+			remoteMedia ?? parse(this._pc?.remoteDescription?.sdp ?? '').media,
+		);
 	}
 
 	createDataChannel(label: string, dataChannelDict: RTCDataChannelInit) {
@@ -853,10 +856,28 @@ export function collectStereoMids(
 	return stereoMids;
 }
 
-export function placeholderMidsFromTransceivers(transceivers: ReadonlyArray<RTCRtpTransceiver>): Set<string> {
+export function placeholderMidsFromTransceivers(
+	transceivers: ReadonlyArray<RTCRtpTransceiver>,
+	remoteMedia: ReadonlyArray<MediaDescription> = [],
+): Set<string> {
+	const receivingMids = new Set(
+		remoteMedia
+			.filter(
+				(media) =>
+					media.mid !== undefined &&
+					media.port !== 0 &&
+					(media.direction === 'sendonly' || media.direction === 'sendrecv' || media.direction === undefined),
+			)
+			.map((media) => getMidString(media.mid!)),
+	);
 	const mids = new Set<string>();
 	for (const transceiver of transceivers) {
-		if (transceiver.currentDirection === 'stopped') {
+		if (
+			transceiver.currentDirection === 'stopped' ||
+			transceiver.currentDirection === 'recvonly' ||
+			transceiver.currentDirection === 'sendrecv' ||
+			(transceiver.mid !== null && receivingMids.has(transceiver.mid))
+		) {
 			continue;
 		}
 		if (transceiver.mid && !transceiver.sender.track) {

fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.test.ts

@@ -0,0 +1,127 @@
+import {type AddTrackRequest, AudioTrackFeature, ParticipantPermission, TrackInfo} from '@livekit/protocol';
+import {EventEmitter} from 'events';
+import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
+import {SignalConnectionState} from '../../api/SignalClient.ts';
+import {publishDefaults, roomOptionDefaults} from '../defaults.ts';
+import {TrackEvent} from '../events.ts';
+import type RTCEngine from '../RTCEngine.ts';
+import LocalAudioTrack from '../track/LocalAudioTrack.ts';
+import {Track} from '../track/Track.ts';
+import LocalParticipant from './LocalParticipant.ts';
+
+class TestMediaStreamTrack extends EventTarget {
+	readonly id = 'microphone-input';
+	readonly kind = 'audio';
+	enabled = true;
+	readyState: MediaStreamTrackState = 'live';
+
+	constructor(private readonly channelCount: number) {
+		super();
+	}
+
+	getSettings(): MediaTrackSettings {
+		return {channelCount: this.channelCount, echoCancellation: true};
+	}
+
+	getConstraints(): MediaTrackConstraints {
+		return {};
+	}
+
+	stop() {
+		this.readyState = 'ended';
+	}
+}
+
+class TestMediaStream {
+	constructor(private readonly tracks: Array<MediaStreamTrack>) {}
+
+	getTracks() {
+		return this.tracks;
+	}
+}
+
+function createPublisher() {
+	const setTrackCodecBitrate = vi.fn();
+	const sendUpdateLocalAudioTrack = vi.fn();
+	const sender = {replaceTrack: vi.fn(async () => undefined)} as unknown as RTCRtpSender;
+	const transceiver = {sender} as RTCRtpTransceiver;
+	const addTrack = vi.fn(
+		async (request: AddTrackRequest) =>
+			new TrackInfo({
+				sid: 'TR_microphone',
+				name: request.name,
+				type: request.type,
+				source: request.source,
+				audioFeatures: request.audioFeatures,
+			}),
+	);
+	const engine = Object.assign(new EventEmitter(), {
+		isClosed: false,
+		logContext: {},
+		client: {currentState: SignalConnectionState.CONNECTED, sendUpdateLocalAudioTrack},
+		pcManager: {publisher: {getTransceivers: () => [transceiver], setTrackCodecBitrate}},
+		createSender: vi.fn(async () => sender),
+		negotiate: vi.fn(async () => undefined),
+		addTrack,
+	});
+	type ParticipantArgs = ConstructorParameters<typeof LocalParticipant>;
+	const participant = new LocalParticipant(
+		'PA_publisher',
+		'publisher',
+		engine as unknown as RTCEngine,
+		{...roomOptionDefaults, publishDefaults},
+		{} as ParticipantArgs[4],
+		{} as ParticipantArgs[5],
+		{} as ParticipantArgs[6],
+		{} as ParticipantArgs[7],
+	);
+	participant.permissions = new ParticipantPermission({canPublish: true});
+	return {participant, addTrack, setTrackCodecBitrate, sendUpdateLocalAudioTrack};
+}
+
+describe('microphone publication stereo metadata', () => {
+	beforeEach(() => {
+		vi.stubGlobal('MediaStreamTrack', TestMediaStreamTrack);
+		vi.stubGlobal('MediaStream', TestMediaStream);
+	});
+
+	afterEach(() => {
+		vi.unstubAllGlobals();
+	});
+
+	it.each([
+		{name: 'mono policy on a two-channel device', channelCount: 2, forceStereo: false, stereo: false},
+		{name: 'automatic stereo on a two-channel device', channelCount: 2, forceStereo: undefined, stereo: true},
+		{name: 'forced stereo on a one-channel device', channelCount: 1, forceStereo: true, stereo: true},
+		{name: 'automatic mono on a one-channel device', channelCount: 1, forceStereo: undefined, stereo: false},
+	])(
+		'keeps the request, SDP policy, and feature updates consistent for $name',
+		async ({channelCount, forceStereo, stereo}) => {
+			const {participant, addTrack, setTrackCodecBitrate, sendUpdateLocalAudioTrack} = createPublisher();
+			const source = new TestMediaStreamTrack(channelCount);
+			const track = new LocalAudioTrack(source as unknown as MediaStreamTrack);
+			track.source = Track.Source.Microphone;
+			await track.runWithTrackChangeLock(async () => undefined);
+			const publication = await participant.publishTrack(track, {
+				audioPreset: {maxBitrate: 64000},
+				forceStereo,
+			});
+
+			expect(addTrack).toHaveBeenCalledTimes(1);
+			const request = addTrack.mock.calls[0]![0];
+			expect(request.stereo).toBe(stereo);
+			expect(request.audioFeatures.includes(AudioTrackFeature.TF_STEREO)).toBe(stereo);
+			expect(request.audioFeatures).toContain(AudioTrackFeature.TF_ECHO_CANCELLATION);
+			expect(setTrackCodecBitrate).toHaveBeenCalledWith(expect.objectContaining({codec: 'opus', maxbr: 64, stereo}));
+
+			expect(publication.getTrackFeatures().includes(AudioTrackFeature.TF_STEREO)).toBe(stereo);
+			track.emit(TrackEvent.AudioTrackFeatureUpdate, track, AudioTrackFeature.TF_ECHO_CANCELLATION, true);
+			expect(sendUpdateLocalAudioTrack).toHaveBeenCalledTimes(1);
+			const [sid, features] = sendUpdateLocalAudioTrack.mock.calls[0]!;
+			expect(sid).toBe('TR_microphone');
+			expect(features.includes(AudioTrackFeature.TF_STEREO)).toBe(stereo);
+			expect(features).toContain(AudioTrackFeature.TF_ECHO_CANCELLATION);
+			track.stop();
+		},
+	);
+});

fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts

@@ -966,7 +966,7 @@ export default class LocalParticipant extends Participant {
 		if (settings.noiseSuppression) {
 			audioFeatures.push(AudioTrackFeature.TF_NOISE_SUPPRESSION);
 		}
-		if (settings.channelCount && settings.channelCount > 1) {
+		if (isStereo) {
 			audioFeatures.push(AudioTrackFeature.TF_STEREO);
 		}
 		if (disableDtx) {

fluxer_app/pkgs/livekit-client/src/room/track/LocalTrackPublication.ts

@@ -90,7 +90,7 @@ export default class LocalTrackPublication extends TrackPublication {
 			if (settings.noiseSuppression) {
 				features.add(AudioTrackFeature.TF_NOISE_SUPPRESSION);
 			}
-			if (settings.channelCount && settings.channelCount > 1) {
+			if (this.options?.forceStereo ?? (settings.channelCount !== undefined && settings.channelCount > 1)) {
 				features.add(AudioTrackFeature.TF_STEREO);
 			}
 			if (!this.options?.dtx) {

fluxer_app/src/features/user/components/modals/tabs/UserVoiceTab.tsx

@@ -294,7 +294,7 @@ export const VoiceTab: React.FC<VoiceTabProps> = observer(({voiceSettings, autoR
 			value: backend,
 			label: getNoiseSuppressionChoiceLabel(i18n, backend),
 		}));
-	const stereoMicrophoneAvailable = isStereoMicrophoneChoiceAvailable();
+	const stereoMicrophoneAvailable = isStereoMicrophoneChoiceAvailable(activeInputLabel);
 	const setPushToTalkEnabled = (enabled: boolean) => {
 		const mode = enabled ? 'voice_push_to_talk' : 'voice_activity';
 		if (enabled && !isNativeDesktop) {

fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.test.ts

@@ -0,0 +1,63 @@
+import type {VoiceProcessingMode} from '@app/features/voice/utils/VoiceProcessingProfile';
+import {beforeEach, describe, expect, it, vi} from 'vitest';
+
+const {deviceModes, voiceSettings} = vi.hoisted(() => ({
+	deviceModes: new Map<string, VoiceProcessingMode>(),
+	voiceSettings: {
+		voiceProcessingMode: 'voice' as VoiceProcessingMode,
+		echoCancellation: true,
+		autoGainControl: false,
+		getNoiseSuppressionBackend: () => 'none',
+		getStereoMicrophone: () => false,
+		getVoiceProcessingModeForDeviceLabel: vi.fn<(label: string | null | undefined) => VoiceProcessingMode>(),
+	},
+}));
+
+vi.mock('@app/features/voice/state/VoiceSettings', () => ({default: voiceSettings}));
+vi.mock('@app/features/voice/commands/VoiceSettingsCommands', () => ({update: vi.fn()}));
+vi.mock('@app/features/voice/engine/VoiceDevicePermissionState', () => ({
+	default: {getState: () => ({inputDevices: []})},
+}));
+vi.mock('@app/features/voice/utils/VoiceDeviceManager', () => ({resolveEffectiveDeviceId: () => null}));
+
+import {isStereoMicrophoneChoiceAvailable} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionChoices';
+
+describe('stereo microphone choice for the selected device', () => {
+	beforeEach(() => {
+		deviceModes.clear();
+		voiceSettings.voiceProcessingMode = 'voice';
+		voiceSettings.echoCancellation = true;
+		voiceSettings.autoGainControl = false;
+		voiceSettings.getVoiceProcessingModeForDeviceLabel.mockReset();
+		voiceSettings.getVoiceProcessingModeForDeviceLabel.mockImplementation(
+			(label) => (label ? deviceModes.get(label) : undefined) ?? voiceSettings.voiceProcessingMode,
+		);
+	});
+
+	it('shows the switch for a device using direct input when the global mode is focused voice', () => {
+		deviceModes.set('USB audio interface', 'studio');
+		expect(isStereoMicrophoneChoiceAvailable('USB audio interface')).toBe(true);
+		expect(voiceSettings.getVoiceProcessingModeForDeviceLabel).toHaveBeenCalledWith('USB audio interface');
+	});
+
+	it('hides the switch for a device with custom processing when the global mode is direct input', () => {
+		voiceSettings.voiceProcessingMode = 'studio';
+		deviceModes.set('Headset microphone', 'custom');
+		expect(isStereoMicrophoneChoiceAvailable('Headset microphone')).toBe(false);
+	});
+
+	it('shows the switch for a custom device profile with processing disabled', () => {
+		deviceModes.set('USB audio interface', 'custom');
+		voiceSettings.echoCancellation = false;
+		expect(isStereoMicrophoneChoiceAvailable('USB audio interface')).toBe(true);
+	});
+
+	it.each([
+		{mode: 'voice' as const, available: false},
+		{mode: 'studio' as const, available: true},
+	])('uses the global $mode profile when the device has no override', ({mode, available}) => {
+		voiceSettings.voiceProcessingMode = mode;
+		expect(isStereoMicrophoneChoiceAvailable('Unconfigured microphone')).toBe(available);
+		expect(isStereoMicrophoneChoiceAvailable(null)).toBe(available);
+	});
+});

fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.ts

@@ -12,7 +12,7 @@ import {
 	readNoiseSuppressionRuntimeCapabilities,
 	supportsStereoCapture,
 } from '@app/features/voice/utils/noise_suppression/NoiseSuppressionSelection';
-import {resolveVoiceProcessingFromState} from '@app/features/voice/utils/VoiceProcessingProfile';
+import {resolveVoiceProcessingFromStateForDeviceLabel} from '@app/features/voice/utils/VoiceProcessingProfile';
 
 export function getNoiseSuppressionChoiceValues(): ReadonlyArray<VoiceNoiseSuppressionBackend> {
 	const capabilities = readNoiseSuppressionRuntimeCapabilities();
@@ -29,8 +29,8 @@ export function setNoiseSuppressionChoice(backend: VoiceNoiseSuppressionBackend)
 	VoiceSettingsCommands.update({noiseSuppressionBackend: backend});
 }
 
-export function isStereoMicrophoneChoiceAvailable(): boolean {
-	return supportsStereoCapture(resolveVoiceProcessingFromState(VoiceSettings));
+export function isStereoMicrophoneChoiceAvailable(deviceLabel: string | null): boolean {
+	return supportsStereoCapture(resolveVoiceProcessingFromStateForDeviceLabel(VoiceSettings, deviceLabel));
 }
 
 export function isStereoMicrophoneEnabled(): boolean {