From 5e0806f47963a2efa25a0f1c765c16a682a10db6 Mon Sep 17 00:00:00 2001 From: Hampus Date: Wed, 30 Sep 2026 20:47:30 +0200 Subject: [PATCH] fix(voice): preserve microphone channels during screen sharing (#3070) --- .../src/room/PCTransport.test.ts | 55 +++++++- .../livekit-client/src/room/PCTransport.ts | 31 ++++- .../room/participant/LocalParticipant.test.ts | 127 ++++++++++++++++++ .../src/room/participant/LocalParticipant.ts | 2 +- .../src/room/track/LocalTrackPublication.ts | 2 +- .../components/modals/tabs/UserVoiceTab.tsx | 2 +- .../NoiseSuppressionChoices.test.ts | 63 +++++++++ .../NoiseSuppressionChoices.ts | 6 +- 8 files changed, 274 insertions(+), 14 deletions(-) create mode 100644 fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.test.ts create mode 100644 fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.test.ts diff --git a/fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts index 0a7169e66..de4de0f6f 100644 --- a/fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts +++ b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts @@ -8,6 +8,7 @@ import type {TrackBitrateInfo} from './PCTransport.ts'; import { applyVideoStartBitrate, collectStereoMids, + conformBundledCodecFmtp, ensureAudioNackAndStereo, ensureOpusFmtp, ensureVideoDDExtension, @@ -282,15 +283,63 @@ describe('placeholderMidsFromTransceivers', () => { return {mid, currentDirection, sender: {track}} as unknown as RTCRtpTransceiver; } - it('keeps the trackless recvonly sections that still hold an m-line', () => { + it('keeps unused trackless sections that still hold an m-line', () => { const mids = placeholderMidsFromTransceivers([ - transceiver('3', null, 'recvonly'), + transceiver('3', null, 'inactive'), + transceiver('4', null, null), transceiver('7', {} as MediaStreamTrack, 'sendonly'), ]); - expect(mids).toEqual(new Set(['3'])); + expect(mids).toEqual(new Set(['3', '4'])); }); it('drops a transceiver that unpublish stopped so its recycled m-section is not fmtp-conformed', () => { expect(placeholderMidsFromTransceivers([transceiver('7', null, 'stopped')])).toEqual(new Set()); }); + + it.each(['recvonly', 'sendrecv'] as const)( + 'preserves a negotiated %s receiver without a local sender', + (direction) => { + expect(placeholderMidsFromTransceivers([transceiver('0', null, direction)])).toEqual(new Set()); + }, + ); + + it.each(['sendonly', 'sendrecv', undefined] as const)( + 'preserves a receiver activated by a remote %s answer before currentDirection updates', + (direction) => { + const incoming = opusMedia('useinbandfec=1'); + incoming.direction = direction; + expect(placeholderMidsFromTransceivers([transceiver('0', null, null)], [incoming])).toEqual(new Set()); + }, + ); + + it('keeps inactive and rejected remote sections eligible for placeholder conformance', () => { + const inactive = opusMedia('useinbandfec=1', '0'); + inactive.direction = 'inactive'; + const rejected = opusMedia('useinbandfec=1', '1'); + rejected.direction = 'sendonly'; + rejected.port = 0; + expect( + placeholderMidsFromTransceivers( + [transceiver('0', null, 'inactive'), transceiver('1', null, null)], + [inactive, rejected], + ), + ).toEqual(new Set(['0', '1'])); + }); + + it('starting a stereo share preserves incoming microphone fmtp while conforming unused placeholders', () => { + const microphone = opusMedia('useinbandfec=1;maxaveragebitrate=64000', '0'); + const placeholder = opusMedia('useinbandfec=1;usedtx=1', '1'); + const screenShare = opusMedia('useinbandfec=1', '2'); + ensureOpusFmtp(screenShare, 128000, true); + const microphoneConfig = opusConfig(microphone); + const mids = placeholderMidsFromTransceivers([ + transceiver('0', null, 'recvonly'), + transceiver('1', null, 'inactive'), + transceiver('2', {} as MediaStreamTrack, 'sendonly'), + ]); + conformBundledCodecFmtp([microphone, placeholder, screenShare], (media) => mids.has(String(media.mid))); + expect(opusConfig(microphone)).toBe(microphoneConfig); + expect(opusConfig(screenShare)).toContain('stereo=1'); + expect(opusConfig(placeholder)).toBe(opusConfig(screenShare)); + }); }); diff --git a/fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts index 099635e6a..024877909 100644 --- a/fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts +++ b/fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts @@ -257,7 +257,7 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter } } }); - const placeholderMids = this.getPlaceholderMids(); + const placeholderMids = this.getPlaceholderMids(sdpParsed.media); if (placeholderMids.size > 0) { conformBundledCodecFmtp(sdpParsed.media, (media) => placeholderMids.has(getMidString(media.mid!))); } @@ -461,8 +461,11 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter } } - private getPlaceholderMids(): Set { - return placeholderMidsFromTransceivers(this._pc?.getTransceivers() ?? []); + private getPlaceholderMids(remoteMedia?: ReadonlyArray): Set { + return placeholderMidsFromTransceivers( + this._pc?.getTransceivers() ?? [], + remoteMedia ?? parse(this._pc?.remoteDescription?.sdp ?? '').media, + ); } createDataChannel(label: string, dataChannelDict: RTCDataChannelInit) { @@ -853,10 +856,28 @@ export function collectStereoMids( return stereoMids; } -export function placeholderMidsFromTransceivers(transceivers: ReadonlyArray): Set { +export function placeholderMidsFromTransceivers( + transceivers: ReadonlyArray, + remoteMedia: ReadonlyArray = [], +): Set { + const receivingMids = new Set( + remoteMedia + .filter( + (media) => + media.mid !== undefined && + media.port !== 0 && + (media.direction === 'sendonly' || media.direction === 'sendrecv' || media.direction === undefined), + ) + .map((media) => getMidString(media.mid!)), + ); const mids = new Set(); for (const transceiver of transceivers) { - if (transceiver.currentDirection === 'stopped') { + if ( + transceiver.currentDirection === 'stopped' || + transceiver.currentDirection === 'recvonly' || + transceiver.currentDirection === 'sendrecv' || + (transceiver.mid !== null && receivingMids.has(transceiver.mid)) + ) { continue; } if (transceiver.mid && !transceiver.sender.track) { diff --git a/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.test.ts b/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.test.ts new file mode 100644 index 000000000..d13592f6a --- /dev/null +++ b/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.test.ts @@ -0,0 +1,127 @@ +import {type AddTrackRequest, AudioTrackFeature, ParticipantPermission, TrackInfo} from '@livekit/protocol'; +import {EventEmitter} from 'events'; +import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; +import {SignalConnectionState} from '../../api/SignalClient.ts'; +import {publishDefaults, roomOptionDefaults} from '../defaults.ts'; +import {TrackEvent} from '../events.ts'; +import type RTCEngine from '../RTCEngine.ts'; +import LocalAudioTrack from '../track/LocalAudioTrack.ts'; +import {Track} from '../track/Track.ts'; +import LocalParticipant from './LocalParticipant.ts'; + +class TestMediaStreamTrack extends EventTarget { + readonly id = 'microphone-input'; + readonly kind = 'audio'; + enabled = true; + readyState: MediaStreamTrackState = 'live'; + + constructor(private readonly channelCount: number) { + super(); + } + + getSettings(): MediaTrackSettings { + return {channelCount: this.channelCount, echoCancellation: true}; + } + + getConstraints(): MediaTrackConstraints { + return {}; + } + + stop() { + this.readyState = 'ended'; + } +} + +class TestMediaStream { + constructor(private readonly tracks: Array) {} + + getTracks() { + return this.tracks; + } +} + +function createPublisher() { + const setTrackCodecBitrate = vi.fn(); + const sendUpdateLocalAudioTrack = vi.fn(); + const sender = {replaceTrack: vi.fn(async () => undefined)} as unknown as RTCRtpSender; + const transceiver = {sender} as RTCRtpTransceiver; + const addTrack = vi.fn( + async (request: AddTrackRequest) => + new TrackInfo({ + sid: 'TR_microphone', + name: request.name, + type: request.type, + source: request.source, + audioFeatures: request.audioFeatures, + }), + ); + const engine = Object.assign(new EventEmitter(), { + isClosed: false, + logContext: {}, + client: {currentState: SignalConnectionState.CONNECTED, sendUpdateLocalAudioTrack}, + pcManager: {publisher: {getTransceivers: () => [transceiver], setTrackCodecBitrate}}, + createSender: vi.fn(async () => sender), + negotiate: vi.fn(async () => undefined), + addTrack, + }); + type ParticipantArgs = ConstructorParameters; + const participant = new LocalParticipant( + 'PA_publisher', + 'publisher', + engine as unknown as RTCEngine, + {...roomOptionDefaults, publishDefaults}, + {} as ParticipantArgs[4], + {} as ParticipantArgs[5], + {} as ParticipantArgs[6], + {} as ParticipantArgs[7], + ); + participant.permissions = new ParticipantPermission({canPublish: true}); + return {participant, addTrack, setTrackCodecBitrate, sendUpdateLocalAudioTrack}; +} + +describe('microphone publication stereo metadata', () => { + beforeEach(() => { + vi.stubGlobal('MediaStreamTrack', TestMediaStreamTrack); + vi.stubGlobal('MediaStream', TestMediaStream); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it.each([ + {name: 'mono policy on a two-channel device', channelCount: 2, forceStereo: false, stereo: false}, + {name: 'automatic stereo on a two-channel device', channelCount: 2, forceStereo: undefined, stereo: true}, + {name: 'forced stereo on a one-channel device', channelCount: 1, forceStereo: true, stereo: true}, + {name: 'automatic mono on a one-channel device', channelCount: 1, forceStereo: undefined, stereo: false}, + ])( + 'keeps the request, SDP policy, and feature updates consistent for $name', + async ({channelCount, forceStereo, stereo}) => { + const {participant, addTrack, setTrackCodecBitrate, sendUpdateLocalAudioTrack} = createPublisher(); + const source = new TestMediaStreamTrack(channelCount); + const track = new LocalAudioTrack(source as unknown as MediaStreamTrack); + track.source = Track.Source.Microphone; + await track.runWithTrackChangeLock(async () => undefined); + const publication = await participant.publishTrack(track, { + audioPreset: {maxBitrate: 64000}, + forceStereo, + }); + + expect(addTrack).toHaveBeenCalledTimes(1); + const request = addTrack.mock.calls[0]![0]; + expect(request.stereo).toBe(stereo); + expect(request.audioFeatures.includes(AudioTrackFeature.TF_STEREO)).toBe(stereo); + expect(request.audioFeatures).toContain(AudioTrackFeature.TF_ECHO_CANCELLATION); + expect(setTrackCodecBitrate).toHaveBeenCalledWith(expect.objectContaining({codec: 'opus', maxbr: 64, stereo})); + + expect(publication.getTrackFeatures().includes(AudioTrackFeature.TF_STEREO)).toBe(stereo); + track.emit(TrackEvent.AudioTrackFeatureUpdate, track, AudioTrackFeature.TF_ECHO_CANCELLATION, true); + expect(sendUpdateLocalAudioTrack).toHaveBeenCalledTimes(1); + const [sid, features] = sendUpdateLocalAudioTrack.mock.calls[0]!; + expect(sid).toBe('TR_microphone'); + expect(features.includes(AudioTrackFeature.TF_STEREO)).toBe(stereo); + expect(features).toContain(AudioTrackFeature.TF_ECHO_CANCELLATION); + track.stop(); + }, + ); +}); diff --git a/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts b/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts index d25f73b35..8ae43e7cb 100644 --- a/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts +++ b/fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts @@ -966,7 +966,7 @@ export default class LocalParticipant extends Participant { if (settings.noiseSuppression) { audioFeatures.push(AudioTrackFeature.TF_NOISE_SUPPRESSION); } - if (settings.channelCount && settings.channelCount > 1) { + if (isStereo) { audioFeatures.push(AudioTrackFeature.TF_STEREO); } if (disableDtx) { diff --git a/fluxer_app/pkgs/livekit-client/src/room/track/LocalTrackPublication.ts b/fluxer_app/pkgs/livekit-client/src/room/track/LocalTrackPublication.ts index de5b88d82..01095c574 100644 --- a/fluxer_app/pkgs/livekit-client/src/room/track/LocalTrackPublication.ts +++ b/fluxer_app/pkgs/livekit-client/src/room/track/LocalTrackPublication.ts @@ -90,7 +90,7 @@ export default class LocalTrackPublication extends TrackPublication { if (settings.noiseSuppression) { features.add(AudioTrackFeature.TF_NOISE_SUPPRESSION); } - if (settings.channelCount && settings.channelCount > 1) { + if (this.options?.forceStereo ?? (settings.channelCount !== undefined && settings.channelCount > 1)) { features.add(AudioTrackFeature.TF_STEREO); } if (!this.options?.dtx) { diff --git a/fluxer_app/src/features/user/components/modals/tabs/UserVoiceTab.tsx b/fluxer_app/src/features/user/components/modals/tabs/UserVoiceTab.tsx index e33315459..6329b1ee5 100644 --- a/fluxer_app/src/features/user/components/modals/tabs/UserVoiceTab.tsx +++ b/fluxer_app/src/features/user/components/modals/tabs/UserVoiceTab.tsx @@ -294,7 +294,7 @@ export const VoiceTab: React.FC = observer(({voiceSettings, autoR value: backend, label: getNoiseSuppressionChoiceLabel(i18n, backend), })); - const stereoMicrophoneAvailable = isStereoMicrophoneChoiceAvailable(); + const stereoMicrophoneAvailable = isStereoMicrophoneChoiceAvailable(activeInputLabel); const setPushToTalkEnabled = (enabled: boolean) => { const mode = enabled ? 'voice_push_to_talk' : 'voice_activity'; if (enabled && !isNativeDesktop) { diff --git a/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.test.ts b/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.test.ts new file mode 100644 index 000000000..cd81f7570 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.test.ts @@ -0,0 +1,63 @@ +import type {VoiceProcessingMode} from '@app/features/voice/utils/VoiceProcessingProfile'; +import {beforeEach, describe, expect, it, vi} from 'vitest'; + +const {deviceModes, voiceSettings} = vi.hoisted(() => ({ + deviceModes: new Map(), + voiceSettings: { + voiceProcessingMode: 'voice' as VoiceProcessingMode, + echoCancellation: true, + autoGainControl: false, + getNoiseSuppressionBackend: () => 'none', + getStereoMicrophone: () => false, + getVoiceProcessingModeForDeviceLabel: vi.fn<(label: string | null | undefined) => VoiceProcessingMode>(), + }, +})); + +vi.mock('@app/features/voice/state/VoiceSettings', () => ({default: voiceSettings})); +vi.mock('@app/features/voice/commands/VoiceSettingsCommands', () => ({update: vi.fn()})); +vi.mock('@app/features/voice/engine/VoiceDevicePermissionState', () => ({ + default: {getState: () => ({inputDevices: []})}, +})); +vi.mock('@app/features/voice/utils/VoiceDeviceManager', () => ({resolveEffectiveDeviceId: () => null})); + +import {isStereoMicrophoneChoiceAvailable} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionChoices'; + +describe('stereo microphone choice for the selected device', () => { + beforeEach(() => { + deviceModes.clear(); + voiceSettings.voiceProcessingMode = 'voice'; + voiceSettings.echoCancellation = true; + voiceSettings.autoGainControl = false; + voiceSettings.getVoiceProcessingModeForDeviceLabel.mockReset(); + voiceSettings.getVoiceProcessingModeForDeviceLabel.mockImplementation( + (label) => (label ? deviceModes.get(label) : undefined) ?? voiceSettings.voiceProcessingMode, + ); + }); + + it('shows the switch for a device using direct input when the global mode is focused voice', () => { + deviceModes.set('USB audio interface', 'studio'); + expect(isStereoMicrophoneChoiceAvailable('USB audio interface')).toBe(true); + expect(voiceSettings.getVoiceProcessingModeForDeviceLabel).toHaveBeenCalledWith('USB audio interface'); + }); + + it('hides the switch for a device with custom processing when the global mode is direct input', () => { + voiceSettings.voiceProcessingMode = 'studio'; + deviceModes.set('Headset microphone', 'custom'); + expect(isStereoMicrophoneChoiceAvailable('Headset microphone')).toBe(false); + }); + + it('shows the switch for a custom device profile with processing disabled', () => { + deviceModes.set('USB audio interface', 'custom'); + voiceSettings.echoCancellation = false; + expect(isStereoMicrophoneChoiceAvailable('USB audio interface')).toBe(true); + }); + + it.each([ + {mode: 'voice' as const, available: false}, + {mode: 'studio' as const, available: true}, + ])('uses the global $mode profile when the device has no override', ({mode, available}) => { + voiceSettings.voiceProcessingMode = mode; + expect(isStereoMicrophoneChoiceAvailable('Unconfigured microphone')).toBe(available); + expect(isStereoMicrophoneChoiceAvailable(null)).toBe(available); + }); +}); diff --git a/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.ts b/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.ts index 302cdacb1..a0b3f1a8f 100644 --- a/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.ts +++ b/fluxer_app/src/features/voice/utils/noise_suppression/NoiseSuppressionChoices.ts @@ -12,7 +12,7 @@ import { readNoiseSuppressionRuntimeCapabilities, supportsStereoCapture, } from '@app/features/voice/utils/noise_suppression/NoiseSuppressionSelection'; -import {resolveVoiceProcessingFromState} from '@app/features/voice/utils/VoiceProcessingProfile'; +import {resolveVoiceProcessingFromStateForDeviceLabel} from '@app/features/voice/utils/VoiceProcessingProfile'; export function getNoiseSuppressionChoiceValues(): ReadonlyArray { const capabilities = readNoiseSuppressionRuntimeCapabilities(); @@ -29,8 +29,8 @@ export function setNoiseSuppressionChoice(backend: VoiceNoiseSuppressionBackend) VoiceSettingsCommands.update({noiseSuppressionBackend: backend}); } -export function isStereoMicrophoneChoiceAvailable(): boolean { - return supportsStereoCapture(resolveVoiceProcessingFromState(VoiceSettings)); +export function isStereoMicrophoneChoiceAvailable(deviceLabel: string | null): boolean { + return supportsStereoCapture(resolveVoiceProcessingFromStateForDeviceLabel(VoiceSettings, deviceLabel)); } export function isStereoMicrophoneEnabled(): boolean {