fix(voice): send screen shares at the quality the user picked (#2840)

fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts

@@ -11,6 +11,7 @@ import {
 	ensureAudioNackAndStereo,
 	ensureOpusFmtp,
 	ensureVideoDDExtension,
+	placeholderMidsFromTransceivers,
 	videoSectionCanReceiveAV1,
 } from './PCTransport.ts';
 import {ddExtensionURI} from './utils.ts';
@@ -239,3 +240,25 @@ describe('ensureVideoDDExtension', () => {
 		expect(sectionOf(sdp, '2').ext).toHaveLength(2);
 	});
 });
+
+describe('placeholderMidsFromTransceivers', () => {
+	function transceiver(
+		mid: string | null,
+		track: MediaStreamTrack | null,
+		currentDirection: RTCRtpTransceiverDirection | null,
+	): RTCRtpTransceiver {
+		return {mid, currentDirection, sender: {track}} as unknown as RTCRtpTransceiver;
+	}
+
+	it('keeps the trackless recvonly sections that still hold an m-line', () => {
+		const mids = placeholderMidsFromTransceivers([
+			transceiver('3', null, 'recvonly'),
+			transceiver('7', {} as MediaStreamTrack, 'sendonly'),
+		]);
+		expect(mids).toEqual(new Set(['3']));
+	});
+
+	it('drops a transceiver that unpublish stopped so its recycled m-section is not fmtp-conformed', () => {
+		expect(placeholderMidsFromTransceivers([transceiver('7', null, 'stopped')])).toEqual(new Set());
+	});
+});

fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts

@@ -849,6 +849,9 @@ export function collectStereoMids(
 export function placeholderMidsFromTransceivers(transceivers: ReadonlyArray<RTCRtpTransceiver>): Set<string> {
 	const mids = new Set<string>();
 	for (const transceiver of transceivers) {
+		if (transceiver.currentDirection === 'stopped') {
+			continue;
+		}
 		if (transceiver.mid && !transceiver.sender.track) {
 			mids.add(transceiver.mid);
 		}

fluxer_app/pkgs/livekit-client/src/room/PCTransportManager.ts

@@ -106,11 +106,6 @@ export class PCTransportManager {
 		this.isPublisherConnectionRequired = mode !== 'subscriber-primary';
 		this.isSubscriberConnectionRequired = mode === 'subscriber-primary';
 		this.publisher = new PCTransport(rtcConfig, loggerOptions);
-		if (subscriberVideoCodecExclusions?.length) {
-			for (const codec of subscriberVideoCodecExclusions) {
-				this.publisher.excludedVideoDecoderMimeTypes.add(`video/${codec}`);
-			}
-		}
 		this._mode = mode;
 		if (mode !== 'publisher-only') {
 			this.subscriber = new PCTransport(rtcConfig, loggerOptions);

fluxer_app/pkgs/livekit-client/src/room/RTCEngine.test.ts

@@ -2,7 +2,8 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import {describe, expect, it} from 'vitest';
-import {selectPublisherCodecPreferences} from './RTCEngine.ts';
+import type {InternalRoomOptions} from '../options.ts';
+import RTCEngine, {selectPublisherCodecPreferences} from './RTCEngine.ts';
 
 function codec(
 	mimeType: string,
@@ -112,3 +113,42 @@ describe('selectPublisherCodecPreferences', () => {
 		expect(selectPublisherCodecPreferences('av1', [codec('video/VP8'), codec('video/rtx')])).toEqual([]);
 	});
 });
+
+describe('publisher data channels before negotiation', () => {
+	function engineWithPublisherChannels(hasPublisherChannels: boolean) {
+		const engine = new RTCEngine({} as InternalRoomOptions);
+		const created: Array<string> = [];
+		const internals = engine as unknown as {
+			_isClosed: boolean;
+			pcManager: unknown;
+			dataChannels: {hasPublisherChannels: boolean; createPublisherChannels: () => void};
+		};
+		internals._isClosed = false;
+		internals.dataChannels = {
+			hasPublisherChannels,
+			createPublisherChannels: () => created.push('publisher'),
+		};
+		internals.pcManager = {
+			requirePublisher: () => {},
+			negotiate: async () => {},
+			publisher: {
+				off: () => {},
+				once: () => {},
+				getTransceivers: () => [{} as RTCRtpTransceiver],
+			},
+		};
+		return {engine, created};
+	}
+
+	it('creates them on a renegotiation that already carries transceivers', async () => {
+		const {engine, created} = engineWithPublisherChannels(false);
+		await engine.negotiate();
+		expect(created).toEqual(['publisher']);
+	});
+
+	it('leaves the existing channels alone', async () => {
+		const {engine, created} = engineWithPublisherChannels(true);
+		await engine.negotiate();
+		expect(created).toEqual([]);
+	});
+});

fluxer_app/pkgs/livekit-client/src/room/RTCEngine.ts

@@ -1559,7 +1559,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
 			}
 
 			this.pcManager.requirePublisher();
-			if (this.pcManager.publisher.getTransceivers().length === 0 && !this.dataChannels.hasPublisherChannels) {
+			if (!this.dataChannels.hasPublisherChannels) {
 				this.createDataChannels();
 			}
 

fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts

@@ -108,6 +108,7 @@ import {
 	isWeb,
 	selectPreferredVideoCodec,
 	sleep,
+	stopTransceiversForSender,
 	supportsVideoCodec,
 	usesLegacySVCEncodings,
 } from '../utils.ts';
@@ -436,8 +437,9 @@ export default class LocalParticipant extends Participant {
 		enabled: boolean,
 		options?: ScreenShareCaptureOptions,
 		publishOptions?: TrackPublishOptions,
+		audioPublishOptions?: TrackPublishOptions,
 	): Promise<LocalTrackPublication | undefined> {
-		return this.setTrackEnabled(Track.Source.ScreenShare, enabled, options, publishOptions);
+		return this.setTrackEnabled(Track.Source.ScreenShare, enabled, options, publishOptions, audioPublishOptions);
 	}
 
 	async setE2EEEnabled(enabled: boolean) {
@@ -474,12 +476,14 @@ export default class LocalParticipant extends Participant {
 		enabled: boolean,
 		options?: ScreenShareCaptureOptions,
 		publishOptions?: TrackPublishOptions,
+		audioPublishOptions?: TrackPublishOptions,
 	): Promise<LocalTrackPublication | undefined>;
 	private async setTrackEnabled(
 		source: Track.Source,
 		enabled: true,
 		options?: VideoCaptureOptions | AudioCaptureOptions | ScreenShareCaptureOptions,
 		publishOptions?: TrackPublishOptions,
+		audioPublishOptions?: TrackPublishOptions,
 	) {
 		this.log.debug('setTrackEnabled', {source, enabled});
 		if (this.republishPromise) {
@@ -548,7 +552,12 @@ export default class LocalParticipant extends Participant {
 					for (const localTrack of localTracks) {
 						this.log.info('publishing track', getLogContextFromTrack(localTrack));
 
-						publishPromises.push(this.publishTrack(localTrack, publishOptions));
+						publishPromises.push(
+							this.publishTrack(
+								localTrack,
+								audioPublishOptions && isAudioTrack(localTrack) ? audioPublishOptions : publishOptions,
+							),
+						);
 					}
 					const publishedTracks = await Promise.all(publishPromises);
 
@@ -1421,29 +1430,30 @@ export default class LocalParticipant extends Participant {
 		const trackSender = track.sender;
 		track.sender = undefined;
 		if (this.engine.pcManager && this.engine.pcManager.currentState < PCTransportState.FAILED && trackSender) {
+			const publisher = this.engine.pcManager.publisher;
 			try {
-				for (const transceiver of this.engine.pcManager.publisher.getTransceivers()) {
-					if (transceiver.sender === trackSender) {
-						transceiver.direction = 'inactive';
-						negotiationNeeded = true;
-					}
-				}
 				try {
 					negotiationNeeded = this.engine.removeTrack(trackSender);
 				} catch (e) {
 					this.log.warn(e);
 					negotiationNeeded = true;
 				}
+				if (stopTransceiversForSender(publisher.getTransceivers(), trackSender)) {
+					negotiationNeeded = true;
+				}
 
 				if (isLocalVideoTrack(track)) {
 					for (const [, trackInfo] of track.simulcastCodecs) {
 						if (trackInfo.sender) {
 							try {
-								negotiationNeeded = this.engine.removeTrack(trackInfo.sender);
+								negotiationNeeded = this.engine.removeTrack(trackInfo.sender) || negotiationNeeded;
 							} catch (e) {
 								this.log.warn(e);
 								negotiationNeeded = true;
 							}
+							if (stopTransceiversForSender(publisher.getTransceivers(), trackInfo.sender)) {
+								negotiationNeeded = true;
+							}
 							trackInfo.sender = undefined;
 						}
 					}

fluxer_app/pkgs/livekit-client/src/room/utils.test.ts

@@ -3,7 +3,14 @@
 // SPDX-License-Identifier: Apache-2.0
 import {afterEach, beforeEach, describe, expect, it} from 'vitest';
 import {screenCaptureToDisplayMediaStreamOptions} from './track/utils.ts';
-import {selectPreferredVideoCodec, supportsAV1, supportsH265, supportsVideoCodec, supportsVP9} from './utils.ts';
+import {
+	selectPreferredVideoCodec,
+	stopTransceiversForSender,
+	supportsAV1,
+	supportsH265,
+	supportsVideoCodec,
+	supportsVP9,
+} from './utils.ts';
 
 const originalNavigator = globalThis.navigator;
 const originalSender = globalThis.RTCRtpSender;
@@ -91,3 +98,46 @@ describe('screenCaptureToDisplayMediaStreamOptions', () => {
 		expect(options.systemAudio).toBe('exclude');
 	});
 });
+
+describe('stopTransceiversForSender', () => {
+	type FakeTransceiver = {sender: RTCRtpSender; direction: string; stopCalls: number; stop?: () => void};
+
+	function fakeTransceiver(sender: RTCRtpSender, canStop = true): FakeTransceiver {
+		const transceiver: FakeTransceiver = {sender, direction: 'sendonly', stopCalls: 0};
+		if (canStop) {
+			transceiver.stop = () => {
+				transceiver.stopCalls += 1;
+				transceiver.direction = 'stopped';
+			};
+		}
+		return transceiver;
+	}
+
+	function asTransceivers(list: Array<FakeTransceiver>): ReadonlyArray<RTCRtpTransceiver> {
+		return list as unknown as ReadonlyArray<RTCRtpTransceiver>;
+	}
+
+	it('stops the transceiver holding the sender so the m-section can be recycled', () => {
+		const sender = {} as RTCRtpSender;
+		const mine = fakeTransceiver(sender);
+		const theirs = fakeTransceiver({} as RTCRtpSender);
+		expect(stopTransceiversForSender(asTransceivers([theirs, mine]), sender)).toBe(true);
+		expect(mine.stopCalls).toBe(1);
+		expect(mine.direction).toBe('stopped');
+		expect(theirs.stopCalls).toBe(0);
+		expect(theirs.direction).toBe('sendonly');
+	});
+
+	it('reports no match when the sender is not on the connection', () => {
+		const mine = fakeTransceiver({} as RTCRtpSender);
+		expect(stopTransceiversForSender(asTransceivers([mine]), {} as RTCRtpSender)).toBe(false);
+		expect(mine.stopCalls).toBe(0);
+	});
+
+	it('falls back to inactive where stop is unavailable', () => {
+		const sender = {} as RTCRtpSender;
+		const mine = fakeTransceiver(sender, false);
+		expect(stopTransceiversForSender(asTransceivers([mine]), sender)).toBe(true);
+		expect(mine.direction).toBe('inactive');
+	});
+});

fluxer_app/pkgs/livekit-client/src/room/utils.ts

@@ -158,6 +158,25 @@ export function negotiateDependencyDescriptor(transceiver: RTCRtpTransceiver): b
 	}
 }
 
+export function stopTransceiversForSender(
+	transceivers: ReadonlyArray<RTCRtpTransceiver>,
+	sender: RTCRtpSender,
+): boolean {
+	let matched = false;
+	for (const transceiver of transceivers) {
+		if (transceiver.sender !== sender) {
+			continue;
+		}
+		matched = true;
+		if (typeof transceiver.stop === 'function') {
+			transceiver.stop();
+		} else {
+			transceiver.direction = 'inactive';
+		}
+	}
+	return matched;
+}
+
 export function isSVCSimulcast(codec?: string, options?: {simulcast?: boolean; scalabilityMode?: string}): boolean {
 	return isSVCCodec(codec) && !!options?.simulcast && !!options.scalabilityMode?.startsWith('L1T');
 }