fix(voice): rank h264 baseline first and gate opus stereo (#2596)

fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts

@@ -3,12 +3,43 @@
 // SPDX-License-Identifier: Apache-2.0
 import type {MediaDescription} from 'sdp-transform';
 import {describe, expect, it} from 'vitest';
-import {appendStartBitrateToFmtp} from './PCTransport.ts';
+import type {TrackBitrateInfo} from './PCTransport.ts';
+import {appendStartBitrateToFmtp, collectStereoMids, ensureAudioNackAndStereo, ensureOpusFmtp} from './PCTransport.ts';
 
 function mediaWithFmtp(entries: Array<{payload: number; config: string}>): MediaDescription {
 	return {fmtp: entries} as unknown as MediaDescription;
 }
 
+function opusMedia(config: string, mid = '0'): MediaDescription {
+	return {
+		type: 'audio',
+		mid,
+		port: 9,
+		protocol: 'UDP/TLS/RTP/SAVPF',
+		rtp: [{payload: 109, codec: 'opus', rate: 48000, encoding: 2}],
+		fmtp: [{payload: 109, config}],
+	} as unknown as MediaDescription;
+}
+
+function offerMedia(mid: string, trackId: string): MediaDescription {
+	const media = opusMedia('useinbandfec=1', mid);
+	media.msid = `- ${trackId}`;
+	return media;
+}
+
+function audioBitrateInfo(mid: string | null, trackId: string, stereo: boolean): TrackBitrateInfo {
+	return {
+		transceiver: {mid, sender: {track: {id: trackId}}} as unknown as RTCRtpTransceiver,
+		codec: 'opus',
+		maxbr: 320,
+		stereo,
+	};
+}
+
+function opusConfig(media: MediaDescription): string {
+	return media.fmtp.find((fmtp) => fmtp.payload === 109)?.config ?? '';
+}
+
 describe('appendStartBitrateToFmtp', () => {
 	it('appends the start bitrate to a non-SVC codec fmtp line', () => {
 		const media = mediaWithFmtp([
@@ -44,3 +75,72 @@ describe('appendStartBitrateToFmtp', () => {
 		expect(media.fmtp[0]?.config).toBe('profile-level-id=42e01f');
 	});
 });
+
+describe('ensureOpusFmtp', () => {
+	it('does not force stereo on a mono publication', () => {
+		const media = opusMedia('maxplaybackrate=48000;stereo=0;useinbandfec=1');
+		ensureOpusFmtp(media, 48000, false);
+		const config = opusConfig(media);
+		expect(config).toContain('minptime=10');
+		expect(config).toContain('useinbandfec=1');
+		expect(config).toContain('usedtx=0');
+		expect(config).toContain('maxaveragebitrate=48000');
+		expect(config).not.toContain('stereo=1');
+	});
+
+	it('keeps stereo for a stereo publication', () => {
+		const media = opusMedia('maxplaybackrate=48000;useinbandfec=1');
+		ensureOpusFmtp(media, 320000, true);
+		const config = opusConfig(media);
+		expect(config).toContain('stereo=1');
+		expect(config).toContain('sprop-stereo=1');
+		expect(config).toContain('maxaveragebitrate=320000');
+	});
+
+	it('preserves a stereo parameter the server negotiated', () => {
+		const media = opusMedia('minptime=10;stereo=1');
+		ensureOpusFmtp(media, 48000, false);
+		expect(opusConfig(media)).toContain('stereo=1');
+	});
+});
+
+describe('ensureAudioNackAndStereo', () => {
+	it('only stamps stereo on the listed mids', () => {
+		const mono = opusMedia('useinbandfec=1', '0');
+		ensureAudioNackAndStereo(mono as never, ['1'], []);
+		expect(opusConfig(mono)).not.toContain('stereo=1');
+
+		const stereo = opusMedia('useinbandfec=1', '1');
+		ensureAudioNackAndStereo(stereo as never, ['1'], []);
+		expect(opusConfig(stereo)).toContain('stereo=1');
+		expect(opusConfig(stereo)).toContain('sprop-stereo=1');
+	});
+});
+
+describe('collectStereoMids', () => {
+	it('matches the offer media section by msid before the transceiver has a mid', () => {
+		const media = [offerMedia('0', 'mic-track'), offerMedia('1', 'screenshare-track')];
+		expect(collectStereoMids([audioBitrateInfo(null, 'screenshare-track', true)], media)).toEqual(['1']);
+	});
+
+	it('stamps stereo on the first offer for a new stereo publication', () => {
+		const media = [offerMedia('0', 'mic-track'), offerMedia('1', 'screenshare-track')];
+		const stereoMids = collectStereoMids([audioBitrateInfo(null, 'screenshare-track', true)], media);
+		for (const m of media) {
+			ensureAudioNackAndStereo(m as never, stereoMids, []);
+		}
+		expect(opusConfig(media[0]!)).not.toContain('stereo=1');
+		expect(opusConfig(media[1]!)).toContain('stereo=1');
+		expect(opusConfig(media[1]!)).toContain('sprop-stereo=1');
+	});
+
+	it('uses the assigned mid once renegotiation has one', () => {
+		const media = [offerMedia('0', 'mic-track'), offerMedia('1', 'screenshare-track')];
+		expect(collectStereoMids([audioBitrateInfo('1', 'screenshare-track', true)], media)).toEqual(['1']);
+	});
+
+	it('leaves mono publications out', () => {
+		const media = [offerMedia('0', 'mic-track')];
+		expect(collectStereoMids([audioBitrateInfo(null, 'mic-track', false)], media)).toEqual([]);
+	});
+});

fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts

@@ -11,11 +11,12 @@ import {NegotiationError, UnexpectedConnectionState} from './errors.ts';
 import type {LoggerOptions} from './types.ts';
 import {ddExtensionURI, isFireFox, isSafari, isSVCCodec} from './utils.ts';
 
-interface TrackBitrateInfo {
+export interface TrackBitrateInfo {
 	cid?: string;
 	transceiver?: RTCRtpTransceiver;
 	codec: string;
 	maxbr: number;
+	stereo?: boolean;
 }
 
 const startBitrateFraction = 0.7;
@@ -25,8 +26,6 @@ const requiredOpusFmtpParameters = {
 	minptime: '10',
 	useinbandfec: '1',
 	usedtx: '0',
-	stereo: '1',
-	'sprop-stereo': '1',
 };
 const debounceInterval = 20;
 export const PCEvents = {
@@ -156,20 +155,18 @@ export default class PCTransport extends EventEmitter {
 			sdpParsed.media.forEach((media) => {
 				const mid = getMidString(media.mid!);
 				if (media.type === 'audio') {
-					ensureOpusFmtp(media);
-					this.trackBitrates.some((trackbr): boolean => {
-						if (!trackbr.transceiver || mid !== trackbr.transceiver.mid) {
-							return false;
-						}
-						const codecPayload = getCodecPayload(media, trackbr.codec);
-						if (codecPayload === 0) {
-							return true;
-						}
-						if (trackbr.codec.toLowerCase() === 'opus') {
-							ensureOpusFmtp(media, trackbr.maxbr > 0 ? trackbr.maxbr * 1000 : opusMaxAverageBitrateBps);
-						}
-						return true;
-					});
+					const trackbr = this.trackBitrates.find(
+						(br) => br.transceiver !== undefined && mid === br.transceiver.mid && br.codec.toLowerCase() === 'opus',
+					);
+					if (trackbr && getCodecPayload(media, trackbr.codec) !== 0) {
+						ensureOpusFmtp(
+							media,
+							trackbr.maxbr > 0 ? trackbr.maxbr * 1000 : opusMaxAverageBitrateBps,
+							trackbr.stereo === true,
+						);
+					} else {
+						ensureOpusFmtp(media);
+					}
 				}
 			});
 			mungedSDP = write(sdpParsed);
@@ -240,10 +237,11 @@ export default class PCTransport extends EventEmitter {
 			const offer = await this.pc.createOffer(options);
 			this.log.debug('original offer', {sdp: offer.sdp, ...this.logContext});
 			const sdpParsed = parse(offer.sdp ?? '');
+			const stereoMids = collectStereoMids(this.trackBitrates, sdpParsed.media);
 			sdpParsed.media.forEach((media) => {
 				ensureIPAddrMatchVersion(media);
 				if (media.type === 'audio') {
-					ensureAudioNackAndStereo(media, ['all'], []);
+					ensureAudioNackAndStereo(media, stereoMids, []);
 				} else if (media.type === 'video') {
 					this.trackBitrates.some((trackbr): boolean => {
 						if (!media.msid || !trackbr.cid || !media.msid.includes(trackbr.cid)) {
@@ -587,7 +585,11 @@ function ensureAudioRedFmtp(media: MediaDescription, opusPayload: number): void
 	}
 }
 
-function ensureOpusFmtp(media: MediaDescription, maxAverageBitrateBps: number = opusMaxAverageBitrateBps): number {
+export function ensureOpusFmtp(
+	media: MediaDescription,
+	maxAverageBitrateBps: number = opusMaxAverageBitrateBps,
+	stereo = false,
+): number {
 	const opusPayload = getCodecPayload(media, 'opus');
 	if (opusPayload <= 0) return 0;
 	media.ptime = opusPacketTimeMs;
@@ -596,6 +598,10 @@ function ensureOpusFmtp(media: MediaDescription, maxAverageBitrateBps: number =
 	for (const [key, value] of Object.entries(requiredOpusFmtpParameters)) {
 		config = setFmtpParameter(config, key, value);
 	}
+	if (stereo) {
+		config = setFmtpParameter(config, 'stereo', '1');
+		config = setFmtpParameter(config, 'sprop-stereo', '1');
+	}
 	if (maxAverageBitrateBps > 0) {
 		config = setFmtpParameter(config, 'maxaveragebitrate', String(maxAverageBitrateBps));
 	}
@@ -604,18 +610,18 @@ function ensureOpusFmtp(media: MediaDescription, maxAverageBitrateBps: number =
 	return opusPayload;
 }
 
-function ensureAudioNackAndStereo(
+export function ensureAudioNackAndStereo(
 	media: {
 		type: string;
 		port: number;
 		protocol: string;
 		payloads?: string | undefined;
 	} & MediaDescription,
-	_stereoMids: Array<string>,
+	stereoMids: Array<string>,
 	nackMids: Array<string>,
 ) {
 	const mid = getMidString(media.mid!);
-	const opusPayload = ensureOpusFmtp(media);
+	const opusPayload = ensureOpusFmtp(media, opusMaxAverageBitrateBps, stereoMids.includes(mid));
 	if (opusPayload > 0) {
 		if (!media.rtcpFb) {
 			media.rtcpFb = [];
@@ -629,6 +635,32 @@ function ensureAudioNackAndStereo(
 	}
 }
 
+export function collectStereoMids(
+	trackBitrates: Array<TrackBitrateInfo>,
+	media: Array<MediaDescription>,
+): Array<string> {
+	const stereoMids: Array<string> = [];
+	for (const trackbr of trackBitrates) {
+		if (trackbr.stereo !== true || !trackbr.transceiver) {
+			continue;
+		}
+		if (trackbr.transceiver.mid) {
+			stereoMids.push(getMidString(trackbr.transceiver.mid));
+			continue;
+		}
+		const trackId = trackbr.transceiver.sender.track?.id;
+		if (trackId === undefined) {
+			continue;
+		}
+		for (const m of media) {
+			if (m.type === 'audio' && m.mid !== undefined && m.msid?.includes(trackId)) {
+				stereoMids.push(getMidString(m.mid));
+			}
+		}
+	}
+	return stereoMids;
+}
+
 function extractStereoAndNackAudioFromOffer(offer: RTCSessionDescriptionInit): {
 	stereoMids: Array<string>;
 	nackMids: Array<string>;

fluxer_app/pkgs/livekit-client/src/room/RTCEngine.test.ts

@@ -27,7 +27,59 @@ describe('selectPublisherCodecPreferences', () => {
 		const highProfile = codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f');
 		const rtx = codec('video/rtx');
 		const preferences = selectPublisherCodecPreferences('h264', [openH264, rtx, externalBaseline, highProfile]);
-		expect(preferences).toEqual([externalBaseline, highProfile, openH264, rtx]);
+		expect(preferences).toEqual([externalBaseline, openH264, highProfile, rtx]);
+	});
+
+	it('ranks Constrained Baseline above Main, High and Constrained High', () => {
+		const constrainedBaseline = codec(
+			'video/H264',
+			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
+		);
+		const mainProfile = codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f');
+		const highProfileLevel31 = codec(
+			'video/H264',
+			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f',
+		);
+		const highProfileLevel51 = codec(
+			'video/H264',
+			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640033',
+		);
+		const constrainedHigh = codec(
+			'video/H264',
+			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c1f',
+		);
+		const preferences = selectPublisherCodecPreferences('h264', [
+			mainProfile,
+			highProfileLevel31,
+			highProfileLevel51,
+			constrainedHigh,
+			constrainedBaseline,
+		]);
+		expect(preferences).toEqual([
+			constrainedBaseline,
+			mainProfile,
+			highProfileLevel31,
+			highProfileLevel51,
+			constrainedHigh,
+		]);
+	});
+
+	it('keeps Constrained Baseline packetization-mode=1 ahead of Constrained Baseline packetization-mode=0 and High', () => {
+		const constrainedBaselineMode0 = codec(
+			'video/H264',
+			'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f',
+		);
+		const constrainedBaselineMode1 = codec(
+			'video/H264',
+			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
+		);
+		const highProfile = codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640033');
+		const preferences = selectPublisherCodecPreferences('h264', [
+			highProfile,
+			constrainedBaselineMode0,
+			constrainedBaselineMode1,
+		]);
+		expect(preferences).toEqual([constrainedBaselineMode1, constrainedBaselineMode0, highProfile]);
 	});
 
 	it('keeps non-H.264 codecs in browser capability order and appends RTX', () => {

fluxer_app/pkgs/livekit-client/src/room/RTCEngine.ts

@@ -96,7 +96,7 @@ const videoCodecMimeTypes: Record<VideoCodec, Array<string>> = {
 	vp8: ['video/vp8'],
 };
 const h264OpenH264ProfileLevelId = '42e01f';
-const h264PreferredHardwareProfileLevelIds = new Set(['42001f', '4d001f', '64001f']);
+const h264PreferredHardwareProfileLevelIds = new Set(['42001f']);
 type RtpCodecCapability = RTCRtpCapabilities['codecs'][number] & {sdpFmtpLine?: string};
 
 enum PCState {
@@ -1656,8 +1656,8 @@ function getH264PublisherCodecScore(codec: RtpCodecCapability): number {
 	const packetizationMode = getFmtpParameter(codec.sdpFmtpLine, 'packetization-mode');
 	const packetizationScore = packetizationMode === '1' ? 0 : 1;
 	if (profileLevelId && h264PreferredHardwareProfileLevelIds.has(profileLevelId)) return packetizationScore;
-	if (profileLevelId && profileLevelId !== h264OpenH264ProfileLevelId) return 10 + packetizationScore;
-	if (profileLevelId === h264OpenH264ProfileLevelId) return 20 + packetizationScore;
+	if (profileLevelId === h264OpenH264ProfileLevelId) return 10 + packetizationScore;
+	if (profileLevelId) return 20 + packetizationScore;
 	return 30 + packetizationScore;
 }
 

fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts

@@ -1040,6 +1040,7 @@ export default class LocalParticipant extends Participant {
 							transceiver: trackTransceiver,
 							codec: 'opus',
 							maxbr: encodings[0]?.maxBitrate ? encodings[0].maxBitrate / 1000 : 0,
+							stereo: isStereo,
 						});
 					}
 				} else if (track.codec) {