fix(voice): start bitrate for non-SVC screen share codecs (#2434)

fluxer_app/pkgs/livekit-client/VENDORING.md

@@ -101,6 +101,21 @@ source edits in this package.
     sender track back, including `LocalVideoTrack`'s secondary simulcast senders,
     and aggregate every cleanup failure instead of discarding it.
 
+13. **Start bitrate for every video codec** (`src/room/PCTransport.ts`,
+    `src/room/participant/LocalParticipant.ts`, `src/room/participant/publishUtils.ts`)
+    `x-google-start-bitrate` was reachable only by AV1 and VP9, gated twice: the
+    publish path registered a track bitrate only for SVC codecs, and the offer
+    munging returned early for everything else. H264, H265 and VP8 therefore
+    opened at the Chromium default and had to ramp, which showed up as a 3000 kbps
+    screen share encoding at 346 kbps twenty seconds in. The bitrate is now
+    registered for every video codec from the highest encoding
+    (`maxEncodingBitrate()`, so a simulcast ladder contributes its top layer), and
+    the offer munging applies the start bitrate whenever a max bitrate is known.
+    The dependency descriptor extension stays SVC-only.
+    `appendStartBitrateToFmtp()` holds the fmtp edit so it can be tested, and
+    `setTrackCodecBitrate()` now replaces an entry for the same cid or transceiver
+    instead of appending, since `trackBitrates` is never cleared.
+
 ## Updating from upstream
 
 1. Check the upstream changelog for the target version.

fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts

@@ -0,0 +1,46 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type {MediaDescription} from 'sdp-transform';
+import {describe, expect, it} from 'vitest';
+import {appendStartBitrateToFmtp} from './PCTransport.ts';
+
+function mediaWithFmtp(entries: Array<{payload: number; config: string}>): MediaDescription {
+	return {fmtp: entries} as unknown as MediaDescription;
+}
+
+describe('appendStartBitrateToFmtp', () => {
+	it('appends the start bitrate to a non-SVC codec fmtp line', () => {
+		const media = mediaWithFmtp([
+			{payload: 96, config: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f'},
+		]);
+		appendStartBitrateToFmtp(media, 96, 2100);
+		expect(media.fmtp[0]?.config).toBe(
+			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f;x-google-start-bitrate=2100',
+		);
+	});
+
+	it('only touches the fmtp line for the matching payload', () => {
+		const media = mediaWithFmtp([
+			{payload: 96, config: 'profile-level-id=42e01f'},
+			{payload: 98, config: 'profile-id=0'},
+		]);
+		appendStartBitrateToFmtp(media, 98, 1400);
+		expect(media.fmtp[0]?.config).toBe('profile-level-id=42e01f');
+		expect(media.fmtp[1]?.config).toBe('profile-id=0;x-google-start-bitrate=1400');
+	});
+
+	it('never appends a second start bitrate', () => {
+		const media = mediaWithFmtp([{payload: 96, config: 'profile-level-id=42e01f;x-google-start-bitrate=900'}]);
+		appendStartBitrateToFmtp(media, 96, 2100);
+		expect(media.fmtp[0]?.config).toBe('profile-level-id=42e01f;x-google-start-bitrate=900');
+	});
+
+	it('does nothing without a usable bitrate or a matching payload', () => {
+		const media = mediaWithFmtp([{payload: 96, config: 'profile-level-id=42e01f'}]);
+		appendStartBitrateToFmtp(media, 96, 0);
+		appendStartBitrateToFmtp(media, 96, -1);
+		appendStartBitrateToFmtp(media, 111, 2100);
+		expect(media.fmtp[0]?.config).toBe('profile-level-id=42e01f');
+	});
+});

fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts

@@ -18,7 +18,7 @@ interface TrackBitrateInfo {
 	maxbr: number;
 }
 
-const startBitrateForSVC = 0.7;
+const startBitrateFraction = 0.7;
 const opusMaxAverageBitrateBps = 510000;
 const opusPacketTimeMs = 10;
 const requiredOpusFmtpParameters = {
@@ -35,6 +35,21 @@ export const PCEvents = {
 	RTPVideoPayloadTypes: 'rtpVideoPayloadTypes',
 } as const;
 
+export function appendStartBitrateToFmtp(media: MediaDescription, codecPayload: number, startBitrate: number): void {
+	if (startBitrate <= 0) {
+		return;
+	}
+	for (const fmtp of media.fmtp) {
+		if (fmtp.payload !== codecPayload) {
+			continue;
+		}
+		if (!fmtp.config.includes('x-google-start-bitrate')) {
+			fmtp.config += `;x-google-start-bitrate=${startBitrate}`;
+		}
+		return;
+	}
+}
+
 export default class PCTransport extends EventEmitter {
 	private _pc: RTCPeerConnection | null;
 
@@ -248,18 +263,10 @@ export default class PCTransport extends EventEmitter {
 						if (isSVCCodec(trackbr.codec) && !isSafari()) {
 							this.ensureVideoDDExtensionForSVC(media, sdpParsed);
 						}
-						if (!isSVCCodec(trackbr.codec)) {
+						if (trackbr.maxbr <= 0) {
 							return true;
 						}
-						const startBitrate = Math.round(trackbr.maxbr * startBitrateForSVC);
+						appendStartBitrateToFmtp(media, codecPayload, Math.round(trackbr.maxbr * startBitrateFraction));
-						for (const fmtp of media.fmtp) {
-							if (fmtp.payload === codecPayload) {
-								if (!fmtp.config.includes('x-google-start-bitrate')) {
-									fmtp.config += `;x-google-start-bitrate=${startBitrate}`;
-								}
-								break;
-							}
-						}
 						return true;
 					});
 				}
@@ -336,7 +343,16 @@ export default class PCTransport extends EventEmitter {
 	}
 
 	setTrackCodecBitrate(info: TrackBitrateInfo) {
+		const existing = this.trackBitrates.findIndex(
+			(trackbr) =>
+				(info.cid !== undefined && trackbr.cid === info.cid) ||
+				(info.transceiver !== undefined && trackbr.transceiver === info.transceiver),
+		);
+		if (existing === -1) {
 			this.trackBitrates.push(info);
+			return;
+		}
+		this.trackBitrates[existing] = info;
 	}
 
 	setConfiguration(rtcConfig: RTCConfiguration) {

fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts

@@ -97,7 +97,12 @@ import {
 import Participant from './Participant.ts';
 import type {ParticipantTrackPermission} from './ParticipantTrackPermission.ts';
 import {trackPermissionToProto} from './ParticipantTrackPermission.ts';
-import {computeTrackBackupEncodings, computeVideoEncodings, getDefaultDegradationPreference} from './publishUtils.ts';
+import {
+	computeTrackBackupEncodings,
+	computeVideoEncodings,
+	getDefaultDegradationPreference,
+	maxEncodingBitrate,
+} from './publishUtils.ts';
 import type RemoteParticipant from './RemoteParticipant.ts';
 
 type PendingSignalRequestValues = {
@@ -1037,14 +1042,17 @@ export default class LocalParticipant extends Participant {
 							maxbr: encodings[0]?.maxBitrate ? encodings[0].maxBitrate / 1000 : 0,
 						});
 					}
-				} else if (track.codec && isSVCCodec(track.codec) && encodings[0]?.maxBitrate) {
+				} else if (track.codec) {
+					const maxBitrate = maxEncodingBitrate(encodings);
+					if (maxBitrate > 0) {
 						this.engine.pcManager.publisher.setTrackCodecBitrate({
 							cid: req.cid,
 							codec: track.codec,
-						maxbr: encodings[0].maxBitrate / 1000,
+							maxbr: maxBitrate / 1000,
 						});
 					}
 				}
+			}
 
 			await this.engine.negotiate();
 		};

fluxer_app/pkgs/livekit-client/src/room/participant/publishUtils.maxEncodingBitrate.test.ts

@@ -0,0 +1,21 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import {describe, expect, it} from 'vitest';
+import {maxEncodingBitrate} from './publishUtils.ts';
+
+describe('maxEncodingBitrate', () => {
+	it('returns the highest layer bitrate for a simulcast ladder', () => {
+		expect(maxEncodingBitrate([{maxBitrate: 150_000}, {maxBitrate: 500_000}, {maxBitrate: 1_700_000}])).toBe(1_700_000);
+	});
+
+	it('returns the single encoding bitrate for a non-simulcast publish', () => {
+		expect(maxEncodingBitrate([{maxBitrate: 3_000_000}])).toBe(3_000_000);
+	});
+
+	it('returns zero when no encoding declares a bitrate', () => {
+		expect(maxEncodingBitrate([{}])).toBe(0);
+		expect(maxEncodingBitrate([])).toBe(0);
+		expect(maxEncodingBitrate(undefined)).toBe(0);
+	});
+});

fluxer_app/pkgs/livekit-client/src/room/participant/publishUtils.ts

@@ -370,6 +370,19 @@ export class ScalabilityMode {
 	}
 }
 
+export function maxEncodingBitrate(encodings: Array<RTCRtpEncodingParameters> | undefined): number {
+	if (!encodings) {
+		return 0;
+	}
+	let maxBitrate = 0;
+	for (const encoding of encodings) {
+		if (typeof encoding.maxBitrate === 'number' && encoding.maxBitrate > maxBitrate) {
+			maxBitrate = encoding.maxBitrate;
+		}
+	}
+	return maxBitrate;
+}
+
 export function getDefaultDegradationPreference(track: LocalVideoTrack): RTCDegradationPreference {
 	if (track.source === Track.Source.ScreenShare) return 'maintain-resolution';
 	if (track.constraints.height && unwrapConstraint(track.constraints.height) >= 1080) return 'maintain-resolution';