feat(voice): rework screen share delivery behind an experiment (#2859)

fluxer_app/pkgs/livekit-client/src/options.ts

@@ -52,6 +52,8 @@ export interface InternalRoomOptions {
 
 	singlePeerConnection: boolean;
 	subscriberVideoCodecExclusions?: Array<VideoCodec>;
+	screenShareDelivery?: boolean;
+	h264HardwareProfiles?: ReadonlySet<string>;
 	dataStream?: RoomDataStreamOptions;
 }
 export interface RoomDataStreamOptions {

fluxer_app/pkgs/livekit-client/src/room/PCTransport.test.ts

@@ -55,44 +55,110 @@ function opusConfig(media: MediaDescription): string {
 }
 
 describe('applyVideoStartBitrate', () => {
-	it('adds a start bitrate to a non-SVC codec section', () => {
-		const media = videoMedia('camera-track', [
-			{payload: 96, config: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f'},
-		]);
-		expect(applyVideoStartBitrate(media, 'camera-track', 'H264', 1000)).toBe(96);
-		expect(media.fmtp[0]?.config).toBe(
-			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f;x-google-start-bitrate=900',
+	function multiPayloadScreenMedia(): MediaDescription {
+		return videoMedia(
+			'screen-track',
+			[
+				{payload: 116, config: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f'},
+				{payload: 102, config: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f'},
+				{payload: 108, config: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f'},
+			],
+			[
+				{payload: 116, codec: 'H264'},
+				{payload: 117, codec: 'rtx'},
+				{payload: 102, codec: 'H264'},
+				{payload: 108, codec: 'H264'},
+			],
 		);
+	}
+
+	it('adds a start bitrate to a non-SVC codec section', () => {
+		for (const screenShareDelivery of [false, true]) {
+			const media = videoMedia('camera-track', [
+				{payload: 96, config: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f'},
+			]);
+			expect(applyVideoStartBitrate(media, 'camera-track', 'H264', 1000, false, screenShareDelivery)).toBe(96);
+			expect(media.fmtp[0]?.config).toBe(
+				'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f;x-google-start-bitrate=900',
+			);
+		}
 	});
 
-	it('caps camera start bitrates but not screen share start bitrates', () => {
+	it('caps camera start bitrates but not screen share start bitrates while screen share delivery is off', () => {
 		const camera = videoMedia('camera-track', [{payload: 96, config: 'profile-level-id=42e01f'}]);
 		applyVideoStartBitrate(camera, 'camera-track', 'H264', 3000);
 		expect(camera.fmtp[0]?.config).toBe('profile-level-id=42e01f;x-google-start-bitrate=1000');
 
 		const screen = videoMedia('screen-track', [{payload: 96, config: 'profile-level-id=42e01f'}]);
-		applyVideoStartBitrate(screen, 'screen-track', 'H264', 3000, true);
-		expect(screen.fmtp[0]?.config).toBe('profile-level-id=42e01f;x-google-start-bitrate=2700');
+		applyVideoStartBitrate(screen, 'screen-track', 'H264', 6000, true);
+		expect(screen.fmtp[0]?.config).toBe('profile-level-id=42e01f;x-google-start-bitrate=5400');
+	});
+
+	it('caps camera and screen share start bitrates at their own ceilings', () => {
+		const camera = videoMedia('camera-track', [{payload: 96, config: 'profile-level-id=42e01f'}]);
+		applyVideoStartBitrate(camera, 'camera-track', 'H264', 3000, false, true);
+		expect(camera.fmtp[0]?.config).toBe('profile-level-id=42e01f;x-google-start-bitrate=1000');
+
+		const screen = videoMedia('screen-track', [{payload: 96, config: 'profile-level-id=42e01f'}]);
+		applyVideoStartBitrate(screen, 'screen-track', 'H264', 6000, true, true);
+		expect(screen.fmtp[0]?.config).toBe('profile-level-id=42e01f;x-google-start-bitrate=1500');
+	});
+
+	it('leaves a small screen share start bitrate on the floor while screen share delivery is off', () => {
+		const screen = videoMedia('screen-track', [{payload: 96, config: 'profile-level-id=42e01f'}]);
+		applyVideoStartBitrate(screen, 'screen-track', 'H264', 300, true);
+		expect(screen.fmtp[0]?.config).toBe('profile-level-id=42e01f;x-google-start-bitrate=270');
+	});
+
+	it('keeps a screen share start bitrate above the frame dropper cliff', () => {
+		const screen = videoMedia('screen-track', [{payload: 96, config: 'profile-level-id=42e01f'}]);
+		applyVideoStartBitrate(screen, 'screen-track', 'H264', 300, true, true);
+		expect(screen.fmtp[0]?.config).toBe('profile-level-id=42e01f;x-google-start-bitrate=600');
+	});
+
+	it('stamps only the lead payload type while screen share delivery is off', () => {
+		const media = multiPayloadScreenMedia();
+		expect(applyVideoStartBitrate(media, 'screen-track', 'H264', 6000, true)).toBe(116);
+		expect(media.fmtp.find((fmtp) => fmtp.payload === 116)?.config).toBe(
+			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f;x-google-start-bitrate=5400',
+		);
+		expect(media.fmtp.find((fmtp) => fmtp.payload === 102)?.config).toBe(
+			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f',
+		);
+		expect(media.fmtp.find((fmtp) => fmtp.payload === 108)?.config).toBe(
+			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
+		);
+	});
+
+	it('stamps every payload type the codec is offered under, not only the lead one', () => {
+		const media = multiPayloadScreenMedia();
+		expect(applyVideoStartBitrate(media, 'screen-track', 'H264', 6000, true, true)).toBe(116);
+		for (const fmtp of media.fmtp) {
+			expect(fmtp.config).toContain('x-google-start-bitrate=1500');
+		}
+		expect(media.fmtp.some((fmtp) => fmtp.payload === 117)).toBe(false);
 	});
 
 	it('only touches the fmtp line for the matching payload', () => {
-		const media = videoMedia(
-			'screen-track',
-			[
-				{payload: 96, config: 'profile-level-id=42e01f'},
-				{payload: 98, config: 'profile-id=0'},
-			],
-			[
-				{payload: 96, codec: 'H264'},
-				{payload: 98, codec: 'VP9'},
-			],
-		);
-		applyVideoStartBitrate(media, 'screen-track', 'VP9', 1500, true);
-		expect(media.fmtp[0]?.config).toBe('profile-level-id=42e01f');
-		expect(media.fmtp[1]?.config).toBe('profile-id=0;x-google-start-bitrate=1350');
+		for (const screenShareDelivery of [false, true]) {
+			const media = videoMedia(
+				'screen-track',
+				[
+					{payload: 96, config: 'profile-level-id=42e01f'},
+					{payload: 98, config: 'profile-id=0'},
+				],
+				[
+					{payload: 96, codec: 'H264'},
+					{payload: 98, codec: 'VP9'},
+				],
+			);
+			applyVideoStartBitrate(media, 'screen-track', 'VP9', 1500, true, screenShareDelivery);
+			expect(media.fmtp[0]?.config).toBe('profile-level-id=42e01f');
+			expect(media.fmtp[1]?.config).toBe('profile-id=0;x-google-start-bitrate=1350');
+		}
 	});
 
-	it('never appends a second start bitrate', () => {
+	it('never appends a second start bitrate while screen share delivery is off', () => {
 		const media = videoMedia('camera-track', [
 			{payload: 96, config: 'profile-level-id=42e01f;x-google-start-bitrate=900'},
 		]);
@@ -100,11 +166,21 @@ describe('applyVideoStartBitrate', () => {
 		expect(media.fmtp[0]?.config).toBe('profile-level-id=42e01f;x-google-start-bitrate=900');
 	});
 
+	it('replaces a start bitrate an earlier offer wrote instead of keeping it', () => {
+		const media = videoMedia('camera-track', [
+			{payload: 96, config: 'profile-level-id=42e01f;x-google-start-bitrate=900'},
+		]);
+		applyVideoStartBitrate(media, 'camera-track', 'H264', 2000, false, true);
+		expect(media.fmtp[0]?.config).toBe('profile-level-id=42e01f;x-google-start-bitrate=1000');
+	});
+
 	it('leaves other tracks and missing codecs alone', () => {
-		const media = videoMedia('camera-track', [{payload: 96, config: 'profile-level-id=42e01f'}]);
-		expect(applyVideoStartBitrate(media, 'other-track', 'H264', 2000)).toBeUndefined();
-		expect(applyVideoStartBitrate(media, 'camera-track', 'AV1', 2000)).toBe(0);
-		expect(media.fmtp[0]?.config).toBe('profile-level-id=42e01f');
+		for (const screenShareDelivery of [false, true]) {
+			const media = videoMedia('camera-track', [{payload: 96, config: 'profile-level-id=42e01f'}]);
+			expect(applyVideoStartBitrate(media, 'other-track', 'H264', 2000, false, screenShareDelivery)).toBeUndefined();
+			expect(applyVideoStartBitrate(media, 'camera-track', 'AV1', 2000, false, screenShareDelivery)).toBe(0);
+			expect(media.fmtp[0]?.config).toBe('profile-level-id=42e01f');
+		}
 	});
 });
 

fluxer_app/pkgs/livekit-client/src/room/PCTransport.ts

@@ -25,6 +25,12 @@ const startBitrateMultiplier = 0.9;
 
 const maxStartBitrateKbps = 1000;
 
+const maxScreenShareStartBitrateKbps = 1500;
+
+const minScreenShareStartBitrateKbps = 600;
+
+const startBitrateParameter = 'x-google-start-bitrate';
+
 const debounceInterval = 20;
 
 const opusMaxAverageBitrateBps = 510000;
@@ -43,32 +49,49 @@ export function applyVideoStartBitrate(
 	codec: string,
 	maxbr: number,
 	isScreenShare = false,
+	screenShareDelivery = false,
 ): number | undefined {
 	if (!media.msid?.includes(cid)) {
 		return undefined;
 	}
 
-	const codecPayload = media.rtp.find((rtp) => rtp.codec.toUpperCase() === codec.toUpperCase())?.payload ?? 0;
-	if (codecPayload === 0) {
+	const codecPayloads = media.rtp
+		.filter((rtp) => rtp.codec.toUpperCase() === codec.toUpperCase())
+		.map((rtp) => rtp.payload);
+	if (codecPayloads.length === 0 || codecPayloads[0] === 0) {
 		return 0;
 	}
 
 	const calculatedStartBitrate = Math.round(maxbr * startBitrateMultiplier);
-	const startBitrate = isScreenShare ? calculatedStartBitrate : Math.min(calculatedStartBitrate, maxStartBitrateKbps);
-
-	const fmtp = media.fmtp.find((entry) => entry.payload === codecPayload);
-	if (fmtp) {
-		if (!fmtp.config.includes('x-google-start-bitrate')) {
-			fmtp.config += `;x-google-start-bitrate=${startBitrate}`;
-		}
-	} else {
-		media.fmtp.push({
-			payload: codecPayload,
-			config: `x-google-start-bitrate=${startBitrate}`,
-		});
+	let startBitrate = Math.min(calculatedStartBitrate, maxStartBitrateKbps);
+	if (isScreenShare) {
+		startBitrate = screenShareDelivery
+			? Math.max(minScreenShareStartBitrateKbps, Math.min(calculatedStartBitrate, maxScreenShareStartBitrateKbps))
+			: calculatedStartBitrate;
 	}
 
-	return codecPayload;
+	if (!screenShareDelivery) {
+		const codecPayload = codecPayloads[0];
+		const fmtp = media.fmtp.find((entry) => entry.payload === codecPayload);
+		if (fmtp) {
+			if (!fmtp.config.includes(startBitrateParameter)) {
+				fmtp.config += `;${startBitrateParameter}=${startBitrate}`;
+			}
+		} else {
+			media.fmtp.push({
+				payload: codecPayload,
+				config: `${startBitrateParameter}=${startBitrate}`,
+			});
+		}
+		return codecPayload;
+	}
+
+	for (const payload of codecPayloads) {
+		const fmtp = ensureFmtp(media, payload);
+		fmtp.config = setFmtpParameter(fmtp.config, startBitrateParameter, String(startBitrate));
+	}
+
+	return codecPayloads[0];
 }
 
 export const PCEvents = {
@@ -120,6 +143,8 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter
 
 	excludedVideoDecoderMimeTypes: Set<string> = new Set();
 
+	private screenShareDelivery: boolean;
+
 	onOffer?: (offer: RTCSessionDescriptionInit, offerId: number) => void;
 
 	onIceCandidate?: (candidate: RTCIceCandidate) => void;
@@ -136,9 +161,10 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter
 
 	onTrack?: (ev: RTCTrackEvent) => void;
 
-	constructor(config?: RTCConfiguration, loggerOptions: LoggerOptions = {}) {
+	constructor(config?: RTCConfiguration, loggerOptions: LoggerOptions = {}, screenShareDelivery: boolean = false) {
 		super();
 		this.loggerOptions = loggerOptions;
+		this.screenShareDelivery = screenShareDelivery;
 		this.log = getLogger(loggerOptions.loggerName ?? LoggerNames.PCTransport, () => this.logContext);
 		this.iceLog = getLogger(LoggerNames.ICE, () => this.logContext);
 		this.config = config;
@@ -386,6 +412,7 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter
 							trackbr.codec,
 							trackbr.maxbr,
 							trackbr.isScreenShare,
+							this.screenShareDelivery,
 						);
 						if (codecPayload === undefined) {
 							return false;
@@ -444,7 +471,10 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter
 		for (const transceiver of this.getTransceivers()) {
 			if (transceiver.receiver.track?.kind !== 'video') continue;
 			if ((transceiver as {stopped?: boolean}).stopped) continue;
-			if (transceiver.direction !== 'recvonly' && transceiver.direction !== 'sendrecv') continue;
+			const receives = this.screenShareDelivery
+				? transceiver.direction === 'recvonly'
+				: transceiver.direction === 'recvonly' || transceiver.direction === 'sendrecv';
+			if (!receives) continue;
 			if (typeof transceiver.setCodecPreferences !== 'function') continue;
 			try {
 				transceiver.setCodecPreferences(allowed);

fluxer_app/pkgs/livekit-client/src/room/PCTransportManager.ts

@@ -98,6 +98,7 @@ export class PCTransportManager {
 		loggerOptions: LoggerOptions,
 		rtcConfig?: RTCConfiguration,
 		subscriberVideoCodecExclusions?: Array<VideoCodec>,
+		screenShareDelivery: boolean = false,
 	) {
 		this.loggerOptions = loggerOptions;
 		this.log = getLogger(loggerOptions.loggerName ?? LoggerNames.PCManager, () => this.logContext);
@@ -105,15 +106,10 @@ export class PCTransportManager {
 
 		this.isPublisherConnectionRequired = mode !== 'subscriber-primary';
 		this.isSubscriberConnectionRequired = mode === 'subscriber-primary';
-		this.publisher = new PCTransport(rtcConfig, loggerOptions);
+		this.publisher = new PCTransport(rtcConfig, loggerOptions, screenShareDelivery);
 		this._mode = mode;
 		if (mode !== 'publisher-only') {
-			this.subscriber = new PCTransport(rtcConfig, loggerOptions);
-			if (subscriberVideoCodecExclusions?.length) {
-				for (const codec of subscriberVideoCodecExclusions) {
-					this.subscriber.excludedVideoDecoderMimeTypes.add(`video/${codec}`);
-				}
-			}
+			this.subscriber = new PCTransport(rtcConfig, loggerOptions, screenShareDelivery);
 			this.subscriber.onConnectionStateChange = this.updateState;
 			this.subscriber.onIceConnectionStateChange = this.updateState;
 			this.subscriber.onSignalingStatechange = this.updateState;
@@ -131,6 +127,13 @@ export class PCTransportManager {
 			};
 		}
 
+		const receivingTransport = screenShareDelivery ? (this.subscriber ?? this.publisher) : this.subscriber;
+		if (receivingTransport) {
+			for (const codec of subscriberVideoCodecExclusions ?? []) {
+				receivingTransport.excludedVideoDecoderMimeTypes.add(`video/${codec}`);
+			}
+		}
+
 		this.publisher.onConnectionStateChange = this.updateState;
 		this.publisher.onIceConnectionStateChange = this.updateState;
 		this.publisher.onSignalingStatechange = this.updateState;

fluxer_app/pkgs/livekit-client/src/room/RTCEngine.test.ts

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 import {describe, expect, it} from 'vitest';
 import type {InternalRoomOptions} from '../options.ts';
+import {EngineEvent} from './events.ts';
 import RTCEngine, {selectPublisherCodecPreferences} from './RTCEngine.ts';
 
 function codec(
@@ -19,20 +20,51 @@ function codec(
 }
 
 describe('selectPublisherCodecPreferences', () => {
-	it('offers the profiles a hardware encoder accepts before Constrained Baseline, which is always software on Windows', () => {
-		const constrainedBaseline = codec(
-			'video/H264',
-			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
-		);
-		const baseline = codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f');
-		const highProfile = codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f');
+	const constrainedBaselineLine = 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f';
+	const baselineLine = 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f';
+	const mainLine = 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f';
+	const highLine = 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f';
+
+	it('keeps Main and Baseline ahead of Constrained Baseline while screen share delivery is off', () => {
+		const constrainedBaseline = codec('video/H264', constrainedBaselineLine);
+		const baseline = codec('video/H264', baselineLine);
+		const highProfile = codec('video/H264', highLine);
 		const rtx = codec('video/rtx');
 		const preferences = selectPublisherCodecPreferences('h264', [constrainedBaseline, rtx, baseline, highProfile]);
 		expect(preferences).toEqual([highProfile, baseline, constrainedBaseline, rtx]);
 	});
 
-	it('offers High first out of the capabilities Chromium reports, so the only hardware profile this server registers wins', () => {
-		const capabilities = [
+	it('offers High first, then the one profile this server always registers, then the ones it registers nowhere', () => {
+		const constrainedBaseline = codec('video/H264', constrainedBaselineLine);
+		const baseline = codec('video/H264', baselineLine);
+		const highProfile = codec('video/H264', highLine);
+		const rtx = codec('video/rtx');
+		const preferences = selectPublisherCodecPreferences(
+			'h264',
+			[constrainedBaseline, rtx, baseline, highProfile],
+			true,
+		);
+		expect(preferences).toEqual([highProfile, constrainedBaseline, baseline, rtx]);
+	});
+
+	it('leads with Main and then Baseline while screen share delivery is off', () => {
+		const constrainedBaseline = codec('video/H264', constrainedBaselineLine);
+		const mainProfile = codec('video/H264', mainLine);
+		const baseline = codec('video/H264', baselineLine);
+		const preferences = selectPublisherCodecPreferences('h264', [mainProfile, baseline, constrainedBaseline]);
+		expect(preferences).toEqual([mainProfile, baseline, constrainedBaseline]);
+	});
+
+	it('never leads with Main or Baseline, which this server registers nowhere and deletes from the answer', () => {
+		const constrainedBaseline = codec('video/H264', constrainedBaselineLine);
+		const mainProfile = codec('video/H264', mainLine);
+		const baseline = codec('video/H264', baselineLine);
+		const preferences = selectPublisherCodecPreferences('h264', [mainProfile, baseline, constrainedBaseline], true);
+		expect(preferences).toEqual([constrainedBaseline, mainProfile, baseline]);
+	});
+
+	function chromiumCapabilities() {
+		return [
 			codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f'),
 			codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f'),
 			codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f'),
@@ -41,7 +73,10 @@ describe('selectPublisherCodecPreferences', () => {
 			codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f'),
 			codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640034'),
 		];
-		const preferences = selectPublisherCodecPreferences('h264', capabilities);
+	}
+
+	it('sorts the capabilities Chromium reports by the old table while screen share delivery is off', () => {
+		const preferences = selectPublisherCodecPreferences('h264', chromiumCapabilities());
 		expect(preferences.map((entry) => entry.sdpFmtpLine)).toEqual([
 			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640034',
 			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f',
@@ -53,16 +88,23 @@ describe('selectPublisherCodecPreferences', () => {
 		]);
 	});
 
-	it('ranks High, Constrained High, Main and Baseline above Constrained Baseline, whatever level each one carries', () => {
-		const constrainedBaseline = codec(
-			'video/H264',
+	it('offers High first out of the capabilities Chromium reports, so the only hardware profile this server registers wins', () => {
+		const preferences = selectPublisherCodecPreferences('h264', chromiumCapabilities(), true);
+		expect(preferences.map((entry) => entry.sdpFmtpLine)).toEqual([
+			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640034',
 			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
-		);
-		const mainProfile = codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f');
-		const highProfileLevel31 = codec(
-			'video/H264',
-			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f',
-		);
+			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f',
+			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f',
+			'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f',
+			'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f',
+			'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f',
+		]);
+	});
+
+	function levelSpread() {
+		const constrainedBaseline = codec('video/H264', constrainedBaselineLine);
+		const mainProfile = codec('video/H264', mainLine);
+		const highProfileLevel31 = codec('video/H264', highLine);
 		const highProfileLevel51 = codec(
 			'video/H264',
 			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640033',
@@ -71,6 +113,11 @@ describe('selectPublisherCodecPreferences', () => {
 			'video/H264',
 			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c1f',
 		);
+		return {constrainedBaseline, mainProfile, highProfileLevel31, highProfileLevel51, constrainedHigh};
+	}
+
+	it('ranks High, Constrained High, Main and Baseline above Constrained Baseline while screen share delivery is off', () => {
+		const {constrainedBaseline, mainProfile, highProfileLevel31, highProfileLevel51, constrainedHigh} = levelSpread();
 		const preferences = selectPublisherCodecPreferences('h264', [
 			mainProfile,
 			highProfileLevel31,
@@ -87,15 +134,28 @@ describe('selectPublisherCodecPreferences', () => {
 		]);
 	});
 
-	it('ranks packetization-mode=1 above packetization-mode=0, which no hardware encoder takes', () => {
+	it('ranks High and Constrained High above Constrained Baseline, whatever level each one reports', () => {
+		const {constrainedBaseline, mainProfile, highProfileLevel31, highProfileLevel51, constrainedHigh} = levelSpread();
+		const preferences = selectPublisherCodecPreferences(
+			'h264',
+			[mainProfile, highProfileLevel31, highProfileLevel51, constrainedHigh, constrainedBaseline],
+			true,
+		);
+		expect(preferences).toEqual([
+			highProfileLevel31,
+			highProfileLevel51,
+			constrainedHigh,
+			constrainedBaseline,
+			mainProfile,
+		]);
+	});
+
+	it('ranks packetization-mode=1 above packetization-mode=0 in both arms, which no hardware encoder takes', () => {
 		const constrainedBaselineMode0 = codec(
 			'video/H264',
 			'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f',
 		);
-		const constrainedBaselineMode1 = codec(
-			'video/H264',
-			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
-		);
+		const constrainedBaselineMode1 = codec('video/H264', constrainedBaselineLine);
 		const highProfileMode0 = codec(
 			'video/H264',
 			'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=640033',
@@ -104,18 +164,10 @@ describe('selectPublisherCodecPreferences', () => {
 			'video/H264',
 			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640033',
 		);
-		const preferences = selectPublisherCodecPreferences('h264', [
-			highProfileMode0,
-			constrainedBaselineMode0,
-			constrainedBaselineMode1,
-			highProfileMode1,
-		]);
-		expect(preferences).toEqual([
-			highProfileMode1,
-			constrainedBaselineMode1,
-			highProfileMode0,
-			constrainedBaselineMode0,
-		]);
+		const capabilities = [highProfileMode0, constrainedBaselineMode0, constrainedBaselineMode1, highProfileMode1];
+		const expected = [highProfileMode1, constrainedBaselineMode1, highProfileMode0, constrainedBaselineMode0];
+		expect(selectPublisherCodecPreferences('h264', capabilities)).toEqual(expected);
+		expect(selectPublisherCodecPreferences('h264', capabilities, true)).toEqual(expected);
 	});
 
 	it('puts the chosen codec first and keeps every other codec in browser capability order', () => {
@@ -123,12 +175,13 @@ describe('selectPublisherCodecPreferences', () => {
 		const vp8 = codec('video/VP8');
 		const rtx = codec('video/rtx');
 		expect(selectPublisherCodecPreferences('vp9', [vp8, rtx, vp9])).toEqual([vp9, vp8, rtx]);
+		expect(selectPublisherCodecPreferences('vp9', [vp8, rtx, vp9], true)).toEqual([vp9, vp8, rtx]);
 	});
 
 	it('keeps the other codecs so a later publication on the same connection can negotiate them', () => {
 		const vp9 = codec('video/VP9');
 		const av1 = codec('video/AV1');
-		const h264 = codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f');
+		const h264 = codec('video/H264', constrainedBaselineLine);
 		const preferences = selectPublisherCodecPreferences('vp9', [av1, h264, vp9]);
 		expect(preferences.map((entry) => entry.mimeType)).toEqual(['video/VP9', 'video/AV1', 'video/H264']);
 	});
@@ -136,17 +189,65 @@ describe('selectPublisherCodecPreferences', () => {
 	it('ranks the H.264 profiles it leaves behind the chosen codec', () => {
 		const vp8 = codec('video/VP8');
 		const highProfile = codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640033');
-		const constrainedBaseline = codec(
-			'video/H264',
-			'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
-		);
-		const preferences = selectPublisherCodecPreferences('vp8', [vp8, highProfile, constrainedBaseline]);
-		expect(preferences).toEqual([vp8, highProfile, constrainedBaseline]);
+		const constrainedBaseline = codec('video/H264', constrainedBaselineLine);
+		const capabilities = [vp8, highProfile, constrainedBaseline];
+		expect(selectPublisherCodecPreferences('vp8', capabilities)).toEqual([vp8, highProfile, constrainedBaseline]);
+		expect(selectPublisherCodecPreferences('vp8', capabilities, true, new Set(['42e0']))).toEqual([
+			vp8,
+			constrainedBaseline,
+			highProfile,
+		]);
 	});
 
 	it('returns nothing when the sender cannot encode the chosen codec', () => {
 		expect(selectPublisherCodecPreferences('av1', [codec('video/VP8'), codec('video/rtx')])).toEqual([]);
 	});
+
+	it('ignores the profiles this host measured while screen share delivery is off', () => {
+		const constrainedBaseline = codec('video/H264', constrainedBaselineLine);
+		const highProfile = codec('video/H264', highLine);
+		const capabilities = [highProfile, constrainedBaseline];
+		expect(selectPublisherCodecPreferences('h264', capabilities, false, new Set(['42e0']))).toEqual([
+			highProfile,
+			constrainedBaseline,
+		]);
+	});
+
+	it('only lets a profile this host encodes in hardware outrank Constrained Baseline', () => {
+		const constrainedBaseline = codec('video/H264', constrainedBaselineLine);
+		const highProfile = codec('video/H264', highLine);
+		const capabilities = [highProfile, constrainedBaseline];
+		expect(selectPublisherCodecPreferences('h264', capabilities, true, new Set(['42e0']))).toEqual([
+			constrainedBaseline,
+			highProfile,
+		]);
+		expect(selectPublisherCodecPreferences('h264', capabilities, true, new Set(['6400', '42e0']))).toEqual([
+			highProfile,
+			constrainedBaseline,
+		]);
+	});
+
+	it('falls back to the static order when nothing was measured or nothing is hardware', () => {
+		const constrainedBaseline = codec('video/H264', constrainedBaselineLine);
+		const highProfile = codec('video/H264', highLine);
+		const capabilities = [constrainedBaseline, highProfile];
+		expect(selectPublisherCodecPreferences('h264', capabilities, true)).toEqual([highProfile, constrainedBaseline]);
+		expect(selectPublisherCodecPreferences('h264', capabilities, true, new Set())).toEqual([
+			highProfile,
+			constrainedBaseline,
+		]);
+	});
+
+	it('keeps packetization-mode=1 ahead of a hardware packetization-mode=0 profile', () => {
+		const constrainedBaselineMode1 = codec('video/H264', constrainedBaselineLine);
+		const highProfileMode0 = codec(
+			'video/H264',
+			'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=64001f',
+		);
+		expect(
+			selectPublisherCodecPreferences('h264', [highProfileMode0, constrainedBaselineMode1], true, new Set(['6400'])),
+		).toEqual([constrainedBaselineMode1, highProfileMode0]);
+	});
 });
 
 describe('publisher data channels before negotiation', () => {
@@ -187,3 +288,66 @@ describe('publisher data channels before negotiation', () => {
 		expect(created).toEqual([]);
 	});
 });
+
+describe('negotiation aborts', () => {
+	function negotiableEngine() {
+		const engine = new RTCEngine({} as InternalRoomOptions);
+		const controllers: Array<AbortController> = [];
+		const internals = engine as unknown as {
+			_isClosed: boolean;
+			pcManager: unknown;
+			dataChannels: {hasPublisherChannels: boolean};
+			pendingNegotiationAborts: Set<() => void>;
+		};
+		internals._isClosed = false;
+		internals.dataChannels = {hasPublisherChannels: true};
+		internals.pcManager = {
+			requirePublisher: () => {},
+			publisher: {
+				off: () => {},
+				once: () => {},
+				getTransceivers: () => [{} as RTCRtpTransceiver],
+			},
+			negotiate: (abortController: AbortController) =>
+				new Promise<void>((_resolve, reject) => {
+					controllers.push(abortController);
+					abortController.signal.addEventListener('abort', () => reject(new Error('negotiation aborted')), {
+						once: true,
+					});
+				}),
+		};
+		return {engine, controllers, pendingAborts: internals.pendingNegotiationAborts};
+	}
+
+	it('keeps one listener pair however many negotiations are in flight, and aborts every one', async () => {
+		const {engine, controllers, pendingAborts} = negotiableEngine();
+		const closingListeners = engine.listenerCount(EngineEvent.Closing);
+		const restartingListeners = engine.listenerCount(EngineEvent.Restarting);
+
+		const pending = Array.from({length: 20}, () => engine.negotiate());
+		expect(engine.listenerCount(EngineEvent.Closing)).toBe(closingListeners);
+		expect(engine.listenerCount(EngineEvent.Restarting)).toBe(restartingListeners);
+		expect(pendingAborts.size).toBe(20);
+
+		engine.emit(EngineEvent.Closing);
+		await expect(Promise.all(pending)).resolves.toHaveLength(20);
+		expect(controllers.every((controller) => controller.signal.aborted)).toBe(true);
+		expect(pendingAborts.size).toBe(0);
+	});
+
+	it('re-arms after a restart so a later negotiation is not aborted on arrival', async () => {
+		const {engine, controllers, pendingAborts} = negotiableEngine();
+		const firstBatch = [engine.negotiate(), engine.negotiate()];
+		engine.emit(EngineEvent.Restarting);
+		await expect(Promise.all(firstBatch)).resolves.toHaveLength(2);
+
+		const afterRestart = engine.negotiate();
+		expect(controllers).toHaveLength(3);
+		expect(controllers[2].signal.aborted).toBe(false);
+		expect(pendingAborts.size).toBe(1);
+
+		engine.emit(EngineEvent.Restarting);
+		await expect(afterRestart).resolves.toBeUndefined();
+		expect(pendingAborts.size).toBe(0);
+	});
+});

fluxer_app/pkgs/livekit-client/src/room/RTCEngine.ts

@@ -129,9 +129,18 @@ const h264ProfileRanks = new Map([
 	['4200', 3],
 	['42e0', 4],
 ]);
+const h264DeliveryProfileRanks = new Map([
+	['6400', 0],
+	['640c', 1],
+	['42e0', 2],
+	['4d00', 3],
+	['4200', 4],
+]);
 const h264UnrankedProfileScore = 5;
 const h264MissingProfileScore = 6;
+const h264NonHardwareProfilePenalty = 8;
 const h264PacketizationMode0Score = 10;
+const h264DeliveryPacketizationMode0Score = 20;
 type RtpCodecCapability = RTCRtpCapabilities['codecs'][number] & {sdpFmtpLine?: string};
 
 enum PCState {
@@ -250,6 +259,8 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
 
 	private transportConnectingSince?: number;
 
+	private pendingNegotiationAborts = new Set<() => void>();
+
 	constructor(private options: InternalRoomOptions) {
 		super();
 		this.log = getLogger(options.loggerName ?? LoggerNames.Engine, () => this.logContext);
@@ -286,6 +297,14 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
 		this.client.onRequestResponse = (response) => this.emit(EngineEvent.SignalRequestResponse, response);
 		this.client.onParticipantUpdate = (updates) => this.emit(EngineEvent.ParticipantUpdate, updates);
 		this.client.onJoined = (joinResponse) => this.emit(EngineEvent.Joined, joinResponse);
+
+		const abortPendingNegotiations = () => {
+			for (const abort of Array.from(this.pendingNegotiationAborts)) {
+				abort();
+			}
+		};
+		this.on(EngineEvent.Closing, abortPendingNegotiations);
+		this.on(EngineEvent.Restarting, abortPendingNegotiations);
 	}
 
 	get logContext() {
@@ -504,6 +523,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
 				this.loggerOptions,
 				rtcConfig,
 				this.options.subscriberVideoCodecExclusions,
+				this.options.screenShareDelivery ?? false,
 			);
 		} else {
 			this.participantSid = joinResponse.participant?.sid;
@@ -517,6 +537,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
 				this.loggerOptions,
 				rtcConfig,
 				this.options.subscriberVideoCodecExclusions,
+				this.options.screenShareDelivery ?? false,
 			);
 		}
 
@@ -1041,7 +1062,12 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
 		if (typeof RTCRtpSender === 'undefined' || typeof RTCRtpSender.getCapabilities !== 'function') return;
 		const capabilities = RTCRtpSender.getCapabilities('video');
 		if (!capabilities) return;
-		const preferences = selectPublisherCodecPreferences(codec, capabilities.codecs);
+		const preferences = selectPublisherCodecPreferences(
+			codec,
+			capabilities.codecs,
+			this.options.screenShareDelivery ?? false,
+			this.options.h264HardwareProfiles,
+		);
 		if (preferences.length === 0) {
 			this.log.warn('sender cannot encode the requested codec, leaving the browser order in place', {
 				...this.logContext,
@@ -1583,8 +1609,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
 			if (this.isClosed) {
 				reject(new NegotiationError('cannot negotiate on closed engine'));
 			}
-			this.on(EngineEvent.Closing, handleClosed);
-			this.on(EngineEvent.Restarting, handleClosed);
+			this.pendingNegotiationAborts.add(handleClosed);
 			this.pcManager.publisher.off(PCEvents.RTPVideoPayloadTypes, this.onRtpMapAvailable);
 			this.pcManager.publisher.once(PCEvents.RTPVideoPayloadTypes, this.onRtpMapAvailable);
 
@@ -1606,8 +1631,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
 					reject(new Error(String(e)));
 				}
 			} finally {
-				this.off(EngineEvent.Closing, handleClosed);
-				this.off(EngineEvent.Restarting, handleClosed);
+				this.pendingNegotiationAborts.delete(handleClosed);
 			}
 		});
 	}
@@ -1817,17 +1841,36 @@ function getFmtpParameter(sdpFmtpLine: string | undefined, key: string): string
 	return null;
 }
 
-function getH264PublisherCodecScore(codec: RtpCodecCapability): number {
+function getH264PublisherCodecScore(
+	codec: RtpCodecCapability,
+	screenShareDelivery: boolean,
+	hardwareProfiles: ReadonlySet<string> | undefined,
+): number {
 	const profileLevelId = getFmtpParameter(codec.sdpFmtpLine, 'profile-level-id');
 	const packetizationMode = getFmtpParameter(codec.sdpFmtpLine, 'packetization-mode');
-	const packetizationScore = packetizationMode === '1' ? 0 : h264PacketizationMode0Score;
+	const mode0Score = screenShareDelivery ? h264DeliveryPacketizationMode0Score : h264PacketizationMode0Score;
+	const packetizationScore = packetizationMode === '1' ? 0 : mode0Score;
 	if (!profileLevelId) return packetizationScore + h264MissingProfileScore;
-	return packetizationScore + (h264ProfileRanks.get(profileLevelId.slice(0, 4)) ?? h264UnrankedProfileScore);
+	const profile = profileLevelId.slice(0, 4);
+	if (!screenShareDelivery) {
+		return packetizationScore + (h264ProfileRanks.get(profile) ?? h264UnrankedProfileScore);
+	}
+	const isSoftwareOnly = hardwareProfiles !== undefined && hardwareProfiles.size > 0 && !hardwareProfiles.has(profile);
+	const hardwareScore = isSoftwareOnly ? h264NonHardwareProfilePenalty : 0;
+	return packetizationScore + hardwareScore + (h264DeliveryProfileRanks.get(profile) ?? h264UnrankedProfileScore);
 }
 
-function preferHardwareH264Codecs(codecs: ReadonlyArray<RtpCodecCapability>): Array<RtpCodecCapability> {
+function preferHardwareH264Codecs(
+	codecs: ReadonlyArray<RtpCodecCapability>,
+	screenShareDelivery: boolean,
+	hardwareProfiles: ReadonlySet<string> | undefined,
+): Array<RtpCodecCapability> {
 	return codecs
-		.map((codec, index) => ({codec, index, score: getH264PublisherCodecScore(codec)}))
+		.map((codec, index) => ({
+			codec,
+			index,
+			score: getH264PublisherCodecScore(codec, screenShareDelivery, hardwareProfiles),
+		}))
 		.sort((a, b) => a.score - b.score || a.index - b.index)
 		.map((entry) => entry.codec);
 }
@@ -1835,14 +1878,17 @@ function preferHardwareH264Codecs(codecs: ReadonlyArray<RtpCodecCapability>): Ar
 export function selectPublisherCodecPreferences(
 	codec: VideoCodec,
 	codecs: ReadonlyArray<RtpCodecCapability>,
+	screenShareDelivery: boolean = false,
+	h264HardwareProfiles?: ReadonlySet<string>,
 ): Array<RtpCodecCapability> {
 	const mimeTypes = new Set(videoCodecMimeTypes[codec]);
 	const selected = codecs.filter((entry) => mimeTypes.has(entry.mimeType.toLowerCase()));
 	if (selected.length === 0) return [];
-	const preferred = codec === 'h264' ? preferHardwareH264Codecs(selected) : selected;
+	const preferred =
+		codec === 'h264' ? preferHardwareH264Codecs(selected, screenShareDelivery, h264HardwareProfiles) : selected;
 	const isH264 = (entry: RtpCodecCapability): boolean => entry.mimeType.toLowerCase() === 'video/h264';
 	const remaining = codecs.filter((entry) => !mimeTypes.has(entry.mimeType.toLowerCase()));
-	const rankedH264 = preferHardwareH264Codecs(remaining.filter(isH264));
+	const rankedH264 = preferHardwareH264Codecs(remaining.filter(isH264), screenShareDelivery, h264HardwareProfiles);
 	let nextH264 = 0;
 	const rest = remaining.map((entry) => (isH264(entry) ? rankedH264[nextH264++] : entry));
 	return [...preferred, ...rest];

fluxer_app/pkgs/livekit-client/src/room/Room.ts

@@ -63,6 +63,7 @@ import {
 	publishDefaults,
 	roomConnectOptionDefaults,
 	roomOptionDefaults,
+	screenShareDeliveryPublishDefaults,
 	videoDefaults,
 } from './defaults.ts';
 import {ConnectionError, ConnectionErrorReason, UnexpectedConnectionState, UnsupportedServer} from './errors.ts';
@@ -227,7 +228,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
 			...options?.videoCaptureDefaults,
 		};
 		this.options.publishDefaults = {
-			...publishDefaults,
+			...(this.options.screenShareDelivery ? screenShareDeliveryPublishDefaults : publishDefaults),
 			...options?.publishDefaults,
 		};
 

fluxer_app/pkgs/livekit-client/src/room/defaults.ts

@@ -8,7 +8,7 @@ import {AudioPresets, BackupCodecPolicy, ScreenSharePresets, VideoPresets} from
 
 export const defaultVideoCodec = 'h264';
 
-export const publishDefaults: TrackPublishDefaults = {
+export const screenShareDeliveryPublishDefaults: TrackPublishDefaults = {
 	audioPreset: AudioPresets.music,
 	dtx: false,
 	red: true,
@@ -18,10 +18,14 @@ export const publishDefaults: TrackPublishDefaults = {
 	videoCodec: defaultVideoCodec,
 	backupCodec: {codec: 'h264'},
 	backupCodecPolicy: BackupCodecPolicy.SIMULCAST,
-	degradationPreference: 'maintain-resolution',
 	preConnectBuffer: false,
 } as const;
 
+export const publishDefaults: TrackPublishDefaults = {
+	...screenShareDeliveryPublishDefaults,
+	degradationPreference: 'maintain-resolution',
+};
+
 export const audioDefaults: AudioCaptureOptions = {
 	deviceId: {ideal: 'default'},
 	autoGainControl: true,

fluxer_app/pkgs/livekit-client/src/room/participant/LocalParticipant.ts

@@ -123,6 +123,11 @@ import {
 } from './publishUtils.ts';
 import type RemoteParticipant from './RemoteParticipant.ts';
 
+function hasSingleRidlessEncoding(track: LocalVideoTrack): boolean {
+	const encodings = track.sender?.getParameters().encodings;
+	return encodings?.length === 1 && !encodings[0].rid;
+}
+
 export default class LocalParticipant extends Participant {
 	override audioTrackPublications: Map<string, LocalTrackPublication>;
 
@@ -809,6 +814,7 @@ export default class LocalParticipant extends Participant {
 			...this.roomOptions.publishDefaults,
 			...options,
 		};
+		track.screenShareDelivery = this.roomOptions.screenShareDelivery ?? false;
 		const isStereoInput =
 			('channelCount' in track.mediaStreamTrack.getSettings() &&
 				track.mediaStreamTrack.getSettings().channelCount === 2) ||
@@ -1781,7 +1787,14 @@ export default class LocalParticipant extends Participant {
 		if (!pub.videoTrack) {
 			return;
 		}
-		const newCodecs = await pub.videoTrack.setPublishingCodecs(update.subscribedCodecs);
+		let subscribedCodecs = update.subscribedCodecs;
+		if (this.roomOptions.screenShareDelivery && hasSingleRidlessEncoding(pub.videoTrack)) {
+			subscribedCodecs = subscribedCodecs.filter((codec) => codec.qualities.some((quality) => quality.enabled));
+			if (subscribedCodecs.length === 0) {
+				return;
+			}
+		}
+		const newCodecs = await pub.videoTrack.setPublishingCodecs(subscribedCodecs);
 		for await (const codec of newCodecs) {
 			if (isBackupCodec(codec)) {
 				this.log.debug(`publish ${codec} for ${pub.videoTrack.sid}`, getLogContextFromTrack(pub));

fluxer_app/pkgs/livekit-client/src/room/track/LocalTrack.ts

@@ -41,6 +41,8 @@ export default abstract class LocalTrack<TrackKind extends Track.Kind = Track.Ki
 
 	codec?: VideoCodec;
 
+	screenShareDelivery: boolean = false;
+
 	get constraints() {
 		return this._constraints;
 	}
@@ -561,6 +563,10 @@ export default abstract class LocalTrack<TrackKind extends Track.Kind = Track.Ki
 		);
 
 	private debouncedTrackMuteHandler = debounce(async () => {
+		if (this.screenShareDelivery && this.source === Track.Source.ScreenShare) {
+			this.log.debug('screen share capture went idle, keeping upstream published', this.logContext);
+			return;
+		}
 		await this.pauseUpstream();
 	}, 5000);