feat(voice): scale screen share bitrate to the selected rung (#2414)

fluxer_app/src/features/voice/engine/AdaptiveScreenShareEngine.ts

@@ -9,11 +9,10 @@ import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
 import VoiceSettings, {type ScreenshareResolution, type StreamingMode} from '@app/features/voice/state/VoiceSettings';
 import {isNativeScreenShareTrack} from '@app/features/voice/utils/native_screen_capture_bridge/shared';
 import {
+	getScreenShareBitrateBps,
 	getScreenShareDimensions,
-	getScreenShareEncoding,
 	resolveScreenShareFrameRate,
 	SCREEN_SHARE_DEGRADATION_PREFERENCE,
-	SCREEN_SHARE_FORCED_VIDEO_BITRATE_BPS,
 	SUPPORTED_SCREEN_SHARE_FRAME_RATES,
 	type SupportedScreenShareFrameRate,
 } from '@app/features/voice/utils/ScreenShareOptions';
@@ -30,8 +29,6 @@ const FRAME_RATE_GRACE_FPS = 8;
 const STEP_DOWN_COOLDOWN_MS = 10_000;
 const STEP_UP_COOLDOWN_MS = 5_000;
 const MAX_STEP_UP_COOLDOWN_MS = 30_000;
-const BANDWIDTH_BITRATE_STEP_FACTOR = 0.6;
-const OBSERVED_TARGET_BITRATE_HEADROOM = 1.1;
 
 const SCREEN_SHARE_240P_DESCRIPTOR = msg({
 	message: '240p',
@@ -78,11 +75,6 @@ const QUALITY_LIMITED_DESCRIPTOR = msg({
 	comment: 'Fallback reason label in adaptive screen-share quality toasts.',
 	context: 'adaptive-screen-share',
 });
-const LOWERED_SCREEN_SHARE_BITRATE_DESCRIPTOR = msg({
-	message: 'Lowered screen share bitrate - {reason}',
-	comment: 'Toast shown after adaptive screen-share quality lowers bitrate. {reason} is the limitation reason.',
-	context: 'adaptive-screen-share',
-});
 const LOWERED_SCREEN_SHARE_FRAME_RATE_DESCRIPTOR = msg({
 	message: 'Lowered screen share to {frameRate} FPS - {reason}',
 	comment:
@@ -129,7 +121,6 @@ export interface OutboundVideoAdaptationStats {
 	framesPerSecond?: number;
 	frameWidth?: number;
 	frameHeight?: number;
-	targetBitrate?: number;
 }
 
 export type PreviousOutboundVideoSample = OutboundVideoAdaptationStats | null;
@@ -171,12 +162,6 @@ function shouldPreferFrameRateStepDown(mode: StreamingMode, reason: QualityLimit
 	return reason === 'cpu';
 }
 
-function getSenderMaxBitrate(sender: RTCRtpSender): number | null {
-	const encodings = sender.getParameters().encodings;
-	const maxBitrate = encodings?.find((encoding) => typeof encoding.maxBitrate === 'number')?.maxBitrate;
-	return typeof maxBitrate === 'number' && maxBitrate > 0 ? maxBitrate : null;
-}
-
 function getHigherResolution(
 	resolution: ScreenshareResolution,
 	configuredResolution: ScreenshareResolution,
@@ -255,7 +240,6 @@ export function extractOutboundVideoAdaptationStats(stats: RTCStatsReport): Outb
 			framesPerSecond?: number;
 			frameWidth?: number;
 			frameHeight?: number;
-			targetBitrate?: number;
 		};
 		if (!isOutboundVideoReport(report)) continue;
 		return {
@@ -266,7 +250,6 @@ export function extractOutboundVideoAdaptationStats(stats: RTCStatsReport): Outb
 			framesPerSecond: typeof report.framesPerSecond === 'number' ? report.framesPerSecond : undefined,
 			frameWidth: typeof report.frameWidth === 'number' ? report.frameWidth : undefined,
 			frameHeight: typeof report.frameHeight === 'number' ? report.frameHeight : undefined,
-			targetBitrate: typeof report.targetBitrate === 'number' ? report.targetBitrate : undefined,
 		};
 	}
 	return null;
@@ -299,16 +282,6 @@ export function isOutboundFrameRateBelowTarget(
 	return frameRate < Math.max(1, targetFrameRate - FRAME_RATE_GRACE_FPS);
 }
 
-export function computeAdaptiveBitrate(
-	frameRate: number,
-	maxBitrateBps?: number,
-	observedTargetBitrateBps?: number,
-): number {
-	const base = getScreenShareEncoding(frameRate, maxBitrateBps).maxBitrate ?? 0;
-	if (typeof observedTargetBitrateBps !== 'number' || observedTargetBitrateBps <= 0) return base;
-	return Math.min(base, Math.round(observedTargetBitrateBps * OBSERVED_TARGET_BITRATE_HEADROOM));
-}
-
 export function computeResolutionScale(
 	fromResolution: ScreenshareResolution,
 	toResolution: ScreenshareResolution,
@@ -323,16 +296,15 @@ export async function applyResolutionFrameRateAndBitrate(
 	sender: RTCRtpSender,
 	resolution: ScreenshareResolution,
 	frameRate: SupportedScreenShareFrameRate,
-	maxBitrateBps?: number,
-	observedTargetBitrateBps?: number,
 ): Promise<void> {
 	const dimensions = getScreenShareDimensions(resolution);
+	const maxBitrate = getScreenShareBitrateBps(resolution, frameRate);
 	if (isNativeScreenShareTrack(track.mediaStreamTrack)) {
 		const updated = updateVoiceEngineV2ScreenEncodingFromMediaEngine({
 			width: dimensions.width,
 			height: dimensions.height,
 			frameRate,
-			maxBitrateBps: computeAdaptiveBitrate(frameRate, maxBitrateBps, observedTargetBitrateBps),
+			maxBitrateBps: maxBitrate,
 		});
 		if (!updated) {
 			logger.warn('Skipped native screen-share encoding update because v2 screen state is unavailable', {
@@ -361,7 +333,6 @@ export async function applyResolutionFrameRateAndBitrate(
 	}
 	const parameters = sender.getParameters();
 	const encodings = parameters.encodings?.length ? parameters.encodings : [{}];
-	const maxBitrate = computeAdaptiveBitrate(frameRate, maxBitrateBps, observedTargetBitrateBps);
 	parameters.degradationPreference = SCREEN_SHARE_DEGRADATION_PREFERENCE;
 	parameters.encodings = encodings.map((encoding) => ({
 		...encoding,
@@ -391,7 +362,6 @@ class AdaptiveScreenShareEngine extends Store {
 	private lastStepUpAt = 0;
 	private stepUpCooldownMs = STEP_UP_COOLDOWN_MS;
 	private previousSample: PreviousOutboundVideoSample = null;
-	private bandwidthBitrateStepActive = false;
 
 	get qualitySnapshot(): AdaptiveQualitySnapshot {
 		const configured = getConfiguredQuality();
@@ -449,7 +419,6 @@ class AdaptiveScreenShareEngine extends Store {
 			this.lastStepUpAt = 0;
 			this.stepUpCooldownMs = STEP_UP_COOLDOWN_MS;
 			this.previousSample = null;
-			this.bandwidthBitrateStepActive = false;
 			this.configuredResolution = null;
 			this.configuredFrameRate = null;
 			this.effectiveResolution = null;
@@ -474,7 +443,6 @@ class AdaptiveScreenShareEngine extends Store {
 				screenShare.sender,
 				snapshot.configuredResolution,
 				snapshot.configuredFrameRate,
-				SCREEN_SHARE_FORCED_VIDEO_BITRATE_BPS,
 			);
 			logger.info('Restored configured screen share quality', {
 				resolution: snapshot.configuredResolution,
@@ -528,7 +496,6 @@ class AdaptiveScreenShareEngine extends Store {
 				this.stepUpStreak = 0;
 			} else if (stats.qualityLimitationReason === 'none') {
 				this.stepDownStreak = 0;
-				this.bandwidthBitrateStepActive = false;
 				if (this.isAdapted) {
 					this.stepUpStreak++;
 				}
@@ -566,30 +533,20 @@ class AdaptiveScreenShareEngine extends Store {
 		const nextFrameRate = shouldPreferFrameRateStepDown(streamingMode, reason)
 			? getLowerFrameRate(currentFrameRate)
 			: null;
-		const currentSenderMaxBitrate =
-			getSenderMaxBitrate(screenShare.sender) ??
-			computeAdaptiveBitrate(currentFrameRate, SCREEN_SHARE_FORCED_VIDEO_BITRATE_BPS, stats.targetBitrate);
-		const canApplyBandwidthBitrateStep =
-			reason === 'bandwidth' && !nextFrameRate && !this.bandwidthBitrateStepActive && currentSenderMaxBitrate > 0;
-		const nextResolution =
-			nextFrameRate || canApplyBandwidthBitrateStep ? currentResolution : getLowerResolution(currentResolution);
+		const nextResolution = nextFrameRate ? currentResolution : getLowerResolution(currentResolution);
 		if (!nextResolution) {
 			this.stepDownStreak = 0;
 			return;
 		}
-		const nextMaxBitrateBps = canApplyBandwidthBitrateStep
-			? Math.max(1, Math.round(currentSenderMaxBitrate * BANDWIDTH_BITRATE_STEP_FACTOR))
-			: SCREEN_SHARE_FORCED_VIDEO_BITRATE_BPS;
 		const nextEffectiveFrameRate = nextFrameRate ?? currentFrameRate;
-		const stepKind = nextFrameRate ? 'framerate' : canApplyBandwidthBitrateStep ? 'bitrate' : 'resolution';
+		const nextMaxBitrateBps = getScreenShareBitrateBps(nextResolution, nextEffectiveFrameRate);
+		const stepKind = nextFrameRate ? 'framerate' : 'resolution';
 		try {
 			await applyResolutionFrameRateAndBitrate(
 				screenShare.track,
 				screenShare.sender,
 				nextResolution,
 				nextEffectiveFrameRate,
-				nextMaxBitrateBps,
-				canApplyBandwidthBitrateStep ? undefined : stats.targetBitrate,
 			);
 			this.update(() => {
 				this.effectiveResolution = nextResolution;
@@ -599,7 +556,6 @@ class AdaptiveScreenShareEngine extends Store {
 				this.stepUpStreak = 0;
 				this.lastStepDownAt = Date.now();
 				this.stepUpCooldownMs = Math.min(this.stepUpCooldownMs * 2, MAX_STEP_UP_COOLDOWN_MS);
-				this.bandwidthBitrateStepActive = canApplyBandwidthBitrateStep;
 			});
 			logger.info('Reduced screen share quality adaptively', {
 				fromResolution: currentResolution,
@@ -611,9 +567,7 @@ class AdaptiveScreenShareEngine extends Store {
 				reason,
 			});
 			const reasonLabel = formatLimitationReasonForToast(reason);
-			if (stepKind === 'bitrate') {
-				showAdaptiveScreenShareToast(i18n._(LOWERED_SCREEN_SHARE_BITRATE_DESCRIPTOR, {reason: reasonLabel}));
-			} else if (stepKind === 'framerate') {
+			if (stepKind === 'framerate') {
 				showAdaptiveScreenShareToast(
 					i18n._(LOWERED_SCREEN_SHARE_FRAME_RATE_DESCRIPTOR, {
 						frameRate: nextEffectiveFrameRate,
@@ -656,7 +610,6 @@ class AdaptiveScreenShareEngine extends Store {
 				this.isAdapted = false;
 				this.stepUpStreak = 0;
 				this.stepUpCooldownMs = STEP_UP_COOLDOWN_MS;
-				this.bandwidthBitrateStepActive = false;
 			});
 			return;
 		}
@@ -666,7 +619,6 @@ class AdaptiveScreenShareEngine extends Store {
 				screenShare.sender,
 				nextResolution,
 				configuredFrameRate,
-				SCREEN_SHARE_FORCED_VIDEO_BITRATE_BPS,
 			);
 			this.update(() => {
 				this.effectiveResolution = nextResolution;
@@ -674,7 +626,6 @@ class AdaptiveScreenShareEngine extends Store {
 				this.isAdapted = nextResolution !== configuredResolution;
 				this.stepUpStreak = 0;
 				this.lastStepUpAt = Date.now();
-				this.bandwidthBitrateStepActive = false;
 				if (!this.isAdapted) {
 					this.stepUpCooldownMs = STEP_UP_COOLDOWN_MS;
 				}

fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.ts

@@ -38,6 +38,7 @@ import {
 } from '@app/features/voice/engine/v2/VoiceEngineV2AppAdapterAssertions';
 import {VoiceEngineV2AppReconnectPolicy} from '@app/features/voice/engine/v2/VoiceEngineV2AppReconnectPolicy';
 import VoiceRegionTeleport from '@app/features/voice/state/VoiceRegionTeleport';
+import {SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS} from '@app/features/voice/utils/ScreenShareOptions';
 import {
 	getVideoDecoderExclusionsSync,
 	loadVideoDecoderExclusions,
@@ -139,6 +140,10 @@ function createWebAudioMixOption(): RoomOptions['webAudioMix'] {
 	return true;
 }
 
+const ROOM_PUBLISH_DEFAULTS: RoomOptions['publishDefaults'] = {
+	screenShareEncoding: {maxBitrate: SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS, maxFramerate: 30, priority: 'high'},
+};
+
 function createRoomOptions(
 	e2eeKey: string | null,
 	subscriberVideoCodecExclusions: RoomOptions['subscriberVideoCodecExclusions'],
@@ -151,6 +156,7 @@ function createRoomOptions(
 		adaptiveStream: false,
 		dynacast: true,
 		webAudioMix: createWebAudioMixOption(),
+		publishDefaults: ROOM_PUBLISH_DEFAULTS,
 		subscriberVideoCodecExclusions,
 	};
 	let e2eeKeyProvider: ExternalE2EEKeyProvider | null = null;
@@ -675,6 +681,7 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
 			adaptiveStream: false,
 			dynacast: true,
 			webAudioMix: createWebAudioMixOption(),
+			publishDefaults: ROOM_PUBLISH_DEFAULTS,
 			subscriberVideoCodecExclusions: cachedExclusions && cachedExclusions.length > 0 ? cachedExclusions : undefined,
 		};
 		if (!this.isLatestConnectionAttempt(attemptId) || this.connectionState.room !== existingRoom) {

fluxer_app/src/features/voice/engine/voice_screen_share_manager/shared.ts

@@ -12,17 +12,20 @@ import {
 import {getGpuEncoderReportSync, loadGpuEncoderReport} from '@app/features/voice/utils/GpuEncoderCapabilities';
 import {loadNativeHardwareEncoderCapabilities} from '@app/features/voice/utils/NativeHardwareEncoderCapabilities';
 import {
+	capScreenShareEncodingToDimensions,
 	getScreenShareEncoding,
 	resolveStreamingModeSettings,
 	SCREEN_SHARE_DEGRADATION_PREFERENCE,
-	SCREEN_SHARE_FORCED_VIDEO_BITRATE_BPS,
+	SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
 } from '@app/features/voice/utils/ScreenShareOptions';
 import {ScreenShareRollbackIncompleteError} from '@app/features/voice/utils/ScreenShareRollbackIncompleteError';
 import {classifyVideoEncoderAcceleration} from '@app/features/voice/utils/VideoAccelerationClassification';
 import {
 	BackupCodecPolicy,
+	type LocalParticipant,
 	type LocalTrackPublication,
 	type ScreenShareCaptureOptions,
+	Track,
 	type TrackPublishOptions,
 	type VideoCodec,
 	type VideoEncoding,
@@ -176,26 +179,13 @@ export async function releaseScreenShareCaptureCleanup(snapshot: ScreenShareCapt
 	}
 }
 
-function getCodecSpecificScreenShareBitrateCeiling(codec: VideoCodec): number | undefined {
-	const acceleration = getCodecCapabilityReport()[codec].hardwareAccelerated;
-	if (codec === 'h264' && acceleration === 'hardware') return 20000000;
-	if ((codec === 'av1' || codec === 'vp9') && acceleration === 'software') return 40000000;
-	return undefined;
-}
-
-function clampScreenShareEncoding(encoding: VideoEncoding | undefined, codec: VideoCodec): VideoEncoding | undefined {
+function clampScreenShareEncoding(encoding: VideoEncoding | undefined): VideoEncoding | undefined {
 	if (!encoding) return undefined;
-	const maxBitrateBps = SCREEN_SHARE_FORCED_VIDEO_BITRATE_BPS;
-	const codecCeilingBps = getCodecSpecificScreenShareBitrateCeiling(codec);
-	const bitrateCeilingBps =
-		maxBitrateBps !== undefined && codecCeilingBps !== undefined
-			? Math.min(maxBitrateBps, codecCeilingBps)
-			: (maxBitrateBps ?? codecCeilingBps);
 	return {
 		...encoding,
 		maxBitrate:
-			typeof encoding.maxBitrate === 'number' && bitrateCeilingBps !== undefined
-				? Math.min(encoding.maxBitrate, bitrateCeilingBps)
+			typeof encoding.maxBitrate === 'number'
+				? Math.min(encoding.maxBitrate, SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS)
 				: encoding.maxBitrate,
 		priority: encoding.priority ?? 'high',
 	};
@@ -210,13 +200,10 @@ export function getEffectiveScreenShareEncoding(publishOptions?: TrackPublishOpt
 			VoiceSettings.getScreenshareResolution(),
 			VoiceSettings.getVideoFrameRate(),
 		);
-		screenShareEncoding = getScreenShareEncoding(settings.frameRate);
+		screenShareEncoding = getScreenShareEncoding(settings.resolution, settings.frameRate);
 	}
 	if (!screenShareEncoding) return undefined;
-	return clampScreenShareEncoding(
-		adjustScreenShareEncodingForCodec(screenShareEncoding, preferredVideoCodec),
-		preferredVideoCodec,
-	);
+	return clampScreenShareEncoding(adjustScreenShareEncodingForCodec(screenShareEncoding, preferredVideoCodec));
 }
 
 function getScreenShareScalabilityModeForCodec(codec: VideoCodec): TrackPublishOptions['scalabilityMode'] | undefined {
@@ -313,7 +300,7 @@ export async function getEffectivePublishOptions(
 		videoCodec: preferredVideoCodec,
 		screenShareEncoding: getEffectiveScreenShareEncoding({...publishOptions, videoCodec: preferredVideoCodec}),
 		degradationPreference: SCREEN_SHARE_DEGRADATION_PREFERENCE,
-		simulcast: isSvcScreenShareCodec(preferredVideoCodec) ? false : publishOptions?.simulcast,
+		simulcast: isSvcScreenShareCodec(preferredVideoCodec) ? false : (publishOptions?.simulcast ?? false),
 		scalabilityMode,
 		...(backupCodec !== undefined ? {backupCodec} : {}),
 		...(backupCodecPolicy !== undefined ? {backupCodecPolicy} : {}),
@@ -332,6 +319,34 @@ export function applyScreenShareContentHint(
 	}
 }
 
+function distributeScreenShareBitrate(
+	encodings: ReadonlyArray<RTCRtpEncodingParameters>,
+	maxBitrate: number,
+): Array<number> {
+	if (encodings.length === 1) return [maxBitrate];
+	const weights = encodings.map((encoding) =>
+		typeof encoding.maxBitrate === 'number' && encoding.maxBitrate > 0 ? encoding.maxBitrate : 0,
+	);
+	if (weights.some((weight) => weight <= 0)) {
+		return encodings.map(() => Math.floor(maxBitrate / encodings.length));
+	}
+	const total = weights.reduce((sum, weight) => sum + weight, 0);
+	return weights.map((weight) => Math.floor((weight / total) * maxBitrate));
+}
+
+export function getPublishedScreenShareMaxBitrateBps(
+	participant: LocalParticipant | null | undefined,
+): number | undefined {
+	const sender = participant?.getTrackPublication(Track.Source.ScreenShare)?.videoTrack?.sender;
+	if (!sender) return undefined;
+	let total = 0;
+	for (const encoding of sender.getParameters().encodings ?? []) {
+		if (typeof encoding.maxBitrate !== 'number') return undefined;
+		total += encoding.maxBitrate;
+	}
+	return total > 0 ? total : undefined;
+}
+
 export async function enforceScreenShareSenderParameters(
 	sender: RTCRtpSender | undefined,
 	publishOptions?: TrackPublishOptions,
@@ -339,17 +354,24 @@ export async function enforceScreenShareSenderParameters(
 ): Promise<boolean> {
 	if (!sender) return false;
 	const preferredVideoCodec = codecOverride ?? publishOptions?.videoCodec ?? getPreferredScreenShareCodec();
-	const screenShareEncoding = getEffectiveScreenShareEncoding({...publishOptions, videoCodec: preferredVideoCodec});
+	const effectiveEncoding = getEffectiveScreenShareEncoding({...publishOptions, videoCodec: preferredVideoCodec});
+	const screenShareEncoding = effectiveEncoding
+		? capScreenShareEncodingToDimensions(effectiveEncoding, sender.track?.getSettings())
+		: undefined;
 	const scalabilityMode = getEffectiveScreenShareScalabilityMode(preferredVideoCodec, publishOptions, {
 		respectExplicit: !codecOverride || codecOverride === publishOptions?.videoCodec,
 	});
 	try {
 		const params = sender.getParameters();
 		const encodings = params.encodings?.length ? params.encodings : [{}];
+		const bitrates =
+			screenShareEncoding?.maxBitrate !== undefined
+				? distributeScreenShareBitrate(encodings, screenShareEncoding.maxBitrate)
+				: undefined;
 		params.degradationPreference = SCREEN_SHARE_DEGRADATION_PREFERENCE;
-		params.encodings = encodings.map((encoding) => ({
+		params.encodings = encodings.map((encoding, index) => ({
 			...encoding,
-			...(screenShareEncoding?.maxBitrate !== undefined ? {maxBitrate: screenShareEncoding.maxBitrate} : {}),
+			...(bitrates ? {maxBitrate: bitrates[index]} : {}),
 			...(screenShareEncoding?.maxFramerate !== undefined ? {maxFramerate: screenShareEncoding.maxFramerate} : {}),
 			priority: screenShareEncoding?.priority ?? encoding.priority ?? 'high',
 			networkPriority: screenShareEncoding?.priority ?? encoding.networkPriority ?? 'high',