feat(voice): ship the screen share delivery rework to everyone (#2931)

This commit is contained in:
Hampus authored and GitHub committed 2026-09-24 14:57:40 +02:00
1 parent 88038a1d5b
commit a0825e77c4
53 files changed
+171 -1806

No files matched your search

@@ -11,7 +11,6 @@ import {
getLocalScreenShareVideoPublications,
isLiveLocalTrackPublication,
} from '@app/features/voice/engine/VoiceTrackPublicationUtils';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {
buildScreenShareCodecProfile,
@@ -357,7 +356,6 @@ class ScreenShareCodecNegotiation {
encoderMode: ScreenShareEncoderMode;
order: ReadonlyArray<VideoCodec>;
} | null = null;
private frozenDelivery: {trackSid: string; enabled: boolean} | null = null;
private lastCodecChangeAt = 0;
private suppressionTimer: ReturnType<typeof setTimeout> | null = null;
private publishedTrackSid: string | null = null;
@@ -452,22 +450,9 @@ class ScreenShareCodecNegotiation {
this.localCodecs = buildLocalCodecAdvertisements(this.resolveCodecPreferenceOrder());
}
private resolveScreenShareDelivery(room: Room | null = this.room): boolean {
const trackSid = this.getLocalScreenSharePublication(room)?.trackSid ?? null;
if (trackSid === null) {
this.frozenDelivery = null;
return ScreenShareDeliveryRollout.enabled;
}
if (this.frozenDelivery?.trackSid === trackSid) return this.frozenDelivery.enabled;
const enabled = ScreenShareDeliveryRollout.enabled;
this.frozenDelivery = {trackSid, enabled};
return enabled;
}
private resolveCodecPreferenceOrder(
preference: CodecPreference = VoiceSettings.getPreferredScreenShareCodec(),
): ReadonlyArray<VideoCodec> {
if (!this.resolveScreenShareDelivery()) return getScreenShareCodecPreferenceOrder(preference);
const encoderMode = VoiceSettings.getScreenShareEncoderMode();
const frozen = this.frozenCodecPreference;
if (
@@ -523,7 +508,7 @@ class ScreenShareCodecNegotiation {
unknownParticipants: number;
} {
const now = Date.now();
const viewerIdentities = this.resolveScreenShareDelivery(room) ? this.getScreenShareViewerIdentities(room) : null;
const viewerIdentities = this.getScreenShareViewerIdentities(room);
const knownRemoteCodecs: Array<Array<FluxerCodecAdvertisement>> = [];
const participants: Array<{identity: string; firstSeenAt: number}> = [];
for (const participant of room?.remoteParticipants.values() ?? []) {
@@ -640,7 +625,6 @@ class ScreenShareCodecNegotiation {
this.selectedCodec = null;
this.localCodecs = [];
this.frozenCodecPreference = null;
this.frozenDelivery = null;
this.lastCodecChangeAt = 0;
this.publishedTrackSid = null;
this.clearSuppressionTimer();
@@ -749,7 +733,6 @@ class ScreenShareCodecNegotiation {
const {knownRemoteCodecs, unknownParticipants} = this.getRemoteCodecInputs(room);
this.scheduleGraceReevaluations(room, bindingRevision);
const previousCodec = this.selectedCodec;
const delivery = this.resolveScreenShareDelivery(room);
this.negotiationSnapshot = transitionScreenShareCodecNegotiationSnapshot(this.negotiationSnapshot, {
type: 'negotiation.evaluate',
localCodecs: this.localCodecs,
@@ -757,7 +740,7 @@ class ScreenShareCodecNegotiation {
unknownParticipants,
reason,
codecPreference: this.resolveCodecPreferenceOrder(),
publishedCodec: delivery ? this.observePublishedScreenShareCodec(room) : null,
publishedCodec: this.observePublishedScreenShareCodec(room),
});
const selection = this.negotiationSnapshot.context.selection;
if (!selection) return null;
@@ -765,11 +748,7 @@ class ScreenShareCodecNegotiation {
this.selectedCodec = selection.codec;
return selection;
}
if (
delivery &&
previousCodec !== null &&
Date.now() - this.lastCodecChangeAt < SCREEN_SHARE_CODEC_CHANGE_SUPPRESSION_MS
) {
if (previousCodec !== null && Date.now() - this.lastCodecChangeAt < SCREEN_SHARE_CODEC_CHANGE_SUPPRESSION_MS) {
logger.debug('Suppressed a screen share codec change inside the change window', {
codec: selection.codec,
previousCodec,
@@ -779,7 +758,7 @@ class ScreenShareCodecNegotiation {
return selection;
}
this.selectedCodec = selection.codec;
if (delivery) this.armCodecChangeSuppression();
this.armCodecChangeSuppression();
this.mediaSessionId = createId('media');
logger.info('Selected screen share codec from XState capability intersection', selection);
await this.publishSessionUpdate(room, selection);
@@ -9,12 +9,7 @@ import {
type VoiceGatewayStateEvent,
type VoiceGatewayStateSnapshot,
} from '@app/features/voice/engine/VoiceGatewayStateMachine';
import {describe, expect, it, vi} from 'vitest';
vi.mock('@app/features/voice/state/ScreenShareDeliveryRollout', () => ({
ScreenShareDeliveryRollout: {enabled: false},
default: {enabled: false},
}));
import {describe, expect, it} from 'vitest';
function voiceState(overrides: Partial<VoiceState> = {}): VoiceState {
return {
@@ -19,12 +19,7 @@ import {
type VoiceMediaGraphStreamTileState,
} from '@app/features/voice/engine/VoiceMediaGraphTileState';
import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
import {describe, expect, it, vi} from 'vitest';
vi.mock('@app/features/voice/state/ScreenShareDeliveryRollout', () => ({
ScreenShareDeliveryRollout: {enabled: false},
default: {enabled: false},
}));
import {describe, expect, it} from 'vitest';
const SIMULATION_SEEDS: ReadonlyArray<number> = Array.from({length: 20}, (_, index) => 1 + index * 7919);
const SIMULATION_STEP_COUNT = 1000;
@@ -11,7 +11,6 @@ import {
voiceMediaGraphStatsTrackKey,
} from '@app/features/voice/engine/VoiceMediaGraphStatsObservations';
import type {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import type {VoiceEngineV2PerTrackStats} from '@fluxer/voice_engine_v2';
const VOICE_MEDIA_GRAPH_STATS_TRACK_LIMIT = 1024;
@@ -77,9 +76,7 @@ function perTrackStatToInfo(track: VoiceEngineV2PerTrackStats): VoiceMediaGraphP
if (track.kind !== 'video') return null;
const width = track.frameWidth ?? track.sourceFrameWidth;
const height = track.frameHeight ?? track.sourceFrameHeight;
const fps = ScreenShareDeliveryRollout.enabled
? (track.effectiveFramesPerSecond ?? track.framesPerSecond)
: (track.effectiveFramesPerSecond ?? track.framesPerSecond ?? track.sourceFramesPerSecond);
const fps = track.effectiveFramesPerSecond ?? track.framesPerSecond;
const info: VoiceMediaGraphPartialTrackInfo = {};
if (isPositiveDimension(width) && isPositiveDimension(height)) {
info.width = width;
@@ -193,9 +190,7 @@ function observationToPartialTrackInfo(
): VoiceMediaGraphPartialTrackInfo | null {
const width = observation.width ?? observation.sourceWidth ?? undefined;
const height = observation.height ?? observation.sourceHeight ?? undefined;
const fps = ScreenShareDeliveryRollout.enabled
? (observation.fps ?? undefined)
: (observation.fps ?? observation.sourceFps ?? undefined);
const fps = observation.fps ?? undefined;
const info: VoiceMediaGraphPartialTrackInfo = {};
if (isPositiveDimension(width) && isPositiveDimension(height)) {
info.width = width;
@@ -16,7 +16,6 @@ import {
} from '@app/features/voice/engine/VoiceStreamWatchState';
import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
import ParticipantVolume from '@app/features/voice/state/ParticipantVolume';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import {ScreenShareWatchErrorCode, ScreenShareWatchFailures} from '@app/features/voice/state/ScreenShareWatchFailures';
import {monitorScreenShareDecodeHealth} from '@app/features/voice/utils/ScreenShareCodecDiagnostics';
import {markScreenShareDecodeFailure} from '@app/features/voice/utils/VideoDecoderCapabilities';
@@ -108,30 +107,6 @@ interface ParticipantSpeakingDisposer {
dispose: () => void;
}
type LatencyTunedReceiver = RTCRtpReceiver & {
jitterBufferTarget?: number;
playoutDelayHint?: number;
};
function getRemoteTrackReceiver(track: RemoteTrack): LatencyTunedReceiver | undefined {
return (track as RemoteTrack & {receiver?: LatencyTunedReceiver}).receiver;
}
function applyInteractiveReceiverBuffer(track: RemoteTrack, pub: RemoteTrackPublication): void {
const receiver = getRemoteTrackReceiver(track);
if (!receiver) return;
try {
if (pub.kind === Track.Kind.Video && pub.source === Track.Source.ScreenShare) {
receiver.jitterBufferTarget = 80;
receiver.playoutDelayHint = 0.04;
} else if (pub.kind === Track.Kind.Audio) {
receiver.jitterBufferTarget = 60;
}
} catch (error) {
logger.debug('Failed to apply interactive receiver buffer target', {error, source: pub.source, kind: pub.kind});
}
}
export function bindRoomEvents(
room: Room,
attemptId: number,
@@ -141,7 +116,6 @@ export function bindRoomEvents(
dependencies: RoomEventDependencies,
): void {
const guard = dependencies.connection.createGuardedHandler;
const screenShareDeliveryEnabled = ScreenShareDeliveryRollout.enabled;
const participantSpeakingDisposers = new Map<string, ParticipantSpeakingDisposer>();
const screenShareDecodeMonitorCancels = new Map<string, () => void>();
const remoteTrackLifecycleDisposers = new Map<string, () => void>();
@@ -443,9 +417,6 @@ export function bindRoomEvents(
trackSid: pub.trackSid,
});
}
if (!screenShareDeliveryEnabled) {
applyInteractiveReceiverBuffer(track, pub);
}
monitorDecodeHealth(track, pub);
dependencies.remoteSpeaking.attachIfApplicable(participant, pub, track);
bindRemoteTrackLifecycleIfApplicable(participant, pub, track);
@@ -1,7 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import assert from 'node:assert/strict';
import {isElectronPlatform} from '@app/features/platform/types/Platform';
import {Logger} from '@app/features/platform/utils/AppLogger';
import {Store} from '@app/features/voice/engine/Store';
import {sendVoiceStateDisconnect} from '@app/features/voice/engine/VoiceChannelConnector';
@@ -37,7 +36,6 @@ import {
isReadyToRepublishTrack,
} from '@app/features/voice/engine/v2/VoiceEngineV2AppAdapterAssertions';
import {VoiceEngineV2AppReconnectPolicy} from '@app/features/voice/engine/v2/VoiceEngineV2AppReconnectPolicy';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import VoiceRegionTeleport from '@app/features/voice/state/VoiceRegionTeleport';
import {
findVideoPublishCodecPolicyViolation,
@@ -45,10 +43,7 @@ import {
} from '@app/features/voice/utils/CodecCapabilityDetector';
import {getH264HardwareProfilesSync} from '@app/features/voice/utils/GpuEncoderCapabilities';
import {setNoiseSuppressionScopeGuildId} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionSelection';
import {
SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS,
SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
} from '@app/features/voice/utils/ScreenShareOptions';
import {SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS} from '@app/features/voice/utils/ScreenShareOptions';
import {
clearScreenShareDecodeFailures,
getVideoDecoderExclusionsSync,
@@ -151,9 +146,7 @@ function createWebAudioMixOption(): RoomOptions['webAudioMix'] {
function createRoomPublishDefaults(): RoomOptions['publishDefaults'] {
return {
screenShareEncoding: {
maxBitrate: ScreenShareDeliveryRollout.enabled
? SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS
: SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
maxBitrate: SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
maxFramerate: 30,
priority: 'high',
},
@@ -164,7 +157,6 @@ function createRoomPublishDefaults(): RoomOptions['publishDefaults'] {
function createRoomOptions(
e2eeKey: string | null,
subscriberVideoCodecExclusions: RoomOptions['subscriberVideoCodecExclusions'],
screenShareDelivery: boolean,
): {
roomOptions: RoomOptions;
e2eeKeyProvider: ExternalE2EEKeyProvider | null;
@@ -176,8 +168,7 @@ function createRoomOptions(
webAudioMix: createWebAudioMixOption(),
publishDefaults: createRoomPublishDefaults(),
subscriberVideoCodecExclusions,
screenShareDelivery,
h264HardwareProfiles: screenShareDelivery ? getH264HardwareProfilesSync()?.profiles : undefined,
h264HardwareProfiles: getH264HardwareProfilesSync()?.profiles,
};
let e2eeKeyProvider: ExternalE2EEKeyProvider | null = null;
let e2eeWorker: Worker | null = null;
@@ -196,19 +187,11 @@ function createRoomOptions(
return {roomOptions, e2eeKeyProvider, e2eeWorker};
}
function createRoomConnectOptions(screenShareDelivery: boolean): RoomConnectOptions {
function createRoomConnectOptions(): RoomConnectOptions {
const connectOptions: RoomConnectOptions = {
autoSubscribe: false,
};
assert.equal(connectOptions.autoSubscribe, false, 'LiveKit connect options must not auto-subscribe');
if (!screenShareDelivery && isElectronPlatform()) {
connectOptions.rtcConfig = {iceTransportPolicy: 'relay'};
assert.equal(
connectOptions.rtcConfig.iceTransportPolicy,
'relay',
'Electron LiveKit connects must force relay ICE',
);
}
return connectOptions;
}
@@ -564,12 +547,7 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
logger.warn('Aborting LiveKit room creation after codec probing because attempt is stale', {attemptId});
return;
}
const screenShareDelivery = ScreenShareDeliveryRollout.enabled;
const {roomOptions, e2eeKeyProvider, e2eeWorker} = createRoomOptions(
e2eeKey,
subscriberVideoCodecExclusions,
screenShareDelivery,
);
const {roomOptions, e2eeKeyProvider, e2eeWorker} = createRoomOptions(e2eeKey, subscriberVideoCodecExclusions);
const room = new LiveKitRoom(roomOptions);
ownE2EEWorker(room, e2eeWorker);
let roomClosed = false;
@@ -602,7 +580,7 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
return;
}
logger.info('Attempting to connect to LiveKit', {endpoint, guildId, channelId: resolvedChannelId});
const connectOptions = createRoomConnectOptions(screenShareDelivery);
const connectOptions = createRoomConnectOptions();
room
.connect(endpoint, token, connectOptions)
.then(() => {
@@ -704,14 +682,12 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
const connectionId = raw.connection_id ?? null;
this.abortHotSwap();
const cachedExclusions = getVideoDecoderExclusionsSync();
const screenShareDelivery = ScreenShareDeliveryRollout.enabled;
const roomOptions: RoomOptions = {
adaptiveStream: false,
dynacast: true,
webAudioMix: createWebAudioMixOption(),
publishDefaults: createRoomPublishDefaults(),
subscriberVideoCodecExclusions: cachedExclusions && cachedExclusions.length > 0 ? cachedExclusions : undefined,
screenShareDelivery,
};
if (!this.isLatestConnectionAttempt(attemptId) || this.connectionState.room !== existingRoom) {
logger.warn('Region hot-swap: aborted before room creation because attempt is stale', {attemptId});
@@ -728,7 +704,7 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
this.abortHotSwap();
}
});
const connectOptions = createRoomConnectOptions(screenShareDelivery);
const connectOptions = createRoomConnectOptions();
logger.info('Region hot-swap: connecting to new endpoint', {endpoint, guildId, channelId});
newRoom
.connect(endpoint, token, connectOptions)
@@ -63,7 +63,6 @@ import ActiveScreenShareSource, {
type PublishedScreenShareSource,
} from '@app/features/voice/state/ActiveScreenShareSource';
import LocalVoiceState from '@app/features/voice/state/LocalVoiceState';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {
prepareHighFidelityScreenShareAudioTrack,
@@ -183,7 +182,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
encoderVerificationTimer: (() => void) | null = null;
private screenShareTrackingHolds = 0;
private screenShareCapturePaused = false;
private screenShareDeliveryArmed = false;
private screenShareTrackingGeneration = 0;
private readonly reconciledCodecPairs = new Set<string>();
private readonly verifiedCodecCorrectionsByTrack = new WeakMap<MediaStreamTrack, number>();
@@ -679,7 +677,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
preferredTrack?: LocalVideoTrack,
): void {
this.cleanupActiveScreenShareEndListenerInternal();
this.screenShareDeliveryArmed = ScreenShareDeliveryRollout.enabled;
const publication = preferredTrack ? undefined : participant.getTrackPublication(Track.Source.ScreenShare);
const videoTrack = preferredTrack ?? publication?.videoTrack;
const mediaStreamTrack = videoTrack?.mediaStreamTrack;
@@ -730,15 +727,11 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
};
const onMuted = (): void => reportCapturePaused(true);
const onUnmuted = (): void => reportCapturePaused(false);
const tracksCapturePauses = this.screenShareDeliveryArmed;
mediaStreamTrack.addEventListener('ended', onEnded);
if (tracksCapturePauses) {
mediaStreamTrack.addEventListener('mute', onMuted);
mediaStreamTrack.addEventListener('unmute', onUnmuted);
}
mediaStreamTrack.addEventListener('mute', onMuted);
mediaStreamTrack.addEventListener('unmute', onUnmuted);
this.activeScreenShareEndListener = () => {
mediaStreamTrack.removeEventListener('ended', onEnded);
if (!tracksCapturePauses) return;
mediaStreamTrack.removeEventListener('mute', onMuted);
mediaStreamTrack.removeEventListener('unmute', onUnmuted);
};
@@ -1152,7 +1145,7 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
nextConstraints.frameRate = {ideal: resolution.frameRate, max: resolution.frameRate};
}
if (JSON.stringify(currentConstraints) === JSON.stringify(nextConstraints)) return 'unchanged';
if (this.screenShareDeliveryArmed && screenShareSourceHasClonedTracks(screenShareTrack)) {
if (screenShareSourceHasClonedTracks(screenShareTrack)) {
logger.warn('Screen share capture geometry needs a restart while a backup codec holds a clone of the source', {
resolution,
});
@@ -1202,7 +1195,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
this.ensureScreenShareKeepAliveSinkInternal(participant);
updateLocalParticipantFromRoom(room);
this.syncLocalScreenShareAudioStateInternal(participant, participant.isScreenShareEnabled);
if (!this.screenShareDeliveryArmed) return true;
return geometryUpdate !== 'needs-capture-restart' && geometryUpdate !== 'failed';
}
@@ -1225,7 +1217,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
}
resetStreamTracking(): void {
this.screenShareDeliveryArmed = false;
this.setScreenShareCapturePausedInternal(false);
this.clearScreenShareKeepAliveSinkInternal();
this.setStreamingPriorityInternal(false);
@@ -32,11 +32,6 @@ const voiceStates: Record<string, Record<string, Record<string, unknown>>> = {};
const settingsUpdate = vi.fn();
const openPremiumModal = vi.fn();
vi.mock('@app/features/voice/state/ScreenShareDeliveryRollout', () => ({
ScreenShareDeliveryRollout: {enabled: true},
default: {enabled: true},
}));
vi.mock('@app/features/voice/utils/GpuEncoderCapabilities', () => ({
getGpuEncoderReportSync: () => gpuReport,
getH264HardwareProfilesSync: () => h264HardwareProfiles,
@@ -6,13 +6,11 @@ import {
stopMediaTrack,
stopUnselectedStreamTracks,
} from '@app/features/voice/engine/voice_screen_share_manager/shared';
import ActiveScreenShareSource from '@app/features/voice/state/ActiveScreenShareSource';
import {rememberCapturedDisplayAudioTrack} from '@app/features/voice/utils/NativeAudioCaptureBridge';
import {ScreenShareAudioCaptureError} from '@app/features/voice/utils/ScreenShareAudioCaptureError';
import type {ScreenShareCaptureOptions} from 'livekit-client';
type DisplayMediaVideoConstraints = MediaTrackConstraints & {
colorSpace?: string;
cursor?: 'always' | 'motion' | 'never';
displaySurface?: 'browser' | 'monitor' | 'window';
};
@@ -20,37 +18,12 @@ type DisplayMediaAudioConstraints = MediaTrackConstraints & {
restrictOwnAudio?: boolean;
suppressLocalAudioPlayback?: boolean;
};
type DisplayMediaTrackSettings = MediaTrackSettings & {
cursor?: 'always' | 'motion' | 'never';
displaySurface?: 'browser' | 'monitor' | 'window';
};
function resolveDisplayMediaCursorCapture(
displaySurface: DisplayMediaVideoConstraints['displaySurface'],
): 'always' | 'motion' | 'never' {
return displaySurface === 'window' ? 'never' : 'always';
}
function getRequestedDisplayMediaVideoConstraints(
options: ScreenShareCaptureOptions | undefined,
): DisplayMediaVideoConstraints | null {
if (typeof options?.video !== 'object' || !options.video) return null;
return options.video as DisplayMediaVideoConstraints;
}
function resolveCapturedDisplayMediaCursorCapture(
track: Pick<MediaStreamTrack, 'getSettings'>,
options?: ScreenShareCaptureOptions,
): 'always' | 'motion' | 'never' {
const requestedVideo = getRequestedDisplayMediaVideoConstraints(options);
const requestedCursor = requestedVideo?.cursor;
const requestedDisplaySurface = requestedVideo?.displaySurface;
if (requestedCursor && requestedCursor !== resolveDisplayMediaCursorCapture(requestedDisplaySurface)) {
return requestedCursor;
}
const settings = track.getSettings() as DisplayMediaTrackSettings;
return resolveDisplayMediaCursorCapture(settings.displaySurface ?? requestedDisplaySurface);
}
export function getDisplayMediaOptions(options?: ScreenShareCaptureOptions): DisplayMediaStreamOptions {
let videoConstraints: MediaTrackConstraints | boolean = options?.video ?? true;
const resolution = options?.resolution;
@@ -95,21 +68,6 @@ export function getDisplayMediaOptions(options?: ScreenShareCaptureOptions): Dis
} as DisplayMediaStreamOptions;
}
function buildCapturedDisplayMediaConstraints(
displayMediaOptions: DisplayMediaStreamOptions,
cursor: 'always' | 'motion' | 'never',
): MediaTrackConstraints {
const requestedVideo =
typeof displayMediaOptions.video === 'object' && displayMediaOptions.video
? (displayMediaOptions.video as DisplayMediaVideoConstraints)
: undefined;
const constraints: DisplayMediaVideoConstraints = {colorSpace: 'rec709', cursor};
if (requestedVideo?.width !== undefined) constraints.width = requestedVideo.width;
if (requestedVideo?.height !== undefined) constraints.height = requestedVideo.height;
if (requestedVideo?.frameRate !== undefined) constraints.frameRate = requestedVideo.frameRate;
return constraints;
}
export async function createDisplayScreenShareTracks(
options?: ScreenShareCaptureOptions,
captureContext?: DisplayScreenShareCaptureContext,
@@ -127,12 +85,6 @@ export async function createDisplayScreenShareTracks(
if (options?.contentHint) {
videoTrack.contentHint = options.contentHint;
}
if (ActiveScreenShareSource.getTarget()?.delivery !== true) {
const cursor = resolveCapturedDisplayMediaCursorCapture(videoTrack, options);
await videoTrack
.applyConstraints(buildCapturedDisplayMediaConstraints(displayMediaOptions, cursor))
.catch(() => undefined);
}
const capturedAudioTrack = stream.getAudioTracks()[0];
const audioTrack = capturedAudioTrack?.readyState === 'live' ? capturedAudioTrack : undefined;
rememberCapturedDisplayAudioTrack(capturedAudioTrack, Boolean(capturedAudioTrack) && !audioTrack);
@@ -4,7 +4,6 @@ import {getDesktopTroubleshootingSettings} from '@app/features/devtools/utils/De
import {Logger} from '@app/features/platform/utils/AppLogger';
import ScreenShareCodecNegotiation from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import ActiveScreenShareSource from '@app/features/voice/state/ActiveScreenShareSource';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import SoftwareEncoderWarning from '@app/features/voice/state/SoftwareEncoderWarning';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {
@@ -23,7 +22,6 @@ import {
resolveScreenShareLayering,
resolveScreenShareSenderCodec,
resolveScreenShareTarget,
SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS,
SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
type ScreenShareContext,
type ScreenShareLayering,
@@ -195,11 +193,13 @@ export async function releaseScreenShareCaptureCleanup(snapshot: ScreenShareCapt
}
}
function clampScreenShareEncoding(encoding: VideoEncoding, delivery: boolean): VideoEncoding {
const ceiling = delivery ? SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS : SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS;
function clampScreenShareEncoding(encoding: VideoEncoding): VideoEncoding {
return {
...encoding,
maxBitrate: typeof encoding.maxBitrate === 'number' ? Math.min(encoding.maxBitrate, ceiling) : encoding.maxBitrate,
maxBitrate:
typeof encoding.maxBitrate === 'number'
? Math.min(encoding.maxBitrate, SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS)
: encoding.maxBitrate,
priority: encoding.priority ?? 'high',
};
}
@@ -208,15 +208,10 @@ export function resolveActiveScreenShareContext(): ScreenShareContext {
return ActiveScreenShareSource.getShareContext() ?? 'display';
}
function resolveScreenShareDeliveryArm(): boolean {
return ActiveScreenShareSource.getTarget()?.delivery ?? ScreenShareDeliveryRollout.enabled;
}
export function resolveConfiguredScreenShareTarget(
context: ScreenShareContext,
sourceDimensions: {width: number; height: number} | null,
): ScreenShareTarget {
const delivery = resolveScreenShareDeliveryArm();
return resolveScreenShareTarget({
mode: VoiceSettings.getStreamingMode(),
storedResolution: VoiceSettings.getScreenshareResolution(),
@@ -225,13 +220,8 @@ export function resolveConfiguredScreenShareTarget(
context,
sourceDimensions,
hintSetting: VoiceSettings.getScreenShareContentHint(),
delivery,
...(delivery
? {
codec: getPreferredScreenShareCodec(),
softwareEncoderClamp: ActiveScreenShareSource.isSoftwareEncoderClamped(),
}
: {}),
codec: getPreferredScreenShareCodec(),
softwareEncoderClamp: ActiveScreenShareSource.isSoftwareEncoderClamped(),
});
}
@@ -242,7 +232,6 @@ export function resolveActiveScreenShareTarget(
}
function recommitClampedScreenShareTarget(committed: ScreenShareTarget): ScreenShareTarget {
if (committed.delivery !== true) return committed;
if (committed.softwareEncoderClamped) return committed;
const resolved = resolveActiveScreenShareTarget(committed.context);
if (!resolved.softwareEncoderClamped) return committed;
@@ -270,7 +259,7 @@ export function getStatsKind(
function resolveScreenShareEncoding(target: ScreenShareTarget, publishOptions?: TrackPublishOptions): VideoEncoding {
if (publishOptions?.screenShareEncoding) {
return clampScreenShareEncoding(publishOptions.screenShareEncoding, target.delivery === true);
return clampScreenShareEncoding(publishOptions.screenShareEncoding);
}
return {maxBitrate: target.maxBitrate, maxFramerate: target.frameRate, priority: 'high'};
}
@@ -863,27 +852,22 @@ export function startScreenShareEncoderMonitor(options: ScreenShareEncoderMonito
let framesEncoded: number | null = null;
let sendSnapshot: ScreenShareSendSnapshot | null = null;
let cpuLimitedTicks = 0;
const delivery = ActiveScreenShareSource.getTarget()?.delivery === true;
const tick = async (): Promise<void> => {
const sender = options.track.sender;
if (!sender) return;
const stats = await sender.getStats();
if (delivery) {
syncScreenShareCaptureSize(sender);
}
syncScreenShareCaptureSize(sender);
const encoded = countEncodedVideoFrames(stats);
ActiveScreenShareSource.setEncoding(encoded !== null && framesEncoded !== null && encoded > framesEncoded);
framesEncoded = encoded;
if (delivery) {
const snapshot = collectScreenShareSendSnapshot(stats);
cpuLimitedTicks = snapshot.cpuLimited ? cpuLimitedTicks + 1 : 0;
const limit = classifyScreenShareSendLimit(sendSnapshot, snapshot, cpuLimitedTicks);
sendSnapshot = snapshot;
if (limit !== ActiveScreenShareSource.getLimit()) {
logger.info('Screen share send limit changed', {limit, codec: options.codec});
}
ActiveScreenShareSource.setLimit(limit);
const snapshot = collectScreenShareSendSnapshot(stats);
cpuLimitedTicks = snapshot.cpuLimited ? cpuLimitedTicks + 1 : 0;
const limit = classifyScreenShareSendLimit(sendSnapshot, snapshot, cpuLimitedTicks);
sendSnapshot = snapshot;
if (limit !== ActiveScreenShareSource.getLimit()) {
logger.info('Screen share send limit changed', {limit, codec: options.codec});
}
ActiveScreenShareSource.setLimit(limit);
if (!verified) {
const verification = verifyScreenShareEncoderStart(stats, options.codec, options.onEncodeFailure);
if (verification === 'failed') return;
@@ -907,8 +891,6 @@ export function startScreenShareEncoderMonitor(options: ScreenShareEncoderMonito
cancelled = true;
clearTimeout(timer);
ActiveScreenShareSource.setEncoding(false);
if (delivery) {
ActiveScreenShareSource.setLimit(null);
}
ActiveScreenShareSource.setLimit(null);
};
}