feat(voice): ship the screen share delivery rework to everyone (#2931)

This commit is contained in:
Hampus authored and GitHub committed 2026-09-24 14:57:40 +02:00
1 parent 88038a1d5b
commit a0825e77c4
53 files changed
+171 -1806

No files matched your search

@@ -6,7 +6,6 @@ import type {VoiceNoiseSuppressionAssignmentResponse} from '@fluxer/schema/src/d
import {
type ExperimentAssignmentsResponse,
INERT_EXPERIMENT_ASSIGNMENTS_RESPONSE,
readScreenShareDeliveryAssignment,
readVoiceNoiseSuppressionAssignment,
} from '@fluxer/schema/src/domains/experiment/ExperimentSchemas';
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
@@ -123,8 +122,6 @@ describe('ExperimentAssignments cold start', () => {
await settle();
expect(ExperimentAssignments.response).toBe(INERT_EXPERIMENT_ASSIGNMENTS_RESPONSE);
expect(ExperimentAssignments.response.assignments.voice_noise_suppression).toBeUndefined();
expect(ExperimentAssignments.response.assignments.screen_share_delivery).toBeUndefined();
expect(readScreenShareDeliveryAssignment(ExperimentAssignments.response).enabled).toBe(false);
});
it('keeps the inert envelope while unauthenticated and retries later', async () => {
@@ -170,32 +167,6 @@ describe('ExperimentAssignments response handling', () => {
expect(ExperimentAssignments.response).toEqual(CANARY_ENVELOPE);
});
it('adopts a screen share delivery assignment beside the voice one', async () => {
await adopt({
...CANARY_ENVELOPE,
assignments: {...CANARY_ENVELOPE.assignments, screen_share_delivery: {enabled: true}},
});
expect(readScreenShareDeliveryAssignment(ExperimentAssignments.response).enabled).toBe(true);
expect(readVoiceNoiseSuppressionAssignment(ExperimentAssignments.response)).toEqual(CANARY_ASSIGNMENT);
});
it('reads screen share delivery as disabled when the envelope omits it', async () => {
await adopt(CANARY_ENVELOPE);
expect(readScreenShareDeliveryAssignment(ExperimentAssignments.response).enabled).toBe(false);
});
it('discards an envelope with a malformed screen share delivery assignment', async () => {
await adopt(CANARY_ENVELOPE);
vi.mocked(http.get).mockResolvedValue(
reply(200, {
...CANARY_ENVELOPE,
assignments: {...CANARY_ENVELOPE.assignments, screen_share_delivery: {enabled: 'yes'}},
}),
);
await vi.advanceTimersByTimeAsync(400_000);
expect(ExperimentAssignments.response).toEqual(CANARY_ENVELOPE);
});
it('accepts an envelope that carries no voice noise suppression assignment', async () => {
await adopt({poll_interval_seconds: 600, poll_jitter_percent: 0, assignments: {}});
expect(ExperimentAssignments.response.assignments.voice_noise_suppression).toBeUndefined();
@@ -2,25 +2,18 @@
import {videoElementHasRenderedFrame} from '@app/features/voice/components/VideoElementFrameState';
import {Store} from '@app/features/voice/engine/Store';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
export const LAST_FRAME_SNAPSHOTS_MAX = 8;
export const LAST_FRAME_SNAPSHOT_WIDTH_MAX = 640;
export const LAST_FRAME_SNAPSHOT_JPEG_QUALITY = 0.7;
const LAST_FRAME_SNAPSHOT_LEGACY_WIDTH_MAX = 1280;
function computeSnapshotDimensions(
sourceWidth: number,
sourceHeight: number,
widthMax: number,
): {width: number; height: number} {
if (sourceWidth <= widthMax) {
function computeSnapshotDimensions(sourceWidth: number, sourceHeight: number): {width: number; height: number} {
if (sourceWidth <= LAST_FRAME_SNAPSHOT_WIDTH_MAX) {
return {width: sourceWidth, height: sourceHeight};
}
const scale = widthMax / sourceWidth;
const scale = LAST_FRAME_SNAPSHOT_WIDTH_MAX / sourceWidth;
return {
width: widthMax,
width: LAST_FRAME_SNAPSHOT_WIDTH_MAX,
height: Math.max(1, Math.round(sourceHeight * scale)),
};
}
@@ -29,12 +22,11 @@ function drawSourceToOffscreenCanvas(
source: CanvasImageSource,
sourceWidth: number,
sourceHeight: number,
widthMax: number,
): OffscreenCanvas | null {
if (typeof OffscreenCanvas === 'undefined') return null;
if (sourceWidth <= 0 || sourceHeight <= 0) return null;
try {
const {width, height} = computeSnapshotDimensions(sourceWidth, sourceHeight, widthMax);
const {width, height} = computeSnapshotDimensions(sourceWidth, sourceHeight);
const canvas = new OffscreenCanvas(width, height);
const context = canvas.getContext('2d');
if (!context) return null;
@@ -45,16 +37,11 @@ function drawSourceToOffscreenCanvas(
}
}
function drawSourceToDataUrl(
source: CanvasImageSource,
sourceWidth: number,
sourceHeight: number,
widthMax: number,
): string | null {
function drawSourceToDataUrl(source: CanvasImageSource, sourceWidth: number, sourceHeight: number): string | null {
if (typeof document === 'undefined') return null;
if (sourceWidth <= 0 || sourceHeight <= 0) return null;
try {
const {width, height} = computeSnapshotDimensions(sourceWidth, sourceHeight, widthMax);
const {width, height} = computeSnapshotDimensions(sourceWidth, sourceHeight);
const canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
@@ -114,38 +101,29 @@ class LastFrameSnapshotCache extends Store {
captureFromVideoElement(key: string, video: HTMLVideoElement | null): void {
if (!key) return;
const deliveryEnabled = ScreenShareDeliveryRollout.enabled;
if (deliveryEnabled && typeof document !== 'undefined' && document.visibilityState === 'hidden') return;
if (typeof document !== 'undefined' && document.visibilityState === 'hidden') return;
if (!videoElementHasRenderedFrame(video)) return;
const renderedVideo = video as HTMLVideoElement;
const widthMax = deliveryEnabled ? LAST_FRAME_SNAPSHOT_WIDTH_MAX : LAST_FRAME_SNAPSHOT_LEGACY_WIDTH_MAX;
if (deliveryEnabled) {
const captureId = this.nextCaptureId;
this.nextCaptureId += 1;
this.captureIds.set(key, captureId);
const canvas = drawSourceToOffscreenCanvas(
renderedVideo,
renderedVideo.videoWidth,
renderedVideo.videoHeight,
widthMax,
);
if (canvas) {
void canvas
.convertToBlob({type: 'image/jpeg', quality: LAST_FRAME_SNAPSHOT_JPEG_QUALITY})
.then((blob) => {
if (this.captureIds.get(key) !== captureId) return;
this.captureIds.delete(key);
this.retainSnapshot(key, URL.createObjectURL(blob));
})
.catch(() => {
if (this.captureIds.get(key) !== captureId) return;
this.captureIds.delete(key);
});
return;
}
this.captureIds.delete(key);
const captureId = this.nextCaptureId;
this.nextCaptureId += 1;
this.captureIds.set(key, captureId);
const canvas = drawSourceToOffscreenCanvas(renderedVideo, renderedVideo.videoWidth, renderedVideo.videoHeight);
if (canvas) {
void canvas
.convertToBlob({type: 'image/jpeg', quality: LAST_FRAME_SNAPSHOT_JPEG_QUALITY})
.then((blob) => {
if (this.captureIds.get(key) !== captureId) return;
this.captureIds.delete(key);
this.retainSnapshot(key, URL.createObjectURL(blob));
})
.catch(() => {
if (this.captureIds.get(key) !== captureId) return;
this.captureIds.delete(key);
});
return;
}
const dataUrl = drawSourceToDataUrl(renderedVideo, renderedVideo.videoWidth, renderedVideo.videoHeight, widthMax);
this.captureIds.delete(key);
const dataUrl = drawSourceToDataUrl(renderedVideo, renderedVideo.videoWidth, renderedVideo.videoHeight);
if (!dataUrl) return;
this.retainSnapshot(key, dataUrl);
}
@@ -11,7 +11,6 @@ import {
getLocalScreenShareVideoPublications,
isLiveLocalTrackPublication,
} from '@app/features/voice/engine/VoiceTrackPublicationUtils';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {
buildScreenShareCodecProfile,
@@ -357,7 +356,6 @@ class ScreenShareCodecNegotiation {
encoderMode: ScreenShareEncoderMode;
order: ReadonlyArray<VideoCodec>;
} | null = null;
private frozenDelivery: {trackSid: string; enabled: boolean} | null = null;
private lastCodecChangeAt = 0;
private suppressionTimer: ReturnType<typeof setTimeout> | null = null;
private publishedTrackSid: string | null = null;
@@ -452,22 +450,9 @@ class ScreenShareCodecNegotiation {
this.localCodecs = buildLocalCodecAdvertisements(this.resolveCodecPreferenceOrder());
}
private resolveScreenShareDelivery(room: Room | null = this.room): boolean {
const trackSid = this.getLocalScreenSharePublication(room)?.trackSid ?? null;
if (trackSid === null) {
this.frozenDelivery = null;
return ScreenShareDeliveryRollout.enabled;
}
if (this.frozenDelivery?.trackSid === trackSid) return this.frozenDelivery.enabled;
const enabled = ScreenShareDeliveryRollout.enabled;
this.frozenDelivery = {trackSid, enabled};
return enabled;
}
private resolveCodecPreferenceOrder(
preference: CodecPreference = VoiceSettings.getPreferredScreenShareCodec(),
): ReadonlyArray<VideoCodec> {
if (!this.resolveScreenShareDelivery()) return getScreenShareCodecPreferenceOrder(preference);
const encoderMode = VoiceSettings.getScreenShareEncoderMode();
const frozen = this.frozenCodecPreference;
if (
@@ -523,7 +508,7 @@ class ScreenShareCodecNegotiation {
unknownParticipants: number;
} {
const now = Date.now();
const viewerIdentities = this.resolveScreenShareDelivery(room) ? this.getScreenShareViewerIdentities(room) : null;
const viewerIdentities = this.getScreenShareViewerIdentities(room);
const knownRemoteCodecs: Array<Array<FluxerCodecAdvertisement>> = [];
const participants: Array<{identity: string; firstSeenAt: number}> = [];
for (const participant of room?.remoteParticipants.values() ?? []) {
@@ -640,7 +625,6 @@ class ScreenShareCodecNegotiation {
this.selectedCodec = null;
this.localCodecs = [];
this.frozenCodecPreference = null;
this.frozenDelivery = null;
this.lastCodecChangeAt = 0;
this.publishedTrackSid = null;
this.clearSuppressionTimer();
@@ -749,7 +733,6 @@ class ScreenShareCodecNegotiation {
const {knownRemoteCodecs, unknownParticipants} = this.getRemoteCodecInputs(room);
this.scheduleGraceReevaluations(room, bindingRevision);
const previousCodec = this.selectedCodec;
const delivery = this.resolveScreenShareDelivery(room);
this.negotiationSnapshot = transitionScreenShareCodecNegotiationSnapshot(this.negotiationSnapshot, {
type: 'negotiation.evaluate',
localCodecs: this.localCodecs,
@@ -757,7 +740,7 @@ class ScreenShareCodecNegotiation {
unknownParticipants,
reason,
codecPreference: this.resolveCodecPreferenceOrder(),
publishedCodec: delivery ? this.observePublishedScreenShareCodec(room) : null,
publishedCodec: this.observePublishedScreenShareCodec(room),
});
const selection = this.negotiationSnapshot.context.selection;
if (!selection) return null;
@@ -765,11 +748,7 @@ class ScreenShareCodecNegotiation {
this.selectedCodec = selection.codec;
return selection;
}
if (
delivery &&
previousCodec !== null &&
Date.now() - this.lastCodecChangeAt < SCREEN_SHARE_CODEC_CHANGE_SUPPRESSION_MS
) {
if (previousCodec !== null && Date.now() - this.lastCodecChangeAt < SCREEN_SHARE_CODEC_CHANGE_SUPPRESSION_MS) {
logger.debug('Suppressed a screen share codec change inside the change window', {
codec: selection.codec,
previousCodec,
@@ -779,7 +758,7 @@ class ScreenShareCodecNegotiation {
return selection;
}
this.selectedCodec = selection.codec;
if (delivery) this.armCodecChangeSuppression();
this.armCodecChangeSuppression();
this.mediaSessionId = createId('media');
logger.info('Selected screen share codec from XState capability intersection', selection);
await this.publishSessionUpdate(room, selection);
@@ -9,12 +9,7 @@ import {
type VoiceGatewayStateEvent,
type VoiceGatewayStateSnapshot,
} from '@app/features/voice/engine/VoiceGatewayStateMachine';
import {describe, expect, it, vi} from 'vitest';
vi.mock('@app/features/voice/state/ScreenShareDeliveryRollout', () => ({
ScreenShareDeliveryRollout: {enabled: false},
default: {enabled: false},
}));
import {describe, expect, it} from 'vitest';
function voiceState(overrides: Partial<VoiceState> = {}): VoiceState {
return {
@@ -19,12 +19,7 @@ import {
type VoiceMediaGraphStreamTileState,
} from '@app/features/voice/engine/VoiceMediaGraphTileState';
import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
import {describe, expect, it, vi} from 'vitest';
vi.mock('@app/features/voice/state/ScreenShareDeliveryRollout', () => ({
ScreenShareDeliveryRollout: {enabled: false},
default: {enabled: false},
}));
import {describe, expect, it} from 'vitest';
const SIMULATION_SEEDS: ReadonlyArray<number> = Array.from({length: 20}, (_, index) => 1 + index * 7919);
const SIMULATION_STEP_COUNT = 1000;
@@ -11,7 +11,6 @@ import {
voiceMediaGraphStatsTrackKey,
} from '@app/features/voice/engine/VoiceMediaGraphStatsObservations';
import type {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import type {VoiceEngineV2PerTrackStats} from '@fluxer/voice_engine_v2';
const VOICE_MEDIA_GRAPH_STATS_TRACK_LIMIT = 1024;
@@ -77,9 +76,7 @@ function perTrackStatToInfo(track: VoiceEngineV2PerTrackStats): VoiceMediaGraphP
if (track.kind !== 'video') return null;
const width = track.frameWidth ?? track.sourceFrameWidth;
const height = track.frameHeight ?? track.sourceFrameHeight;
const fps = ScreenShareDeliveryRollout.enabled
? (track.effectiveFramesPerSecond ?? track.framesPerSecond)
: (track.effectiveFramesPerSecond ?? track.framesPerSecond ?? track.sourceFramesPerSecond);
const fps = track.effectiveFramesPerSecond ?? track.framesPerSecond;
const info: VoiceMediaGraphPartialTrackInfo = {};
if (isPositiveDimension(width) && isPositiveDimension(height)) {
info.width = width;
@@ -193,9 +190,7 @@ function observationToPartialTrackInfo(
): VoiceMediaGraphPartialTrackInfo | null {
const width = observation.width ?? observation.sourceWidth ?? undefined;
const height = observation.height ?? observation.sourceHeight ?? undefined;
const fps = ScreenShareDeliveryRollout.enabled
? (observation.fps ?? undefined)
: (observation.fps ?? observation.sourceFps ?? undefined);
const fps = observation.fps ?? undefined;
const info: VoiceMediaGraphPartialTrackInfo = {};
if (isPositiveDimension(width) && isPositiveDimension(height)) {
info.width = width;
@@ -16,7 +16,6 @@ import {
} from '@app/features/voice/engine/VoiceStreamWatchState';
import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
import ParticipantVolume from '@app/features/voice/state/ParticipantVolume';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import {ScreenShareWatchErrorCode, ScreenShareWatchFailures} from '@app/features/voice/state/ScreenShareWatchFailures';
import {monitorScreenShareDecodeHealth} from '@app/features/voice/utils/ScreenShareCodecDiagnostics';
import {markScreenShareDecodeFailure} from '@app/features/voice/utils/VideoDecoderCapabilities';
@@ -108,30 +107,6 @@ interface ParticipantSpeakingDisposer {
dispose: () => void;
}
type LatencyTunedReceiver = RTCRtpReceiver & {
jitterBufferTarget?: number;
playoutDelayHint?: number;
};
function getRemoteTrackReceiver(track: RemoteTrack): LatencyTunedReceiver | undefined {
return (track as RemoteTrack & {receiver?: LatencyTunedReceiver}).receiver;
}
function applyInteractiveReceiverBuffer(track: RemoteTrack, pub: RemoteTrackPublication): void {
const receiver = getRemoteTrackReceiver(track);
if (!receiver) return;
try {
if (pub.kind === Track.Kind.Video && pub.source === Track.Source.ScreenShare) {
receiver.jitterBufferTarget = 80;
receiver.playoutDelayHint = 0.04;
} else if (pub.kind === Track.Kind.Audio) {
receiver.jitterBufferTarget = 60;
}
} catch (error) {
logger.debug('Failed to apply interactive receiver buffer target', {error, source: pub.source, kind: pub.kind});
}
}
export function bindRoomEvents(
room: Room,
attemptId: number,
@@ -141,7 +116,6 @@ export function bindRoomEvents(
dependencies: RoomEventDependencies,
): void {
const guard = dependencies.connection.createGuardedHandler;
const screenShareDeliveryEnabled = ScreenShareDeliveryRollout.enabled;
const participantSpeakingDisposers = new Map<string, ParticipantSpeakingDisposer>();
const screenShareDecodeMonitorCancels = new Map<string, () => void>();
const remoteTrackLifecycleDisposers = new Map<string, () => void>();
@@ -443,9 +417,6 @@ export function bindRoomEvents(
trackSid: pub.trackSid,
});
}
if (!screenShareDeliveryEnabled) {
applyInteractiveReceiverBuffer(track, pub);
}
monitorDecodeHealth(track, pub);
dependencies.remoteSpeaking.attachIfApplicable(participant, pub, track);
bindRemoteTrackLifecycleIfApplicable(participant, pub, track);
@@ -1,7 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import assert from 'node:assert/strict';
import {isElectronPlatform} from '@app/features/platform/types/Platform';
import {Logger} from '@app/features/platform/utils/AppLogger';
import {Store} from '@app/features/voice/engine/Store';
import {sendVoiceStateDisconnect} from '@app/features/voice/engine/VoiceChannelConnector';
@@ -37,7 +36,6 @@ import {
isReadyToRepublishTrack,
} from '@app/features/voice/engine/v2/VoiceEngineV2AppAdapterAssertions';
import {VoiceEngineV2AppReconnectPolicy} from '@app/features/voice/engine/v2/VoiceEngineV2AppReconnectPolicy';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import VoiceRegionTeleport from '@app/features/voice/state/VoiceRegionTeleport';
import {
findVideoPublishCodecPolicyViolation,
@@ -45,10 +43,7 @@ import {
} from '@app/features/voice/utils/CodecCapabilityDetector';
import {getH264HardwareProfilesSync} from '@app/features/voice/utils/GpuEncoderCapabilities';
import {setNoiseSuppressionScopeGuildId} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionSelection';
import {
SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS,
SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
} from '@app/features/voice/utils/ScreenShareOptions';
import {SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS} from '@app/features/voice/utils/ScreenShareOptions';
import {
clearScreenShareDecodeFailures,
getVideoDecoderExclusionsSync,
@@ -151,9 +146,7 @@ function createWebAudioMixOption(): RoomOptions['webAudioMix'] {
function createRoomPublishDefaults(): RoomOptions['publishDefaults'] {
return {
screenShareEncoding: {
maxBitrate: ScreenShareDeliveryRollout.enabled
? SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS
: SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
maxBitrate: SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
maxFramerate: 30,
priority: 'high',
},
@@ -164,7 +157,6 @@ function createRoomPublishDefaults(): RoomOptions['publishDefaults'] {
function createRoomOptions(
e2eeKey: string | null,
subscriberVideoCodecExclusions: RoomOptions['subscriberVideoCodecExclusions'],
screenShareDelivery: boolean,
): {
roomOptions: RoomOptions;
e2eeKeyProvider: ExternalE2EEKeyProvider | null;
@@ -176,8 +168,7 @@ function createRoomOptions(
webAudioMix: createWebAudioMixOption(),
publishDefaults: createRoomPublishDefaults(),
subscriberVideoCodecExclusions,
screenShareDelivery,
h264HardwareProfiles: screenShareDelivery ? getH264HardwareProfilesSync()?.profiles : undefined,
h264HardwareProfiles: getH264HardwareProfilesSync()?.profiles,
};
let e2eeKeyProvider: ExternalE2EEKeyProvider | null = null;
let e2eeWorker: Worker | null = null;
@@ -196,19 +187,11 @@ function createRoomOptions(
return {roomOptions, e2eeKeyProvider, e2eeWorker};
}
function createRoomConnectOptions(screenShareDelivery: boolean): RoomConnectOptions {
function createRoomConnectOptions(): RoomConnectOptions {
const connectOptions: RoomConnectOptions = {
autoSubscribe: false,
};
assert.equal(connectOptions.autoSubscribe, false, 'LiveKit connect options must not auto-subscribe');
if (!screenShareDelivery && isElectronPlatform()) {
connectOptions.rtcConfig = {iceTransportPolicy: 'relay'};
assert.equal(
connectOptions.rtcConfig.iceTransportPolicy,
'relay',
'Electron LiveKit connects must force relay ICE',
);
}
return connectOptions;
}
@@ -564,12 +547,7 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
logger.warn('Aborting LiveKit room creation after codec probing because attempt is stale', {attemptId});
return;
}
const screenShareDelivery = ScreenShareDeliveryRollout.enabled;
const {roomOptions, e2eeKeyProvider, e2eeWorker} = createRoomOptions(
e2eeKey,
subscriberVideoCodecExclusions,
screenShareDelivery,
);
const {roomOptions, e2eeKeyProvider, e2eeWorker} = createRoomOptions(e2eeKey, subscriberVideoCodecExclusions);
const room = new LiveKitRoom(roomOptions);
ownE2EEWorker(room, e2eeWorker);
let roomClosed = false;
@@ -602,7 +580,7 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
return;
}
logger.info('Attempting to connect to LiveKit', {endpoint, guildId, channelId: resolvedChannelId});
const connectOptions = createRoomConnectOptions(screenShareDelivery);
const connectOptions = createRoomConnectOptions();
room
.connect(endpoint, token, connectOptions)
.then(() => {
@@ -704,14 +682,12 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
const connectionId = raw.connection_id ?? null;
this.abortHotSwap();
const cachedExclusions = getVideoDecoderExclusionsSync();
const screenShareDelivery = ScreenShareDeliveryRollout.enabled;
const roomOptions: RoomOptions = {
adaptiveStream: false,
dynacast: true,
webAudioMix: createWebAudioMixOption(),
publishDefaults: createRoomPublishDefaults(),
subscriberVideoCodecExclusions: cachedExclusions && cachedExclusions.length > 0 ? cachedExclusions : undefined,
screenShareDelivery,
};
if (!this.isLatestConnectionAttempt(attemptId) || this.connectionState.room !== existingRoom) {
logger.warn('Region hot-swap: aborted before room creation because attempt is stale', {attemptId});
@@ -728,7 +704,7 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
this.abortHotSwap();
}
});
const connectOptions = createRoomConnectOptions(screenShareDelivery);
const connectOptions = createRoomConnectOptions();
logger.info('Region hot-swap: connecting to new endpoint', {endpoint, guildId, channelId});
newRoom
.connect(endpoint, token, connectOptions)
@@ -63,7 +63,6 @@ import ActiveScreenShareSource, {
type PublishedScreenShareSource,
} from '@app/features/voice/state/ActiveScreenShareSource';
import LocalVoiceState from '@app/features/voice/state/LocalVoiceState';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {
prepareHighFidelityScreenShareAudioTrack,
@@ -183,7 +182,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
encoderVerificationTimer: (() => void) | null = null;
private screenShareTrackingHolds = 0;
private screenShareCapturePaused = false;
private screenShareDeliveryArmed = false;
private screenShareTrackingGeneration = 0;
private readonly reconciledCodecPairs = new Set<string>();
private readonly verifiedCodecCorrectionsByTrack = new WeakMap<MediaStreamTrack, number>();
@@ -679,7 +677,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
preferredTrack?: LocalVideoTrack,
): void {
this.cleanupActiveScreenShareEndListenerInternal();
this.screenShareDeliveryArmed = ScreenShareDeliveryRollout.enabled;
const publication = preferredTrack ? undefined : participant.getTrackPublication(Track.Source.ScreenShare);
const videoTrack = preferredTrack ?? publication?.videoTrack;
const mediaStreamTrack = videoTrack?.mediaStreamTrack;
@@ -730,15 +727,11 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
};
const onMuted = (): void => reportCapturePaused(true);
const onUnmuted = (): void => reportCapturePaused(false);
const tracksCapturePauses = this.screenShareDeliveryArmed;
mediaStreamTrack.addEventListener('ended', onEnded);
if (tracksCapturePauses) {
mediaStreamTrack.addEventListener('mute', onMuted);
mediaStreamTrack.addEventListener('unmute', onUnmuted);
}
mediaStreamTrack.addEventListener('mute', onMuted);
mediaStreamTrack.addEventListener('unmute', onUnmuted);
this.activeScreenShareEndListener = () => {
mediaStreamTrack.removeEventListener('ended', onEnded);
if (!tracksCapturePauses) return;
mediaStreamTrack.removeEventListener('mute', onMuted);
mediaStreamTrack.removeEventListener('unmute', onUnmuted);
};
@@ -1152,7 +1145,7 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
nextConstraints.frameRate = {ideal: resolution.frameRate, max: resolution.frameRate};
}
if (JSON.stringify(currentConstraints) === JSON.stringify(nextConstraints)) return 'unchanged';
if (this.screenShareDeliveryArmed && screenShareSourceHasClonedTracks(screenShareTrack)) {
if (screenShareSourceHasClonedTracks(screenShareTrack)) {
logger.warn('Screen share capture geometry needs a restart while a backup codec holds a clone of the source', {
resolution,
});
@@ -1202,7 +1195,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
this.ensureScreenShareKeepAliveSinkInternal(participant);
updateLocalParticipantFromRoom(room);
this.syncLocalScreenShareAudioStateInternal(participant, participant.isScreenShareEnabled);
if (!this.screenShareDeliveryArmed) return true;
return geometryUpdate !== 'needs-capture-restart' && geometryUpdate !== 'failed';
}
@@ -1225,7 +1217,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
}
resetStreamTracking(): void {
this.screenShareDeliveryArmed = false;
this.setScreenShareCapturePausedInternal(false);
this.clearScreenShareKeepAliveSinkInternal();
this.setStreamingPriorityInternal(false);
@@ -32,11 +32,6 @@ const voiceStates: Record<string, Record<string, Record<string, unknown>>> = {};
const settingsUpdate = vi.fn();
const openPremiumModal = vi.fn();
vi.mock('@app/features/voice/state/ScreenShareDeliveryRollout', () => ({
ScreenShareDeliveryRollout: {enabled: true},
default: {enabled: true},
}));
vi.mock('@app/features/voice/utils/GpuEncoderCapabilities', () => ({
getGpuEncoderReportSync: () => gpuReport,
getH264HardwareProfilesSync: () => h264HardwareProfiles,
@@ -6,13 +6,11 @@ import {
stopMediaTrack,
stopUnselectedStreamTracks,
} from '@app/features/voice/engine/voice_screen_share_manager/shared';
import ActiveScreenShareSource from '@app/features/voice/state/ActiveScreenShareSource';
import {rememberCapturedDisplayAudioTrack} from '@app/features/voice/utils/NativeAudioCaptureBridge';
import {ScreenShareAudioCaptureError} from '@app/features/voice/utils/ScreenShareAudioCaptureError';
import type {ScreenShareCaptureOptions} from 'livekit-client';
type DisplayMediaVideoConstraints = MediaTrackConstraints & {
colorSpace?: string;
cursor?: 'always' | 'motion' | 'never';
displaySurface?: 'browser' | 'monitor' | 'window';
};
@@ -20,37 +18,12 @@ type DisplayMediaAudioConstraints = MediaTrackConstraints & {
restrictOwnAudio?: boolean;
suppressLocalAudioPlayback?: boolean;
};
type DisplayMediaTrackSettings = MediaTrackSettings & {
cursor?: 'always' | 'motion' | 'never';
displaySurface?: 'browser' | 'monitor' | 'window';
};
function resolveDisplayMediaCursorCapture(
displaySurface: DisplayMediaVideoConstraints['displaySurface'],
): 'always' | 'motion' | 'never' {
return displaySurface === 'window' ? 'never' : 'always';
}
function getRequestedDisplayMediaVideoConstraints(
options: ScreenShareCaptureOptions | undefined,
): DisplayMediaVideoConstraints | null {
if (typeof options?.video !== 'object' || !options.video) return null;
return options.video as DisplayMediaVideoConstraints;
}
function resolveCapturedDisplayMediaCursorCapture(
track: Pick<MediaStreamTrack, 'getSettings'>,
options?: ScreenShareCaptureOptions,
): 'always' | 'motion' | 'never' {
const requestedVideo = getRequestedDisplayMediaVideoConstraints(options);
const requestedCursor = requestedVideo?.cursor;
const requestedDisplaySurface = requestedVideo?.displaySurface;
if (requestedCursor && requestedCursor !== resolveDisplayMediaCursorCapture(requestedDisplaySurface)) {
return requestedCursor;
}
const settings = track.getSettings() as DisplayMediaTrackSettings;
return resolveDisplayMediaCursorCapture(settings.displaySurface ?? requestedDisplaySurface);
}
export function getDisplayMediaOptions(options?: ScreenShareCaptureOptions): DisplayMediaStreamOptions {
let videoConstraints: MediaTrackConstraints | boolean = options?.video ?? true;
const resolution = options?.resolution;
@@ -95,21 +68,6 @@ export function getDisplayMediaOptions(options?: ScreenShareCaptureOptions): Dis
} as DisplayMediaStreamOptions;
}
function buildCapturedDisplayMediaConstraints(
displayMediaOptions: DisplayMediaStreamOptions,
cursor: 'always' | 'motion' | 'never',
): MediaTrackConstraints {
const requestedVideo =
typeof displayMediaOptions.video === 'object' && displayMediaOptions.video
? (displayMediaOptions.video as DisplayMediaVideoConstraints)
: undefined;
const constraints: DisplayMediaVideoConstraints = {colorSpace: 'rec709', cursor};
if (requestedVideo?.width !== undefined) constraints.width = requestedVideo.width;
if (requestedVideo?.height !== undefined) constraints.height = requestedVideo.height;
if (requestedVideo?.frameRate !== undefined) constraints.frameRate = requestedVideo.frameRate;
return constraints;
}
export async function createDisplayScreenShareTracks(
options?: ScreenShareCaptureOptions,
captureContext?: DisplayScreenShareCaptureContext,
@@ -127,12 +85,6 @@ export async function createDisplayScreenShareTracks(
if (options?.contentHint) {
videoTrack.contentHint = options.contentHint;
}
if (ActiveScreenShareSource.getTarget()?.delivery !== true) {
const cursor = resolveCapturedDisplayMediaCursorCapture(videoTrack, options);
await videoTrack
.applyConstraints(buildCapturedDisplayMediaConstraints(displayMediaOptions, cursor))
.catch(() => undefined);
}
const capturedAudioTrack = stream.getAudioTracks()[0];
const audioTrack = capturedAudioTrack?.readyState === 'live' ? capturedAudioTrack : undefined;
rememberCapturedDisplayAudioTrack(capturedAudioTrack, Boolean(capturedAudioTrack) && !audioTrack);
@@ -4,7 +4,6 @@ import {getDesktopTroubleshootingSettings} from '@app/features/devtools/utils/De
import {Logger} from '@app/features/platform/utils/AppLogger';
import ScreenShareCodecNegotiation from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import ActiveScreenShareSource from '@app/features/voice/state/ActiveScreenShareSource';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import SoftwareEncoderWarning from '@app/features/voice/state/SoftwareEncoderWarning';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {
@@ -23,7 +22,6 @@ import {
resolveScreenShareLayering,
resolveScreenShareSenderCodec,
resolveScreenShareTarget,
SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS,
SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
type ScreenShareContext,
type ScreenShareLayering,
@@ -195,11 +193,13 @@ export async function releaseScreenShareCaptureCleanup(snapshot: ScreenShareCapt
}
}
function clampScreenShareEncoding(encoding: VideoEncoding, delivery: boolean): VideoEncoding {
const ceiling = delivery ? SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS : SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS;
function clampScreenShareEncoding(encoding: VideoEncoding): VideoEncoding {
return {
...encoding,
maxBitrate: typeof encoding.maxBitrate === 'number' ? Math.min(encoding.maxBitrate, ceiling) : encoding.maxBitrate,
maxBitrate:
typeof encoding.maxBitrate === 'number'
? Math.min(encoding.maxBitrate, SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS)
: encoding.maxBitrate,
priority: encoding.priority ?? 'high',
};
}
@@ -208,15 +208,10 @@ export function resolveActiveScreenShareContext(): ScreenShareContext {
return ActiveScreenShareSource.getShareContext() ?? 'display';
}
function resolveScreenShareDeliveryArm(): boolean {
return ActiveScreenShareSource.getTarget()?.delivery ?? ScreenShareDeliveryRollout.enabled;
}
export function resolveConfiguredScreenShareTarget(
context: ScreenShareContext,
sourceDimensions: {width: number; height: number} | null,
): ScreenShareTarget {
const delivery = resolveScreenShareDeliveryArm();
return resolveScreenShareTarget({
mode: VoiceSettings.getStreamingMode(),
storedResolution: VoiceSettings.getScreenshareResolution(),
@@ -225,13 +220,8 @@ export function resolveConfiguredScreenShareTarget(
context,
sourceDimensions,
hintSetting: VoiceSettings.getScreenShareContentHint(),
delivery,
...(delivery
? {
codec: getPreferredScreenShareCodec(),
softwareEncoderClamp: ActiveScreenShareSource.isSoftwareEncoderClamped(),
}
: {}),
codec: getPreferredScreenShareCodec(),
softwareEncoderClamp: ActiveScreenShareSource.isSoftwareEncoderClamped(),
});
}
@@ -242,7 +232,6 @@ export function resolveActiveScreenShareTarget(
}
function recommitClampedScreenShareTarget(committed: ScreenShareTarget): ScreenShareTarget {
if (committed.delivery !== true) return committed;
if (committed.softwareEncoderClamped) return committed;
const resolved = resolveActiveScreenShareTarget(committed.context);
if (!resolved.softwareEncoderClamped) return committed;
@@ -270,7 +259,7 @@ export function getStatsKind(
function resolveScreenShareEncoding(target: ScreenShareTarget, publishOptions?: TrackPublishOptions): VideoEncoding {
if (publishOptions?.screenShareEncoding) {
return clampScreenShareEncoding(publishOptions.screenShareEncoding, target.delivery === true);
return clampScreenShareEncoding(publishOptions.screenShareEncoding);
}
return {maxBitrate: target.maxBitrate, maxFramerate: target.frameRate, priority: 'high'};
}
@@ -863,27 +852,22 @@ export function startScreenShareEncoderMonitor(options: ScreenShareEncoderMonito
let framesEncoded: number | null = null;
let sendSnapshot: ScreenShareSendSnapshot | null = null;
let cpuLimitedTicks = 0;
const delivery = ActiveScreenShareSource.getTarget()?.delivery === true;
const tick = async (): Promise<void> => {
const sender = options.track.sender;
if (!sender) return;
const stats = await sender.getStats();
if (delivery) {
syncScreenShareCaptureSize(sender);
}
syncScreenShareCaptureSize(sender);
const encoded = countEncodedVideoFrames(stats);
ActiveScreenShareSource.setEncoding(encoded !== null && framesEncoded !== null && encoded > framesEncoded);
framesEncoded = encoded;
if (delivery) {
const snapshot = collectScreenShareSendSnapshot(stats);
cpuLimitedTicks = snapshot.cpuLimited ? cpuLimitedTicks + 1 : 0;
const limit = classifyScreenShareSendLimit(sendSnapshot, snapshot, cpuLimitedTicks);
sendSnapshot = snapshot;
if (limit !== ActiveScreenShareSource.getLimit()) {
logger.info('Screen share send limit changed', {limit, codec: options.codec});
}
ActiveScreenShareSource.setLimit(limit);
const snapshot = collectScreenShareSendSnapshot(stats);
cpuLimitedTicks = snapshot.cpuLimited ? cpuLimitedTicks + 1 : 0;
const limit = classifyScreenShareSendLimit(sendSnapshot, snapshot, cpuLimitedTicks);
sendSnapshot = snapshot;
if (limit !== ActiveScreenShareSource.getLimit()) {
logger.info('Screen share send limit changed', {limit, codec: options.codec});
}
ActiveScreenShareSource.setLimit(limit);
if (!verified) {
const verification = verifyScreenShareEncoderStart(stats, options.codec, options.onEncodeFailure);
if (verification === 'failed') return;
@@ -907,8 +891,6 @@ export function startScreenShareEncoderMonitor(options: ScreenShareEncoderMonito
cancelled = true;
clearTimeout(timer);
ActiveScreenShareSource.setEncoding(false);
if (delivery) {
ActiveScreenShareSource.setLimit(null);
}
ActiveScreenShareSource.setLimit(null);
};
}
@@ -70,11 +70,6 @@ class ActiveScreenShareSource {
this.encoding = false;
return;
}
if (target.delivery !== true) {
this.target = target;
this.encoding = false;
return;
}
const degradationPreference = this.frozenDegradationPreference ?? target.degradationPreference;
this.frozenDegradationPreference = degradationPreference;
this.softwareEncoderClamped = this.softwareEncoderClamped || target.softwareEncoderClamped;
@@ -1,22 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import ExperimentAssignments from '@app/features/experiment/state/ExperimentAssignments';
import {Logger} from '@app/features/platform/utils/AppLogger';
import {readScreenShareDeliveryAssignment} from '@fluxer/schema/src/domains/experiment/ExperimentSchemas';
const logger = new Logger('ScreenShareDeliveryRollout');
class ScreenShareDeliveryRolloutSelector {
get enabled(): boolean {
try {
return readScreenShareDeliveryAssignment(ExperimentAssignments.response).enabled;
} catch (err) {
logger.warn('Failed to resolve screen share delivery assignment:', err);
return false;
}
}
}
export const ScreenShareDeliveryRollout = new ScreenShareDeliveryRolloutSelector();
export default ScreenShareDeliveryRollout;
@@ -3,7 +3,6 @@
import AppStorage from '@app/features/platform/state/PersistentStorage';
import {Logger} from '@app/features/platform/utils/AppLogger';
import {makePersistent} from '@app/features/platform/utils/MobXPersistence';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import type {
CodecPreference,
ScreenShareContentHint,
@@ -263,9 +262,6 @@ function applyStreamingModeDefaultMigrationV1(parsed: Record<string, unknown>):
}
function applyScreenShareFrameRateFloorMigrationV1(parsed: Record<string, unknown>): boolean {
if (!ScreenShareDeliveryRollout.enabled) {
return false;
}
if (parsed.screenShareFrameRateFloorMigratedV1 === true) {
return false;
}
@@ -1,7 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import type {TrackPublishOptions} from 'livekit-client';
export const OPUS_MAX_AUDIO_BITRATE_BPS = 510000;
@@ -40,9 +39,7 @@ export function buildMicrophonePublishOptions(
export const SCREEN_SHARE_AUDIO_PUBLISH_OPTIONS: TrackPublishOptions = {
audioPreset: {
get maxBitrate(): number {
return ScreenShareDeliveryRollout.enabled ? SCREEN_SHARE_AUDIO_BITRATE_BPS : OPUS_MAX_AUDIO_BITRATE_BPS;
},
maxBitrate: SCREEN_SHARE_AUDIO_BITRATE_BPS,
priority: 'high',
},
dtx: false,
@@ -9,7 +9,6 @@ import {
isFirefoxBrowser,
type NativePlatform,
} from '@app/features/ui/utils/NativeUtils';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {getGpuEncoderReportSync, type HardwareEncodeAnswer} from '@app/features/voice/utils/GpuEncoderCapabilities';
import {
@@ -26,10 +25,7 @@ import {
type ScreenShareCodecRanking,
} from '@app/features/voice/utils/ScreenShareCodecSelection';
import {normaliseStreamingModeForContext} from '@app/features/voice/utils/ScreenShareOptions';
import {
getProbedVideoDecoderExclusionsSync,
getVideoDecoderExclusionsSync,
} from '@app/features/voice/utils/VideoDecoderCapabilities';
import {getProbedVideoDecoderExclusionsSync} from '@app/features/voice/utils/VideoDecoderCapabilities';
import type {TrackPublishDefaults, TrackPublishOptions} from 'livekit-client';
import {BackupCodecPolicy, supportsVideoCodec, type VideoCodec, type VideoEncoding} from 'livekit-client';
@@ -98,7 +94,6 @@ let cachedReportNativeHardwareEncoderKey: object | null | undefined;
let cachedReportHardwareAccelerationDisabled: boolean | undefined;
let cachedReportAv1OptIn: boolean | undefined;
let cachedReportHevcOptIn: boolean | undefined;
let cachedReportDelivery: boolean | undefined;
const runtimeEncodeFailureCodecs = new Set<VideoCodec>();
const observedSoftwareEncodeCodecs = new Set<VideoCodec>();
@@ -214,21 +209,15 @@ function getScreenShareCodecPolicyUnsupported(
return null;
}
function hasPublishPathNativeHardwareEncoder(
codec: VideoCodec,
context: CodecPolicyContext,
delivery: boolean,
): boolean {
function hasPublishPathNativeHardwareEncoder(codec: VideoCodec, context: CodecPolicyContext): boolean {
if (!hasNativeHardwareEncoder(codec)) return false;
const backend = getNativeHardwareEncoderCapabilitiesSync()?.backend;
if (!delivery) return backend !== 'videotoolbox' && backend !== 'nvenc';
if (backend === 'videotoolbox') return true;
if (backend === 'nvenc') return context.platform !== 'linux';
return false;
}
function buildReport(): CodecCapabilityReport {
const delivery = ScreenShareDeliveryRollout.enabled;
const {caps, probedSuccessfully} = probeRawCapabilities();
const context = buildScreenShareCodecPolicyContext();
const gpuReport = getGpuEncoderReportSync();
@@ -243,17 +232,11 @@ function buildReport(): CodecCapabilityReport {
if (observedSoftwareEncodeCodecs.has(codec)) {
return 'software';
}
if (!delivery) {
if (hasPublishPathNativeHardwareEncoder(codec, context, delivery)) {
return 'hardware';
}
return gpuReport ? gpuReport[codec] : 'unknown';
}
const measured = gpuReport ? gpuReport[codec] : 'unknown';
if (measured !== 'unknown') {
return measured;
}
return hasPublishPathNativeHardwareEncoder(codec, context, delivery) ? 'hardware' : 'unknown';
return hasPublishPathNativeHardwareEncoder(codec, context) ? 'hardware' : 'unknown';
}
type DescribedUnsupported = Omit<CodecSupportInfo, 'hardwareAccelerated'>;
function unsupported(codec: keyof CodecCapabilities, info: DescribedUnsupported): CodecSupportInfo {
@@ -271,8 +254,8 @@ function buildReport(): CodecCapabilityReport {
detail: 'This codec failed while publishing during the current session.',
});
}
const supportedByNativeHardware = hasPublishPathNativeHardwareEncoder(codec, context, delivery);
if (caps[codec] || (!delivery && supportedByNativeHardware)) {
const supportedByNativeHardware = hasPublishPathNativeHardwareEncoder(codec, context);
if (caps[codec]) {
return {
supported: true,
reason: 'supported',
@@ -348,10 +331,8 @@ export function getCodecCapabilityReport(): CodecCapabilityReport {
const hardwareAccelerationDisabled = isDesktopHardwareAccelerationDisabled();
const av1OptIn = VoiceSettings.getScreenShareAv1OptIn();
const hevcOptIn = VoiceSettings.getScreenShareHevcOptIn();
const delivery = ScreenShareDeliveryRollout.enabled;
if (
cachedReport &&
cachedReportDelivery === delivery &&
cachedReportGpuKey === currentGpu &&
cachedReportNativeHardwareEncoderKey === currentNativeHardwareEncoder &&
cachedReportHardwareAccelerationDisabled === hardwareAccelerationDisabled &&
@@ -365,7 +346,6 @@ export function getCodecCapabilityReport(): CodecCapabilityReport {
cachedReportHardwareAccelerationDisabled = hardwareAccelerationDisabled;
cachedReportAv1OptIn = av1OptIn;
cachedReportHevcOptIn = hevcOptIn;
cachedReportDelivery = delivery;
return cachedReport;
}
@@ -538,9 +518,7 @@ export function getVideoPublishCodecDenial(codec: VideoCodec): VideoPublishCodec
if (!supportsVideoCodec(codec)) return 'sender-cannot-encode';
if (getScreenShareCodecPolicyUnsupported(codec, buildScreenShareCodecPolicyContext())) return 'policy';
if (runtimeEncodeFailureCodecs.has(codec)) return 'runtime-failed';
const decoderExclusions = ScreenShareDeliveryRollout.enabled
? getProbedVideoDecoderExclusionsSync()
: getVideoDecoderExclusionsSync();
const decoderExclusions = getProbedVideoDecoderExclusionsSync();
if (decoderExclusions?.includes(codec) === true) return 'decoder-excluded';
return null;
}
@@ -701,7 +679,6 @@ export function resetCachedCodecCapabilities(): void {
cachedReportHardwareAccelerationDisabled = undefined;
cachedReportAv1OptIn = undefined;
cachedReportHevcOptIn = undefined;
cachedReportDelivery = undefined;
runtimeEncodeFailureCodecs.clear();
observedSoftwareEncodeCodecs.clear();
resetNativeHardwareEncoderCapabilities();
@@ -2,7 +2,6 @@
import {Logger} from '@app/features/platform/utils/AppLogger';
import {getElectronAPI, isDesktop} from '@app/features/ui/utils/NativeUtils';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import type {GpuDeviceInfo, GpuInfo} from '@app/types/electron.d';
import type {VideoCodec} from 'livekit-client';
@@ -238,12 +237,6 @@ export const H264_ENCODE_PROBE_CONTENT_TYPES: ReadonlyArray<string> = H264_PROBE
(profileLevelId) => `video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=${profileLevelId}`,
);
const CONTROL_H264_PROBE_PROFILE_LEVEL_IDS: ReadonlyArray<string> = ['640028', '4d0028', '420028', '42e028'];
const CONTROL_H264_ENCODE_PROBE_CONTENT_TYPES: ReadonlyArray<string> = CONTROL_H264_PROBE_PROFILE_LEVEL_IDS.map(
(profileLevelId) => `video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=${profileLevelId}`,
);
export const WEBRTC_ENCODE_PROBE_CONTENT_TYPES: Record<VideoCodec, ReadonlyArray<string>> = {
av1: ['video/AV1'],
h265: ['video/H265'],
@@ -415,24 +408,17 @@ export async function probeWebRtcEncodeEfficiency(
): Promise<Record<VideoCodec, HardwareEncodeAnswer> | null> {
const mediaCapabilities = getMediaCapabilities();
if (!mediaCapabilities) return null;
const delivery = ScreenShareDeliveryRollout.enabled;
const video = resolveEncodeProbeVideoConfig(config);
const codecs: ReadonlyArray<VideoCodec> = ['av1', 'h265', 'h264', 'vp9', 'vp8'];
const answers = await Promise.all(
codecs.map((codec) =>
probeContentTypesEncodeEfficiency(
mediaCapabilities,
codec === 'h264' && !delivery
? CONTROL_H264_ENCODE_PROBE_CONTENT_TYPES
: WEBRTC_ENCODE_PROBE_CONTENT_TYPES[codec],
video,
),
probeContentTypesEncodeEfficiency(mediaCapabilities, WEBRTC_ENCODE_PROBE_CONTENT_TYPES[codec], video),
),
);
const result = {} as Record<VideoCodec, HardwareEncodeAnswer>;
codecs.forEach((codec, index) => {
const codecAnswers = answers[index] ?? [];
if (delivery && codec === 'h264') recordH264HardwareProfileProbe(codecAnswers, video);
if (codec === 'h264') recordH264HardwareProfileProbe(codecAnswers, video);
result[codec] = collapseEncodeProbeAnswers(codecAnswers);
});
return result;
@@ -460,7 +446,6 @@ let pendingPromise: Promise<HardwareEncodeReport | null> | null = null;
function fetchReport(): Promise<HardwareEncodeReport | null> {
if (!isDesktop()) {
if (!ScreenShareDeliveryRollout.enabled) return Promise.resolve(null);
return probeH264HardwareProfiles().then(() => null);
}
const electron = getElectronAPI();
@@ -9,12 +9,7 @@ import {
markScreenShareDecodeFailure,
resetVideoDecoderExclusions,
} from '@app/features/voice/utils/VideoDecoderCapabilities';
import {afterEach, describe, expect, it, vi} from 'vitest';
vi.mock('@app/features/voice/state/ScreenShareDeliveryRollout', () => ({
ScreenShareDeliveryRollout: {enabled: true},
default: {enabled: true},
}));
import {afterEach, describe, expect, it} from 'vitest';
function buildStats(inbound: Record<string, unknown>): RTCStatsReport {
return new Map<string, unknown>([
@@ -1,14 +1,12 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Logger} from '@app/features/platform/utils/AppLogger';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import SoftwareEncoderWarning from '@app/features/voice/state/SoftwareEncoderWarning';
import {classifyVideoDecoderAcceleration} from '@app/features/voice/utils/VideoAccelerationClassification';
import type {VideoCodec} from 'livekit-client';
const logger = new Logger('ScreenShareCodecDiagnostics');
const DECODE_SAMPLE_INTERVAL_MS = 5000;
const DECODER_VERIFICATION_DELAY_MS = 5000;
const UNKNOWN_DECODER_IMPLEMENTATION = 'software decoder';
interface CodecStatsEntry {
@@ -144,97 +142,6 @@ function collectInboundVideoStats(stats: RTCStatsReport): {
return {codecs, reports};
}
function findSoftwareVideoDecoderInStats(stats: RTCStatsReport): SoftwareVideoDecoderInfo | null {
const {codecs, reports} = collectInboundVideoStats(stats);
for (const report of reports) {
if (getStatsKind(report, codecs) !== 'video') continue;
const implementation =
typeof report.decoderImplementation === 'string' && report.decoderImplementation.length > 0
? report.decoderImplementation
: null;
const powerEfficientDecoder =
typeof report.powerEfficientDecoder === 'boolean' ? report.powerEfficientDecoder : null;
if (!isSoftwareVideoStats(implementation, powerEfficientDecoder)) continue;
return {
codec: getCodecLabel(report.codecId ? codecs.get(report.codecId)?.mimeType : undefined),
implementation: implementation ?? UNKNOWN_DECODER_IMPLEMENTATION,
powerEfficientDecoder,
};
}
return null;
}
function findStalledVideoDecoderInStats(stats: RTCStatsReport): StalledVideoDecoderInfo | null {
const {codecs, reports} = collectInboundVideoStats(stats);
for (const report of reports) {
if (getStatsKind(report, codecs) !== 'video') continue;
const framesDecoded = finiteNumber(report.framesDecoded);
if (framesDecoded === null || framesDecoded > 0) continue;
const framesReceived = finiteNumber(report.framesReceived);
if (framesReceived === null || framesReceived < 1) continue;
const mimeType = report.codecId ? codecs.get(report.codecId)?.mimeType : undefined;
const codec = getVideoCodecFromMimeType(mimeType);
if (!codec) continue;
return {
codec,
mimeType,
packetsReceived: finiteNumber(report.packetsReceived) ?? 0,
bytesReceived: finiteNumber(report.bytesReceived) ?? 0,
framesDecoded,
framesReceived,
framesDropped: finiteNumber(report.framesDropped),
};
}
return null;
}
function scheduleScreenShareDecoderVerification(
getStats: () => Promise<RTCStatsReport | undefined>,
onDecodeFailure?: (failure: StalledVideoDecoderInfo) => void,
): () => void {
let cancelled = false;
let timer: ReturnType<typeof setTimeout> | null = null;
const scheduleConfirmation = (first: StalledVideoDecoderInfo): void => {
timer = setTimeout(async () => {
timer = null;
try {
const stats = await getStats();
const confirmed = confirmDecodeStall(first, stats ? findStalledVideoDecoderInStats(stats) : null);
if (cancelled || !confirmed) return;
logger.warn('Screen share video decode is stalled', confirmed);
onDecodeFailure?.(confirmed);
} catch (error) {
logger.debug('Failed to confirm the screen share decode stall', {error});
}
}, DECODER_VERIFICATION_DELAY_MS);
};
timer = setTimeout(async () => {
timer = null;
let firstStall: StalledVideoDecoderInfo | null = null;
try {
const stats = await getStats();
if (!stats) return;
firstStall = findStalledVideoDecoderInStats(stats);
const decoder = findSoftwareVideoDecoderInStats(stats);
if (!decoder) return;
logger.warn('Screen share is using a software decoder', decoder);
SoftwareEncoderWarning.triggerDecoderWarning(decoder.codec, decoder.implementation);
} catch (error) {
logger.debug('Failed to verify screen share decoder', {error});
} finally {
if (!cancelled && firstStall) {
scheduleConfirmation(firstStall);
}
}
}, DECODER_VERIFICATION_DELAY_MS);
return () => {
cancelled = true;
if (!timer) return;
clearTimeout(timer);
timer = null;
};
}
export function findInboundVideoDecodeSample(stats: RTCStatsReport): InboundVideoDecodeSample | null {
const {codecs, reports} = collectInboundVideoStats(stats);
for (const report of reports) {
@@ -338,7 +245,6 @@ export function monitorScreenShareDecodeHealth(
getStats: () => Promise<RTCStatsReport | undefined>,
onDecodeStall?: (failure: StalledVideoDecoderInfo) => void,
): () => void {
if (!ScreenShareDeliveryRollout.enabled) return scheduleScreenShareDecoderVerification(getStats, onDecodeStall);
let cancelled = false;
let timer: ReturnType<typeof setTimeout> | null = null;
let previousSample: InboundVideoDecodeSample | null = null;
@@ -8,14 +8,7 @@ import {
resolveScreenShareLayering,
resolveScreenShareTarget,
} from '@app/features/voice/utils/ScreenShareOptions';
import {afterEach, describe, expect, it, vi} from 'vitest';
const rollout = vi.hoisted(() => ({enabled: false}));
vi.mock('@app/features/voice/state/ScreenShareDeliveryRollout', () => ({
ScreenShareDeliveryRollout: rollout,
default: rollout,
}));
import {describe, expect, it, vi} from 'vitest';
vi.mock('@app/features/voice/utils/NativeAudioCaptureBridge', () => ({
rememberCapturedDisplayAudioTrack: () => undefined,
@@ -29,10 +22,6 @@ const {getDisplayMediaOptions} = await import(
'@app/features/voice/engine/voice_screen_share_manager/DisplayMediaCapture'
);
afterEach(() => {
rollout.enabled = false;
});
function collectKeys(value: unknown, keys: Set<string>): Set<string> {
if (typeof value !== 'object' || value === null) return keys;
for (const [key, entry] of Object.entries(value)) {
@@ -87,29 +76,19 @@ describe('display capture constraints', () => {
});
});
describe('the screen share delivery experiment', () => {
it('keeps the source preset and the 90 and 120 FPS rungs off the experiment', () => {
expect(targetOf()).toMatchObject({resolution: 'source', frameRate: 15});
expect(resolveScreenShareFrameRate(120)).toBe(120);
expect(resolveScreenShareFrameRate(90)).toBe(90);
expect(resolveScreenShareFrameRate(60)).toBe(60);
});
it('moves the preset to 1080p30 and lands the faster rungs on 60 FPS on the experiment', () => {
rollout.enabled = true;
describe('screen share quality', () => {
it('uses the 1080p30 preset and lands the faster rungs on 60 FPS', () => {
expect(targetOf()).toMatchObject({resolution: 'high', frameRate: 30});
expect(resolveScreenShareFrameRate(120)).toBe(60);
expect(resolveScreenShareFrameRate(90)).toBe(60);
expect(resolveScreenShareFrameRate(60)).toBe(60);
});
it('reads the rung table off the experiment and the pixel budget on it', () => {
expect(getScreenShareBitrateBps('source', 60)).toBe(6_000_000);
rollout.enabled = true;
it('reads the bitrate off the pixel budget', () => {
expect(getScreenShareBitrateBps('source', 60)).toBe(9_000_000);
});
it('publishes the stored frame rate and the rung bitrate off the experiment', () => {
it('publishes the resolved frame rate and the pixel budget bitrate', () => {
const {publishOptions} = buildScreenShareOptions({
resolution: 'source',
frameRate: 90,
@@ -118,26 +97,18 @@ describe('the screen share delivery experiment', () => {
sourceDimensions: {width: 3840, height: 2160},
});
expect(publishOptions.screenShareEncoding).toEqual({
maxBitrate: 6_000_000,
maxFramerate: 90,
maxBitrate: 9_000_000,
maxFramerate: 60,
priority: 'high',
});
expect(publishOptions.degradationPreference).toBe('maintain-resolution');
});
it('holds the motion hint for every surface off the experiment and only for a camera on it', () => {
expect(targetOf({mode: 'gaming'}).contentHint).toBe('motion');
rollout.enabled = true;
it('holds the motion hint only for a camera', () => {
expect(targetOf({mode: 'gaming'}).contentHint).toBeUndefined();
});
it('ignores the software H.264 clamp off the experiment', () => {
expect(targetOf({mode: 'gaming', softwareEncoderClamp: true})).toMatchObject({
resolution: 'ultra',
frameRate: 60,
softwareEncoderClamped: false,
});
rollout.enabled = true;
it('applies the software H.264 clamp', () => {
expect(targetOf({mode: 'gaming', softwareEncoderClamp: true})).toMatchObject({
resolution: 'medium',
frameRate: 30,
@@ -147,15 +118,7 @@ describe('the screen share delivery experiment', () => {
});
describe('screen share degradation preference', () => {
it('holds the resolution for every share off the experiment', () => {
expect(
resolveScreenShareDegradationPreference({
context: 'display',
rung: 'medium',
contentHint: undefined,
maxBitrate: 3_000_000,
}),
).toBe('maintain-resolution');
it('keeps device shares balanced', () => {
expect(
resolveScreenShareDegradationPreference({
context: 'device',
@@ -167,7 +130,6 @@ describe('screen share degradation preference', () => {
});
it('refuses maintain-framerate below the initial frame dropper cliff', () => {
rollout.enabled = true;
expect(
resolveScreenShareDegradationPreference({
context: 'display',
@@ -1,6 +1,5 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import type {ScreenshareResolution, StreamingMode} from '@app/features/voice/state/VoiceSettings';
import type {
ScreenShareContentHint,
@@ -23,8 +22,7 @@ const DIMENSIONS: Record<
ultra: {width: 2560, height: 1440},
source: {width: 3840, height: 2160},
};
export const SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS = 6_000_000;
export const SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS = 9_000_000;
export const SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS = 9_000_000;
export const SUPPORTED_SCREEN_SHARE_FRAME_RATES = [15, 30, 60, 90, 120] as const;
export type SupportedScreenShareFrameRate = (typeof SUPPORTED_SCREEN_SHARE_FRAME_RATES)[number];
@@ -52,15 +50,7 @@ const BITRATE_RUNGS = [
'source',
] as const satisfies ReadonlyArray<ScreenshareResolution>;
function isScreenShareDeliveryEnabled(delivery: boolean | undefined): boolean {
return delivery ?? ScreenShareDeliveryRollout.enabled;
}
export function resolveScreenShareFrameRate(frameRate: number, delivery?: boolean): SupportedScreenShareFrameRate {
if (!isScreenShareDeliveryEnabled(delivery)) {
if (frameRate >= 120) return 120;
if (frameRate >= 90) return 90;
}
export function resolveScreenShareFrameRate(frameRate: number): SupportedScreenShareFrameRate {
if (frameRate >= 60) return 60;
if (frameRate >= 30) return 30;
return 15;
@@ -83,12 +73,10 @@ function computeScreenShareBitrateBps(
pixels: number,
rung: ScreenshareResolution,
frameRate: SupportedScreenShareFrameRate,
delivery: boolean,
): number {
const rungFloor = BITRATE_KBPS[rung][frameRate] * 1000;
if (!delivery) return rungFloor;
const computed = Math.round(SCREEN_SHARE_BITS_PER_PIXEL_PER_FRAME * pixels * frameRate);
return Math.min(SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS, Math.max(rungFloor, computed));
return Math.min(SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS, Math.max(rungFloor, computed));
}
export function getScreenShareBitrateBps(
@@ -98,15 +86,9 @@ export function getScreenShareBitrateBps(
width: number;
height: number;
} | null,
delivery?: boolean,
): number {
const fit = resolveEffectiveScreenShareDimensions(resolution, sourceDimensions);
return computeScreenShareBitrateBps(
fit.width * fit.height,
fit.rung,
frameRate,
isScreenShareDeliveryEnabled(delivery),
);
return computeScreenShareBitrateBps(fit.width * fit.height, fit.rung, frameRate);
}
export const STREAMING_MODE_PRESETS: Record<
@@ -115,16 +97,6 @@ export const STREAMING_MODE_PRESETS: Record<
resolution: ScreenshareResolution;
frameRate: SupportedScreenShareFrameRate;
}
> = {
gaming: {resolution: 'ultra', frameRate: 60},
screenshare: {resolution: 'source', frameRate: 15},
};
const SCREEN_SHARE_DELIVERY_STREAMING_MODE_PRESETS: Record<
Exclude<StreamingMode, 'custom'>,
{
resolution: ScreenshareResolution;
frameRate: SupportedScreenShareFrameRate;
}
> = {
gaming: {resolution: 'ultra', frameRate: 60},
screenshare: {resolution: 'high', frameRate: 30},
@@ -152,7 +124,6 @@ export interface ScreenShareBuildConfig {
includeAudio: boolean;
contentHint?: ScreenShareCaptureOptions['contentHint'];
degradationPreference?: NonNullable<TrackPublishOptions['degradationPreference']>;
delivery?: boolean;
sourceDimensions?: {
width: number;
height: number;
@@ -208,10 +179,9 @@ export function resolveEffectiveScreenShareDimensions(
}
export function buildScreenShareOptions(config: ScreenShareBuildConfig): BuiltScreenShareOptions {
const delivery = isScreenShareDeliveryEnabled(config.delivery);
const {width, height, rung} = resolveEffectiveScreenShareDimensions(config.resolution, config.sourceDimensions);
const resolvedFrameRate = resolveScreenShareFrameRate(config.frameRate, delivery);
const maxBitrate = computeScreenShareBitrateBps(width * height, rung, resolvedFrameRate, delivery);
const resolvedFrameRate = resolveScreenShareFrameRate(config.frameRate);
const maxBitrate = computeScreenShareBitrateBps(width * height, rung, resolvedFrameRate);
const video: ScreenShareVideoOptions = {
cursor: resolveScreenShareCursorCapture(config.preferredDisplaySurface),
...(config.preferredDisplaySurface ? {displaySurface: config.preferredDisplaySurface} : {}),
@@ -235,7 +205,6 @@ export function buildScreenShareOptions(config: ScreenShareBuildConfig): BuiltSc
contentHint: config.contentHint,
maxBitrate,
degradationPreference: config.degradationPreference,
delivery,
}),
screenShareEncoding: {maxBitrate, maxFramerate: resolvedFrameRate, priority: 'high'},
},
@@ -270,17 +239,14 @@ export function resolveStreamingModeSettings(
customResolution: ScreenshareResolution,
customFrameRate: number,
hasHigherQuality: boolean,
delivery?: boolean,
): {
resolution: ScreenshareResolution;
frameRate: SupportedScreenShareFrameRate;
} {
const enabled = isScreenShareDeliveryEnabled(delivery);
const presets = enabled ? SCREEN_SHARE_DELIVERY_STREAMING_MODE_PRESETS : STREAMING_MODE_PRESETS;
const resolved =
mode === 'custom'
? {resolution: customResolution, frameRate: resolveScreenShareFrameRate(customFrameRate, enabled)}
: (hasHigherQuality ? presets : FREE_STREAMING_MODE_PRESETS)[mode];
? {resolution: customResolution, frameRate: resolveScreenShareFrameRate(customFrameRate)}
: (hasHigherQuality ? STREAMING_MODE_PRESETS : FREE_STREAMING_MODE_PRESETS)[mode];
if (hasHigherQuality) {
return resolved;
}
@@ -319,7 +285,6 @@ export interface ScreenShareQualityInput {
storedFrameRate: number;
entitled: boolean;
context: ScreenShareContext;
delivery?: boolean;
}
export interface ScreenShareTargetInput extends ScreenShareQualityInput {
@@ -341,7 +306,6 @@ export interface ScreenShareTarget {
contentHint: ScreenShareCaptureOptions['contentHint'];
degradationPreference: NonNullable<TrackPublishOptions['degradationPreference']>;
softwareEncoderClamped: boolean;
delivery?: boolean;
presetOwned: boolean;
tierLimited: boolean;
deviceMapped: boolean;
@@ -353,15 +317,11 @@ export interface ScreenShareDegradationInput {
contentHint: ScreenShareCaptureOptions['contentHint'];
maxBitrate: number;
degradationPreference?: NonNullable<TrackPublishOptions['degradationPreference']>;
delivery?: boolean;
}
export function resolveScreenShareDegradationPreference(
target: ScreenShareDegradationInput,
): NonNullable<TrackPublishOptions['degradationPreference']> {
if (!isScreenShareDeliveryEnabled(target.delivery)) {
return target.context === 'device' ? 'balanced' : 'maintain-resolution';
}
if (target.degradationPreference) return target.degradationPreference;
if (target.context === 'device') return 'balanced';
if (target.rung === 'source') return 'maintain-resolution';
@@ -393,17 +353,14 @@ function clampToSoftwareH264Budget(quality: {
return {mode: quality.mode, resolution: cappedResolution, frameRate: cappedFrameRate};
}
function resolveEffectiveScreenShareQuality(
input: ScreenShareQualityInput,
delivery: boolean,
): {
function resolveEffectiveScreenShareQuality(input: ScreenShareQualityInput): {
mode: StreamingMode;
resolution: ScreenshareResolution;
frameRate: SupportedScreenShareFrameRate;
} {
const mode = normaliseStreamingModeForContext(input.mode, input.context);
const resolution = normaliseResolutionForContext(input.storedResolution, input.context, input.entitled);
const settings = resolveStreamingModeSettings(mode, resolution, input.storedFrameRate, input.entitled, delivery);
const settings = resolveStreamingModeSettings(mode, resolution, input.storedFrameRate, input.entitled);
return {mode, resolution: settings.resolution, frameRate: settings.frameRate};
}
@@ -420,23 +377,20 @@ function resolveScreenShareContentHint(
mode: StreamingMode,
hintSetting: ScreenShareContentHint,
context: ScreenShareContext,
delivery: boolean,
): ScreenShareCaptureOptions['contentHint'] {
const hint = resolveScreenShareContentHintForMode(mode, hintSetting);
if (!delivery) return hint;
return hint === 'motion' && context !== 'device' ? undefined : hint;
}
export function resolveScreenShareTarget(input: ScreenShareTargetInput): ScreenShareTarget {
const delivery = isScreenShareDeliveryEnabled(input.delivery);
const clamped = delivery && (input.softwareEncoderClamp === true || shouldClampToSoftwareH264(input.codec));
const configured = resolveEffectiveScreenShareQuality(input, delivery);
const clamped = input.softwareEncoderClamp === true || shouldClampToSoftwareH264(input.codec);
const configured = resolveEffectiveScreenShareQuality(input);
const effective = clamped ? clampToSoftwareH264Budget(configured) : configured;
const configuredOnDisplay = resolveEffectiveScreenShareQuality({...input, context: 'display'}, delivery);
const configuredOnDisplay = resolveEffectiveScreenShareQuality({...input, context: 'display'});
const onDisplay = clamped ? clampToSoftwareH264Budget(configuredOnDisplay) : configuredOnDisplay;
const fit = resolveEffectiveScreenShareDimensions(effective.resolution, input.sourceDimensions);
const maxBitrate = computeScreenShareBitrateBps(fit.width * fit.height, fit.rung, effective.frameRate, delivery);
const contentHint = resolveScreenShareContentHint(effective.mode, input.hintSetting, input.context, delivery);
const maxBitrate = computeScreenShareBitrateBps(fit.width * fit.height, fit.rung, effective.frameRate);
const contentHint = resolveScreenShareContentHint(effective.mode, input.hintSetting, input.context);
return {
mode: effective.mode,
resolution: effective.resolution,
@@ -452,16 +406,14 @@ export function resolveScreenShareTarget(input: ScreenShareTargetInput): ScreenS
rung: fit.rung,
contentHint,
maxBitrate,
delivery,
}),
softwareEncoderClamped: clamped,
delivery,
presetOwned: effective.mode !== 'custom',
tierLimited:
!input.entitled &&
input.mode === 'custom' &&
(PREMIUM_SCREEN_SHARE_RESOLUTIONS.includes(input.storedResolution) ||
resolveScreenShareFrameRate(input.storedFrameRate, delivery) > FREE_TIER_MAX_FRAME_RATE),
resolveScreenShareFrameRate(input.storedFrameRate) > FREE_TIER_MAX_FRAME_RATE),
deviceMapped:
effective.mode !== onDisplay.mode ||
effective.resolution !== onDisplay.resolution ||
@@ -570,13 +522,12 @@ export function resolveScreenShareSenderCodec(
}
export function buildScreenShareSenderParameters(input: ScreenShareSenderParametersInput): ScreenShareSenderParameters {
const delivery = input.target.delivery === true;
const degradationPreference = resolveScreenShareDegradationPreference(input.target);
const targetPixels = input.target.width * input.target.height;
const capturePixels = input.capture ? input.capture.width * input.capture.height : null;
const baseScale = Math.min(...input.encodings.map((encoding) => encoding.scaleResolutionDownBy ?? 1));
const targetScale =
capturePixels === null || (delivery && degradationPreference === 'maintain-framerate')
capturePixels === null || degradationPreference === 'maintain-framerate'
? 1
: Math.max(1, Math.sqrt(capturePixels / targetPixels));
const sentPixels = capturePixels === null ? null : capturePixels / (targetScale * targetScale);
@@ -584,12 +535,7 @@ export function buildScreenShareSenderParameters(input: ScreenShareSenderParamet
sentPixels !== null && sentPixels < targetPixels * SENT_PIXEL_FILL_RATIO
? Math.min(
input.target.maxBitrate,
computeScreenShareBitrateBps(
sentPixels,
resolveBitrateRungForPixels(sentPixels),
input.target.frameRate,
delivery,
),
computeScreenShareBitrateBps(sentPixels, resolveBitrateRungForPixels(sentPixels), input.target.frameRate),
)
: input.target.maxBitrate;
const bitrates = distributeScreenShareBitrate(input.encodings, maxBitrate);
@@ -628,9 +574,8 @@ export function resolveScreenShareQualityPick(
input: ScreenShareQualityInput,
pick: ScreenShareQualityPick,
): ScreenShareQualityPatch | null {
const delivery = isScreenShareDeliveryEnabled(input.delivery);
const effective = resolveEffectiveScreenShareQuality(input, delivery);
const stored = resolveEffectiveScreenShareQuality({...input, mode: 'custom'}, delivery);
const effective = resolveEffectiveScreenShareQuality(input);
const stored = resolveEffectiveScreenShareQuality({...input, mode: 'custom'});
if (pick.axis === 'resolution') {
if (normaliseResolutionForContext(pick.resolution, input.context, input.entitled) !== pick.resolution) return null;
if (pick.resolution === effective.resolution) return null;
@@ -8,7 +8,6 @@ import {resolveConfiguredScreenShareTarget} from '@app/features/voice/engine/voi
import ActiveScreenShareSource from '@app/features/voice/state/ActiveScreenShareSource';
import {clearDesktopSourceIntent, setDesktopSourceIntent} from '@app/features/voice/state/DesktopSourceIntent';
import LocalVoiceState from '@app/features/voice/state/LocalVoiceState';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {
filterRoutableLinuxAudioSources,
@@ -806,7 +805,6 @@ export async function switchConfiguredDisplayScreenShare(
}
export async function restartActiveScreenShareCapture(): Promise<boolean> {
if (!ScreenShareDeliveryRollout.enabled) return false;
if (!didScreenShareStart()) return false;
const publishedSource = ActiveScreenShareSource.getPublishedSource();
const sourceId = ActiveScreenShareSource.getSourceId();
@@ -1,7 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Logger} from '@app/features/platform/utils/AppLogger';
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
import type {VideoCodec} from 'livekit-client';
const logger = new Logger('VideoDecoderCapabilities');
@@ -150,7 +149,7 @@ export function getVideoDecoderExclusionsSync(): Array<VideoCodec> | null {
}
export function markScreenShareDecodeFailure(codec: VideoCodec, reason: string): boolean {
if (ScreenShareDeliveryRollout.enabled && isBaselineWebRtcDecodeCodec(codec)) {
if (isBaselineWebRtcDecodeCodec(codec)) {
logger.warn('Keeping a baseline codec advertised despite a local screen share decode stall', {codec, reason});
return false;
}