mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-18 05:25:40 +09:00
feat(voice): ship the screen share delivery rework to everyone (#2931)
This commit is contained in:
1 parent
88038a1d5b
commit
a0825e77c4
53 files changed
+171
-1806
No files matched your search
@@ -6,7 +6,6 @@ import type {VoiceNoiseSuppressionAssignmentResponse} from '@fluxer/schema/src/d
|
||||
import {
|
||||
type ExperimentAssignmentsResponse,
|
||||
INERT_EXPERIMENT_ASSIGNMENTS_RESPONSE,
|
||||
readScreenShareDeliveryAssignment,
|
||||
readVoiceNoiseSuppressionAssignment,
|
||||
} from '@fluxer/schema/src/domains/experiment/ExperimentSchemas';
|
||||
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
|
||||
@@ -123,8 +122,6 @@ describe('ExperimentAssignments cold start', () => {
|
||||
await settle();
|
||||
expect(ExperimentAssignments.response).toBe(INERT_EXPERIMENT_ASSIGNMENTS_RESPONSE);
|
||||
expect(ExperimentAssignments.response.assignments.voice_noise_suppression).toBeUndefined();
|
||||
expect(ExperimentAssignments.response.assignments.screen_share_delivery).toBeUndefined();
|
||||
expect(readScreenShareDeliveryAssignment(ExperimentAssignments.response).enabled).toBe(false);
|
||||
});
|
||||
|
||||
it('keeps the inert envelope while unauthenticated and retries later', async () => {
|
||||
@@ -170,32 +167,6 @@ describe('ExperimentAssignments response handling', () => {
|
||||
expect(ExperimentAssignments.response).toEqual(CANARY_ENVELOPE);
|
||||
});
|
||||
|
||||
it('adopts a screen share delivery assignment beside the voice one', async () => {
|
||||
await adopt({
|
||||
...CANARY_ENVELOPE,
|
||||
assignments: {...CANARY_ENVELOPE.assignments, screen_share_delivery: {enabled: true}},
|
||||
});
|
||||
expect(readScreenShareDeliveryAssignment(ExperimentAssignments.response).enabled).toBe(true);
|
||||
expect(readVoiceNoiseSuppressionAssignment(ExperimentAssignments.response)).toEqual(CANARY_ASSIGNMENT);
|
||||
});
|
||||
|
||||
it('reads screen share delivery as disabled when the envelope omits it', async () => {
|
||||
await adopt(CANARY_ENVELOPE);
|
||||
expect(readScreenShareDeliveryAssignment(ExperimentAssignments.response).enabled).toBe(false);
|
||||
});
|
||||
|
||||
it('discards an envelope with a malformed screen share delivery assignment', async () => {
|
||||
await adopt(CANARY_ENVELOPE);
|
||||
vi.mocked(http.get).mockResolvedValue(
|
||||
reply(200, {
|
||||
...CANARY_ENVELOPE,
|
||||
assignments: {...CANARY_ENVELOPE.assignments, screen_share_delivery: {enabled: 'yes'}},
|
||||
}),
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(400_000);
|
||||
expect(ExperimentAssignments.response).toEqual(CANARY_ENVELOPE);
|
||||
});
|
||||
|
||||
it('accepts an envelope that carries no voice noise suppression assignment', async () => {
|
||||
await adopt({poll_interval_seconds: 600, poll_jitter_percent: 0, assignments: {}});
|
||||
expect(ExperimentAssignments.response.assignments.voice_noise_suppression).toBeUndefined();
|
||||
|
||||
+27
-49
@@ -2,25 +2,18 @@
|
||||
|
||||
import {videoElementHasRenderedFrame} from '@app/features/voice/components/VideoElementFrameState';
|
||||
import {Store} from '@app/features/voice/engine/Store';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
|
||||
export const LAST_FRAME_SNAPSHOTS_MAX = 8;
|
||||
export const LAST_FRAME_SNAPSHOT_WIDTH_MAX = 640;
|
||||
export const LAST_FRAME_SNAPSHOT_JPEG_QUALITY = 0.7;
|
||||
|
||||
const LAST_FRAME_SNAPSHOT_LEGACY_WIDTH_MAX = 1280;
|
||||
|
||||
function computeSnapshotDimensions(
|
||||
sourceWidth: number,
|
||||
sourceHeight: number,
|
||||
widthMax: number,
|
||||
): {width: number; height: number} {
|
||||
if (sourceWidth <= widthMax) {
|
||||
function computeSnapshotDimensions(sourceWidth: number, sourceHeight: number): {width: number; height: number} {
|
||||
if (sourceWidth <= LAST_FRAME_SNAPSHOT_WIDTH_MAX) {
|
||||
return {width: sourceWidth, height: sourceHeight};
|
||||
}
|
||||
const scale = widthMax / sourceWidth;
|
||||
const scale = LAST_FRAME_SNAPSHOT_WIDTH_MAX / sourceWidth;
|
||||
return {
|
||||
width: widthMax,
|
||||
width: LAST_FRAME_SNAPSHOT_WIDTH_MAX,
|
||||
height: Math.max(1, Math.round(sourceHeight * scale)),
|
||||
};
|
||||
}
|
||||
@@ -29,12 +22,11 @@ function drawSourceToOffscreenCanvas(
|
||||
source: CanvasImageSource,
|
||||
sourceWidth: number,
|
||||
sourceHeight: number,
|
||||
widthMax: number,
|
||||
): OffscreenCanvas | null {
|
||||
if (typeof OffscreenCanvas === 'undefined') return null;
|
||||
if (sourceWidth <= 0 || sourceHeight <= 0) return null;
|
||||
try {
|
||||
const {width, height} = computeSnapshotDimensions(sourceWidth, sourceHeight, widthMax);
|
||||
const {width, height} = computeSnapshotDimensions(sourceWidth, sourceHeight);
|
||||
const canvas = new OffscreenCanvas(width, height);
|
||||
const context = canvas.getContext('2d');
|
||||
if (!context) return null;
|
||||
@@ -45,16 +37,11 @@ function drawSourceToOffscreenCanvas(
|
||||
}
|
||||
}
|
||||
|
||||
function drawSourceToDataUrl(
|
||||
source: CanvasImageSource,
|
||||
sourceWidth: number,
|
||||
sourceHeight: number,
|
||||
widthMax: number,
|
||||
): string | null {
|
||||
function drawSourceToDataUrl(source: CanvasImageSource, sourceWidth: number, sourceHeight: number): string | null {
|
||||
if (typeof document === 'undefined') return null;
|
||||
if (sourceWidth <= 0 || sourceHeight <= 0) return null;
|
||||
try {
|
||||
const {width, height} = computeSnapshotDimensions(sourceWidth, sourceHeight, widthMax);
|
||||
const {width, height} = computeSnapshotDimensions(sourceWidth, sourceHeight);
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
@@ -114,38 +101,29 @@ class LastFrameSnapshotCache extends Store {
|
||||
|
||||
captureFromVideoElement(key: string, video: HTMLVideoElement | null): void {
|
||||
if (!key) return;
|
||||
const deliveryEnabled = ScreenShareDeliveryRollout.enabled;
|
||||
if (deliveryEnabled && typeof document !== 'undefined' && document.visibilityState === 'hidden') return;
|
||||
if (typeof document !== 'undefined' && document.visibilityState === 'hidden') return;
|
||||
if (!videoElementHasRenderedFrame(video)) return;
|
||||
const renderedVideo = video as HTMLVideoElement;
|
||||
const widthMax = deliveryEnabled ? LAST_FRAME_SNAPSHOT_WIDTH_MAX : LAST_FRAME_SNAPSHOT_LEGACY_WIDTH_MAX;
|
||||
if (deliveryEnabled) {
|
||||
const captureId = this.nextCaptureId;
|
||||
this.nextCaptureId += 1;
|
||||
this.captureIds.set(key, captureId);
|
||||
const canvas = drawSourceToOffscreenCanvas(
|
||||
renderedVideo,
|
||||
renderedVideo.videoWidth,
|
||||
renderedVideo.videoHeight,
|
||||
widthMax,
|
||||
);
|
||||
if (canvas) {
|
||||
void canvas
|
||||
.convertToBlob({type: 'image/jpeg', quality: LAST_FRAME_SNAPSHOT_JPEG_QUALITY})
|
||||
.then((blob) => {
|
||||
if (this.captureIds.get(key) !== captureId) return;
|
||||
this.captureIds.delete(key);
|
||||
this.retainSnapshot(key, URL.createObjectURL(blob));
|
||||
})
|
||||
.catch(() => {
|
||||
if (this.captureIds.get(key) !== captureId) return;
|
||||
this.captureIds.delete(key);
|
||||
});
|
||||
return;
|
||||
}
|
||||
this.captureIds.delete(key);
|
||||
const captureId = this.nextCaptureId;
|
||||
this.nextCaptureId += 1;
|
||||
this.captureIds.set(key, captureId);
|
||||
const canvas = drawSourceToOffscreenCanvas(renderedVideo, renderedVideo.videoWidth, renderedVideo.videoHeight);
|
||||
if (canvas) {
|
||||
void canvas
|
||||
.convertToBlob({type: 'image/jpeg', quality: LAST_FRAME_SNAPSHOT_JPEG_QUALITY})
|
||||
.then((blob) => {
|
||||
if (this.captureIds.get(key) !== captureId) return;
|
||||
this.captureIds.delete(key);
|
||||
this.retainSnapshot(key, URL.createObjectURL(blob));
|
||||
})
|
||||
.catch(() => {
|
||||
if (this.captureIds.get(key) !== captureId) return;
|
||||
this.captureIds.delete(key);
|
||||
});
|
||||
return;
|
||||
}
|
||||
const dataUrl = drawSourceToDataUrl(renderedVideo, renderedVideo.videoWidth, renderedVideo.videoHeight, widthMax);
|
||||
this.captureIds.delete(key);
|
||||
const dataUrl = drawSourceToDataUrl(renderedVideo, renderedVideo.videoWidth, renderedVideo.videoHeight);
|
||||
if (!dataUrl) return;
|
||||
this.retainSnapshot(key, dataUrl);
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ import {
|
||||
getLocalScreenShareVideoPublications,
|
||||
isLiveLocalTrackPublication,
|
||||
} from '@app/features/voice/engine/VoiceTrackPublicationUtils';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
|
||||
import {
|
||||
buildScreenShareCodecProfile,
|
||||
@@ -357,7 +356,6 @@ class ScreenShareCodecNegotiation {
|
||||
encoderMode: ScreenShareEncoderMode;
|
||||
order: ReadonlyArray<VideoCodec>;
|
||||
} | null = null;
|
||||
private frozenDelivery: {trackSid: string; enabled: boolean} | null = null;
|
||||
private lastCodecChangeAt = 0;
|
||||
private suppressionTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private publishedTrackSid: string | null = null;
|
||||
@@ -452,22 +450,9 @@ class ScreenShareCodecNegotiation {
|
||||
this.localCodecs = buildLocalCodecAdvertisements(this.resolveCodecPreferenceOrder());
|
||||
}
|
||||
|
||||
private resolveScreenShareDelivery(room: Room | null = this.room): boolean {
|
||||
const trackSid = this.getLocalScreenSharePublication(room)?.trackSid ?? null;
|
||||
if (trackSid === null) {
|
||||
this.frozenDelivery = null;
|
||||
return ScreenShareDeliveryRollout.enabled;
|
||||
}
|
||||
if (this.frozenDelivery?.trackSid === trackSid) return this.frozenDelivery.enabled;
|
||||
const enabled = ScreenShareDeliveryRollout.enabled;
|
||||
this.frozenDelivery = {trackSid, enabled};
|
||||
return enabled;
|
||||
}
|
||||
|
||||
private resolveCodecPreferenceOrder(
|
||||
preference: CodecPreference = VoiceSettings.getPreferredScreenShareCodec(),
|
||||
): ReadonlyArray<VideoCodec> {
|
||||
if (!this.resolveScreenShareDelivery()) return getScreenShareCodecPreferenceOrder(preference);
|
||||
const encoderMode = VoiceSettings.getScreenShareEncoderMode();
|
||||
const frozen = this.frozenCodecPreference;
|
||||
if (
|
||||
@@ -523,7 +508,7 @@ class ScreenShareCodecNegotiation {
|
||||
unknownParticipants: number;
|
||||
} {
|
||||
const now = Date.now();
|
||||
const viewerIdentities = this.resolveScreenShareDelivery(room) ? this.getScreenShareViewerIdentities(room) : null;
|
||||
const viewerIdentities = this.getScreenShareViewerIdentities(room);
|
||||
const knownRemoteCodecs: Array<Array<FluxerCodecAdvertisement>> = [];
|
||||
const participants: Array<{identity: string; firstSeenAt: number}> = [];
|
||||
for (const participant of room?.remoteParticipants.values() ?? []) {
|
||||
@@ -640,7 +625,6 @@ class ScreenShareCodecNegotiation {
|
||||
this.selectedCodec = null;
|
||||
this.localCodecs = [];
|
||||
this.frozenCodecPreference = null;
|
||||
this.frozenDelivery = null;
|
||||
this.lastCodecChangeAt = 0;
|
||||
this.publishedTrackSid = null;
|
||||
this.clearSuppressionTimer();
|
||||
@@ -749,7 +733,6 @@ class ScreenShareCodecNegotiation {
|
||||
const {knownRemoteCodecs, unknownParticipants} = this.getRemoteCodecInputs(room);
|
||||
this.scheduleGraceReevaluations(room, bindingRevision);
|
||||
const previousCodec = this.selectedCodec;
|
||||
const delivery = this.resolveScreenShareDelivery(room);
|
||||
this.negotiationSnapshot = transitionScreenShareCodecNegotiationSnapshot(this.negotiationSnapshot, {
|
||||
type: 'negotiation.evaluate',
|
||||
localCodecs: this.localCodecs,
|
||||
@@ -757,7 +740,7 @@ class ScreenShareCodecNegotiation {
|
||||
unknownParticipants,
|
||||
reason,
|
||||
codecPreference: this.resolveCodecPreferenceOrder(),
|
||||
publishedCodec: delivery ? this.observePublishedScreenShareCodec(room) : null,
|
||||
publishedCodec: this.observePublishedScreenShareCodec(room),
|
||||
});
|
||||
const selection = this.negotiationSnapshot.context.selection;
|
||||
if (!selection) return null;
|
||||
@@ -765,11 +748,7 @@ class ScreenShareCodecNegotiation {
|
||||
this.selectedCodec = selection.codec;
|
||||
return selection;
|
||||
}
|
||||
if (
|
||||
delivery &&
|
||||
previousCodec !== null &&
|
||||
Date.now() - this.lastCodecChangeAt < SCREEN_SHARE_CODEC_CHANGE_SUPPRESSION_MS
|
||||
) {
|
||||
if (previousCodec !== null && Date.now() - this.lastCodecChangeAt < SCREEN_SHARE_CODEC_CHANGE_SUPPRESSION_MS) {
|
||||
logger.debug('Suppressed a screen share codec change inside the change window', {
|
||||
codec: selection.codec,
|
||||
previousCodec,
|
||||
@@ -779,7 +758,7 @@ class ScreenShareCodecNegotiation {
|
||||
return selection;
|
||||
}
|
||||
this.selectedCodec = selection.codec;
|
||||
if (delivery) this.armCodecChangeSuppression();
|
||||
this.armCodecChangeSuppression();
|
||||
this.mediaSessionId = createId('media');
|
||||
logger.info('Selected screen share codec from XState capability intersection', selection);
|
||||
await this.publishSessionUpdate(room, selection);
|
||||
|
||||
@@ -9,12 +9,7 @@ import {
|
||||
type VoiceGatewayStateEvent,
|
||||
type VoiceGatewayStateSnapshot,
|
||||
} from '@app/features/voice/engine/VoiceGatewayStateMachine';
|
||||
import {describe, expect, it, vi} from 'vitest';
|
||||
|
||||
vi.mock('@app/features/voice/state/ScreenShareDeliveryRollout', () => ({
|
||||
ScreenShareDeliveryRollout: {enabled: false},
|
||||
default: {enabled: false},
|
||||
}));
|
||||
import {describe, expect, it} from 'vitest';
|
||||
|
||||
function voiceState(overrides: Partial<VoiceState> = {}): VoiceState {
|
||||
return {
|
||||
|
||||
@@ -19,12 +19,7 @@ import {
|
||||
type VoiceMediaGraphStreamTileState,
|
||||
} from '@app/features/voice/engine/VoiceMediaGraphTileState';
|
||||
import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
|
||||
import {describe, expect, it, vi} from 'vitest';
|
||||
|
||||
vi.mock('@app/features/voice/state/ScreenShareDeliveryRollout', () => ({
|
||||
ScreenShareDeliveryRollout: {enabled: false},
|
||||
default: {enabled: false},
|
||||
}));
|
||||
import {describe, expect, it} from 'vitest';
|
||||
|
||||
const SIMULATION_SEEDS: ReadonlyArray<number> = Array.from({length: 20}, (_, index) => 1 + index * 7919);
|
||||
const SIMULATION_STEP_COUNT = 1000;
|
||||
|
||||
@@ -11,7 +11,6 @@ import {
|
||||
voiceMediaGraphStatsTrackKey,
|
||||
} from '@app/features/voice/engine/VoiceMediaGraphStatsObservations';
|
||||
import type {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import type {VoiceEngineV2PerTrackStats} from '@fluxer/voice_engine_v2';
|
||||
|
||||
const VOICE_MEDIA_GRAPH_STATS_TRACK_LIMIT = 1024;
|
||||
@@ -77,9 +76,7 @@ function perTrackStatToInfo(track: VoiceEngineV2PerTrackStats): VoiceMediaGraphP
|
||||
if (track.kind !== 'video') return null;
|
||||
const width = track.frameWidth ?? track.sourceFrameWidth;
|
||||
const height = track.frameHeight ?? track.sourceFrameHeight;
|
||||
const fps = ScreenShareDeliveryRollout.enabled
|
||||
? (track.effectiveFramesPerSecond ?? track.framesPerSecond)
|
||||
: (track.effectiveFramesPerSecond ?? track.framesPerSecond ?? track.sourceFramesPerSecond);
|
||||
const fps = track.effectiveFramesPerSecond ?? track.framesPerSecond;
|
||||
const info: VoiceMediaGraphPartialTrackInfo = {};
|
||||
if (isPositiveDimension(width) && isPositiveDimension(height)) {
|
||||
info.width = width;
|
||||
@@ -193,9 +190,7 @@ function observationToPartialTrackInfo(
|
||||
): VoiceMediaGraphPartialTrackInfo | null {
|
||||
const width = observation.width ?? observation.sourceWidth ?? undefined;
|
||||
const height = observation.height ?? observation.sourceHeight ?? undefined;
|
||||
const fps = ScreenShareDeliveryRollout.enabled
|
||||
? (observation.fps ?? undefined)
|
||||
: (observation.fps ?? observation.sourceFps ?? undefined);
|
||||
const fps = observation.fps ?? undefined;
|
||||
const info: VoiceMediaGraphPartialTrackInfo = {};
|
||||
if (isPositiveDimension(width) && isPositiveDimension(height)) {
|
||||
info.width = width;
|
||||
|
||||
@@ -16,7 +16,6 @@ import {
|
||||
} from '@app/features/voice/engine/VoiceStreamWatchState';
|
||||
import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
|
||||
import ParticipantVolume from '@app/features/voice/state/ParticipantVolume';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import {ScreenShareWatchErrorCode, ScreenShareWatchFailures} from '@app/features/voice/state/ScreenShareWatchFailures';
|
||||
import {monitorScreenShareDecodeHealth} from '@app/features/voice/utils/ScreenShareCodecDiagnostics';
|
||||
import {markScreenShareDecodeFailure} from '@app/features/voice/utils/VideoDecoderCapabilities';
|
||||
@@ -108,30 +107,6 @@ interface ParticipantSpeakingDisposer {
|
||||
dispose: () => void;
|
||||
}
|
||||
|
||||
type LatencyTunedReceiver = RTCRtpReceiver & {
|
||||
jitterBufferTarget?: number;
|
||||
playoutDelayHint?: number;
|
||||
};
|
||||
|
||||
function getRemoteTrackReceiver(track: RemoteTrack): LatencyTunedReceiver | undefined {
|
||||
return (track as RemoteTrack & {receiver?: LatencyTunedReceiver}).receiver;
|
||||
}
|
||||
|
||||
function applyInteractiveReceiverBuffer(track: RemoteTrack, pub: RemoteTrackPublication): void {
|
||||
const receiver = getRemoteTrackReceiver(track);
|
||||
if (!receiver) return;
|
||||
try {
|
||||
if (pub.kind === Track.Kind.Video && pub.source === Track.Source.ScreenShare) {
|
||||
receiver.jitterBufferTarget = 80;
|
||||
receiver.playoutDelayHint = 0.04;
|
||||
} else if (pub.kind === Track.Kind.Audio) {
|
||||
receiver.jitterBufferTarget = 60;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.debug('Failed to apply interactive receiver buffer target', {error, source: pub.source, kind: pub.kind});
|
||||
}
|
||||
}
|
||||
|
||||
export function bindRoomEvents(
|
||||
room: Room,
|
||||
attemptId: number,
|
||||
@@ -141,7 +116,6 @@ export function bindRoomEvents(
|
||||
dependencies: RoomEventDependencies,
|
||||
): void {
|
||||
const guard = dependencies.connection.createGuardedHandler;
|
||||
const screenShareDeliveryEnabled = ScreenShareDeliveryRollout.enabled;
|
||||
const participantSpeakingDisposers = new Map<string, ParticipantSpeakingDisposer>();
|
||||
const screenShareDecodeMonitorCancels = new Map<string, () => void>();
|
||||
const remoteTrackLifecycleDisposers = new Map<string, () => void>();
|
||||
@@ -443,9 +417,6 @@ export function bindRoomEvents(
|
||||
trackSid: pub.trackSid,
|
||||
});
|
||||
}
|
||||
if (!screenShareDeliveryEnabled) {
|
||||
applyInteractiveReceiverBuffer(track, pub);
|
||||
}
|
||||
monitorDecodeHealth(track, pub);
|
||||
dependencies.remoteSpeaking.attachIfApplicable(participant, pub, track);
|
||||
bindRemoteTrackLifecycleIfApplicable(participant, pub, track);
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import assert from 'node:assert/strict';
|
||||
import {isElectronPlatform} from '@app/features/platform/types/Platform';
|
||||
import {Logger} from '@app/features/platform/utils/AppLogger';
|
||||
import {Store} from '@app/features/voice/engine/Store';
|
||||
import {sendVoiceStateDisconnect} from '@app/features/voice/engine/VoiceChannelConnector';
|
||||
@@ -37,7 +36,6 @@ import {
|
||||
isReadyToRepublishTrack,
|
||||
} from '@app/features/voice/engine/v2/VoiceEngineV2AppAdapterAssertions';
|
||||
import {VoiceEngineV2AppReconnectPolicy} from '@app/features/voice/engine/v2/VoiceEngineV2AppReconnectPolicy';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import VoiceRegionTeleport from '@app/features/voice/state/VoiceRegionTeleport';
|
||||
import {
|
||||
findVideoPublishCodecPolicyViolation,
|
||||
@@ -45,10 +43,7 @@ import {
|
||||
} from '@app/features/voice/utils/CodecCapabilityDetector';
|
||||
import {getH264HardwareProfilesSync} from '@app/features/voice/utils/GpuEncoderCapabilities';
|
||||
import {setNoiseSuppressionScopeGuildId} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionSelection';
|
||||
import {
|
||||
SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS,
|
||||
SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
|
||||
} from '@app/features/voice/utils/ScreenShareOptions';
|
||||
import {SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS} from '@app/features/voice/utils/ScreenShareOptions';
|
||||
import {
|
||||
clearScreenShareDecodeFailures,
|
||||
getVideoDecoderExclusionsSync,
|
||||
@@ -151,9 +146,7 @@ function createWebAudioMixOption(): RoomOptions['webAudioMix'] {
|
||||
function createRoomPublishDefaults(): RoomOptions['publishDefaults'] {
|
||||
return {
|
||||
screenShareEncoding: {
|
||||
maxBitrate: ScreenShareDeliveryRollout.enabled
|
||||
? SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS
|
||||
: SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
|
||||
maxBitrate: SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
|
||||
maxFramerate: 30,
|
||||
priority: 'high',
|
||||
},
|
||||
@@ -164,7 +157,6 @@ function createRoomPublishDefaults(): RoomOptions['publishDefaults'] {
|
||||
function createRoomOptions(
|
||||
e2eeKey: string | null,
|
||||
subscriberVideoCodecExclusions: RoomOptions['subscriberVideoCodecExclusions'],
|
||||
screenShareDelivery: boolean,
|
||||
): {
|
||||
roomOptions: RoomOptions;
|
||||
e2eeKeyProvider: ExternalE2EEKeyProvider | null;
|
||||
@@ -176,8 +168,7 @@ function createRoomOptions(
|
||||
webAudioMix: createWebAudioMixOption(),
|
||||
publishDefaults: createRoomPublishDefaults(),
|
||||
subscriberVideoCodecExclusions,
|
||||
screenShareDelivery,
|
||||
h264HardwareProfiles: screenShareDelivery ? getH264HardwareProfilesSync()?.profiles : undefined,
|
||||
h264HardwareProfiles: getH264HardwareProfilesSync()?.profiles,
|
||||
};
|
||||
let e2eeKeyProvider: ExternalE2EEKeyProvider | null = null;
|
||||
let e2eeWorker: Worker | null = null;
|
||||
@@ -196,19 +187,11 @@ function createRoomOptions(
|
||||
return {roomOptions, e2eeKeyProvider, e2eeWorker};
|
||||
}
|
||||
|
||||
function createRoomConnectOptions(screenShareDelivery: boolean): RoomConnectOptions {
|
||||
function createRoomConnectOptions(): RoomConnectOptions {
|
||||
const connectOptions: RoomConnectOptions = {
|
||||
autoSubscribe: false,
|
||||
};
|
||||
assert.equal(connectOptions.autoSubscribe, false, 'LiveKit connect options must not auto-subscribe');
|
||||
if (!screenShareDelivery && isElectronPlatform()) {
|
||||
connectOptions.rtcConfig = {iceTransportPolicy: 'relay'};
|
||||
assert.equal(
|
||||
connectOptions.rtcConfig.iceTransportPolicy,
|
||||
'relay',
|
||||
'Electron LiveKit connects must force relay ICE',
|
||||
);
|
||||
}
|
||||
return connectOptions;
|
||||
}
|
||||
|
||||
@@ -564,12 +547,7 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
|
||||
logger.warn('Aborting LiveKit room creation after codec probing because attempt is stale', {attemptId});
|
||||
return;
|
||||
}
|
||||
const screenShareDelivery = ScreenShareDeliveryRollout.enabled;
|
||||
const {roomOptions, e2eeKeyProvider, e2eeWorker} = createRoomOptions(
|
||||
e2eeKey,
|
||||
subscriberVideoCodecExclusions,
|
||||
screenShareDelivery,
|
||||
);
|
||||
const {roomOptions, e2eeKeyProvider, e2eeWorker} = createRoomOptions(e2eeKey, subscriberVideoCodecExclusions);
|
||||
const room = new LiveKitRoom(roomOptions);
|
||||
ownE2EEWorker(room, e2eeWorker);
|
||||
let roomClosed = false;
|
||||
@@ -602,7 +580,7 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
|
||||
return;
|
||||
}
|
||||
logger.info('Attempting to connect to LiveKit', {endpoint, guildId, channelId: resolvedChannelId});
|
||||
const connectOptions = createRoomConnectOptions(screenShareDelivery);
|
||||
const connectOptions = createRoomConnectOptions();
|
||||
room
|
||||
.connect(endpoint, token, connectOptions)
|
||||
.then(() => {
|
||||
@@ -704,14 +682,12 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
|
||||
const connectionId = raw.connection_id ?? null;
|
||||
this.abortHotSwap();
|
||||
const cachedExclusions = getVideoDecoderExclusionsSync();
|
||||
const screenShareDelivery = ScreenShareDeliveryRollout.enabled;
|
||||
const roomOptions: RoomOptions = {
|
||||
adaptiveStream: false,
|
||||
dynacast: true,
|
||||
webAudioMix: createWebAudioMixOption(),
|
||||
publishDefaults: createRoomPublishDefaults(),
|
||||
subscriberVideoCodecExclusions: cachedExclusions && cachedExclusions.length > 0 ? cachedExclusions : undefined,
|
||||
screenShareDelivery,
|
||||
};
|
||||
if (!this.isLatestConnectionAttempt(attemptId) || this.connectionState.room !== existingRoom) {
|
||||
logger.warn('Region hot-swap: aborted before room creation because attempt is stale', {attemptId});
|
||||
@@ -728,7 +704,7 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
|
||||
this.abortHotSwap();
|
||||
}
|
||||
});
|
||||
const connectOptions = createRoomConnectOptions(screenShareDelivery);
|
||||
const connectOptions = createRoomConnectOptions();
|
||||
logger.info('Region hot-swap: connecting to new endpoint', {endpoint, guildId, channelId});
|
||||
newRoom
|
||||
.connect(endpoint, token, connectOptions)
|
||||
|
||||
+3
-12
@@ -63,7 +63,6 @@ import ActiveScreenShareSource, {
|
||||
type PublishedScreenShareSource,
|
||||
} from '@app/features/voice/state/ActiveScreenShareSource';
|
||||
import LocalVoiceState from '@app/features/voice/state/LocalVoiceState';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
|
||||
import {
|
||||
prepareHighFidelityScreenShareAudioTrack,
|
||||
@@ -183,7 +182,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
encoderVerificationTimer: (() => void) | null = null;
|
||||
private screenShareTrackingHolds = 0;
|
||||
private screenShareCapturePaused = false;
|
||||
private screenShareDeliveryArmed = false;
|
||||
private screenShareTrackingGeneration = 0;
|
||||
private readonly reconciledCodecPairs = new Set<string>();
|
||||
private readonly verifiedCodecCorrectionsByTrack = new WeakMap<MediaStreamTrack, number>();
|
||||
@@ -679,7 +677,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
preferredTrack?: LocalVideoTrack,
|
||||
): void {
|
||||
this.cleanupActiveScreenShareEndListenerInternal();
|
||||
this.screenShareDeliveryArmed = ScreenShareDeliveryRollout.enabled;
|
||||
const publication = preferredTrack ? undefined : participant.getTrackPublication(Track.Source.ScreenShare);
|
||||
const videoTrack = preferredTrack ?? publication?.videoTrack;
|
||||
const mediaStreamTrack = videoTrack?.mediaStreamTrack;
|
||||
@@ -730,15 +727,11 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
};
|
||||
const onMuted = (): void => reportCapturePaused(true);
|
||||
const onUnmuted = (): void => reportCapturePaused(false);
|
||||
const tracksCapturePauses = this.screenShareDeliveryArmed;
|
||||
mediaStreamTrack.addEventListener('ended', onEnded);
|
||||
if (tracksCapturePauses) {
|
||||
mediaStreamTrack.addEventListener('mute', onMuted);
|
||||
mediaStreamTrack.addEventListener('unmute', onUnmuted);
|
||||
}
|
||||
mediaStreamTrack.addEventListener('mute', onMuted);
|
||||
mediaStreamTrack.addEventListener('unmute', onUnmuted);
|
||||
this.activeScreenShareEndListener = () => {
|
||||
mediaStreamTrack.removeEventListener('ended', onEnded);
|
||||
if (!tracksCapturePauses) return;
|
||||
mediaStreamTrack.removeEventListener('mute', onMuted);
|
||||
mediaStreamTrack.removeEventListener('unmute', onUnmuted);
|
||||
};
|
||||
@@ -1152,7 +1145,7 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
nextConstraints.frameRate = {ideal: resolution.frameRate, max: resolution.frameRate};
|
||||
}
|
||||
if (JSON.stringify(currentConstraints) === JSON.stringify(nextConstraints)) return 'unchanged';
|
||||
if (this.screenShareDeliveryArmed && screenShareSourceHasClonedTracks(screenShareTrack)) {
|
||||
if (screenShareSourceHasClonedTracks(screenShareTrack)) {
|
||||
logger.warn('Screen share capture geometry needs a restart while a backup codec holds a clone of the source', {
|
||||
resolution,
|
||||
});
|
||||
@@ -1202,7 +1195,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
this.ensureScreenShareKeepAliveSinkInternal(participant);
|
||||
updateLocalParticipantFromRoom(room);
|
||||
this.syncLocalScreenShareAudioStateInternal(participant, participant.isScreenShareEnabled);
|
||||
if (!this.screenShareDeliveryArmed) return true;
|
||||
return geometryUpdate !== 'needs-capture-restart' && geometryUpdate !== 'failed';
|
||||
}
|
||||
|
||||
@@ -1225,7 +1217,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
}
|
||||
|
||||
resetStreamTracking(): void {
|
||||
this.screenShareDeliveryArmed = false;
|
||||
this.setScreenShareCapturePausedInternal(false);
|
||||
this.clearScreenShareKeepAliveSinkInternal();
|
||||
this.setStreamingPriorityInternal(false);
|
||||
|
||||
@@ -32,11 +32,6 @@ const voiceStates: Record<string, Record<string, Record<string, unknown>>> = {};
|
||||
const settingsUpdate = vi.fn();
|
||||
const openPremiumModal = vi.fn();
|
||||
|
||||
vi.mock('@app/features/voice/state/ScreenShareDeliveryRollout', () => ({
|
||||
ScreenShareDeliveryRollout: {enabled: true},
|
||||
default: {enabled: true},
|
||||
}));
|
||||
|
||||
vi.mock('@app/features/voice/utils/GpuEncoderCapabilities', () => ({
|
||||
getGpuEncoderReportSync: () => gpuReport,
|
||||
getH264HardwareProfilesSync: () => h264HardwareProfiles,
|
||||
|
||||
-48
@@ -6,13 +6,11 @@ import {
|
||||
stopMediaTrack,
|
||||
stopUnselectedStreamTracks,
|
||||
} from '@app/features/voice/engine/voice_screen_share_manager/shared';
|
||||
import ActiveScreenShareSource from '@app/features/voice/state/ActiveScreenShareSource';
|
||||
import {rememberCapturedDisplayAudioTrack} from '@app/features/voice/utils/NativeAudioCaptureBridge';
|
||||
import {ScreenShareAudioCaptureError} from '@app/features/voice/utils/ScreenShareAudioCaptureError';
|
||||
import type {ScreenShareCaptureOptions} from 'livekit-client';
|
||||
|
||||
type DisplayMediaVideoConstraints = MediaTrackConstraints & {
|
||||
colorSpace?: string;
|
||||
cursor?: 'always' | 'motion' | 'never';
|
||||
displaySurface?: 'browser' | 'monitor' | 'window';
|
||||
};
|
||||
@@ -20,37 +18,12 @@ type DisplayMediaAudioConstraints = MediaTrackConstraints & {
|
||||
restrictOwnAudio?: boolean;
|
||||
suppressLocalAudioPlayback?: boolean;
|
||||
};
|
||||
type DisplayMediaTrackSettings = MediaTrackSettings & {
|
||||
cursor?: 'always' | 'motion' | 'never';
|
||||
displaySurface?: 'browser' | 'monitor' | 'window';
|
||||
};
|
||||
function resolveDisplayMediaCursorCapture(
|
||||
displaySurface: DisplayMediaVideoConstraints['displaySurface'],
|
||||
): 'always' | 'motion' | 'never' {
|
||||
return displaySurface === 'window' ? 'never' : 'always';
|
||||
}
|
||||
|
||||
function getRequestedDisplayMediaVideoConstraints(
|
||||
options: ScreenShareCaptureOptions | undefined,
|
||||
): DisplayMediaVideoConstraints | null {
|
||||
if (typeof options?.video !== 'object' || !options.video) return null;
|
||||
return options.video as DisplayMediaVideoConstraints;
|
||||
}
|
||||
|
||||
function resolveCapturedDisplayMediaCursorCapture(
|
||||
track: Pick<MediaStreamTrack, 'getSettings'>,
|
||||
options?: ScreenShareCaptureOptions,
|
||||
): 'always' | 'motion' | 'never' {
|
||||
const requestedVideo = getRequestedDisplayMediaVideoConstraints(options);
|
||||
const requestedCursor = requestedVideo?.cursor;
|
||||
const requestedDisplaySurface = requestedVideo?.displaySurface;
|
||||
if (requestedCursor && requestedCursor !== resolveDisplayMediaCursorCapture(requestedDisplaySurface)) {
|
||||
return requestedCursor;
|
||||
}
|
||||
const settings = track.getSettings() as DisplayMediaTrackSettings;
|
||||
return resolveDisplayMediaCursorCapture(settings.displaySurface ?? requestedDisplaySurface);
|
||||
}
|
||||
|
||||
export function getDisplayMediaOptions(options?: ScreenShareCaptureOptions): DisplayMediaStreamOptions {
|
||||
let videoConstraints: MediaTrackConstraints | boolean = options?.video ?? true;
|
||||
const resolution = options?.resolution;
|
||||
@@ -95,21 +68,6 @@ export function getDisplayMediaOptions(options?: ScreenShareCaptureOptions): Dis
|
||||
} as DisplayMediaStreamOptions;
|
||||
}
|
||||
|
||||
function buildCapturedDisplayMediaConstraints(
|
||||
displayMediaOptions: DisplayMediaStreamOptions,
|
||||
cursor: 'always' | 'motion' | 'never',
|
||||
): MediaTrackConstraints {
|
||||
const requestedVideo =
|
||||
typeof displayMediaOptions.video === 'object' && displayMediaOptions.video
|
||||
? (displayMediaOptions.video as DisplayMediaVideoConstraints)
|
||||
: undefined;
|
||||
const constraints: DisplayMediaVideoConstraints = {colorSpace: 'rec709', cursor};
|
||||
if (requestedVideo?.width !== undefined) constraints.width = requestedVideo.width;
|
||||
if (requestedVideo?.height !== undefined) constraints.height = requestedVideo.height;
|
||||
if (requestedVideo?.frameRate !== undefined) constraints.frameRate = requestedVideo.frameRate;
|
||||
return constraints;
|
||||
}
|
||||
|
||||
export async function createDisplayScreenShareTracks(
|
||||
options?: ScreenShareCaptureOptions,
|
||||
captureContext?: DisplayScreenShareCaptureContext,
|
||||
@@ -127,12 +85,6 @@ export async function createDisplayScreenShareTracks(
|
||||
if (options?.contentHint) {
|
||||
videoTrack.contentHint = options.contentHint;
|
||||
}
|
||||
if (ActiveScreenShareSource.getTarget()?.delivery !== true) {
|
||||
const cursor = resolveCapturedDisplayMediaCursorCapture(videoTrack, options);
|
||||
await videoTrack
|
||||
.applyConstraints(buildCapturedDisplayMediaConstraints(displayMediaOptions, cursor))
|
||||
.catch(() => undefined);
|
||||
}
|
||||
const capturedAudioTrack = stream.getAudioTracks()[0];
|
||||
const audioTrack = capturedAudioTrack?.readyState === 'live' ? capturedAudioTrack : undefined;
|
||||
rememberCapturedDisplayAudioTrack(capturedAudioTrack, Boolean(capturedAudioTrack) && !audioTrack);
|
||||
|
||||
@@ -4,7 +4,6 @@ import {getDesktopTroubleshootingSettings} from '@app/features/devtools/utils/De
|
||||
import {Logger} from '@app/features/platform/utils/AppLogger';
|
||||
import ScreenShareCodecNegotiation from '@app/features/voice/engine/ScreenShareCodecNegotiation';
|
||||
import ActiveScreenShareSource from '@app/features/voice/state/ActiveScreenShareSource';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import SoftwareEncoderWarning from '@app/features/voice/state/SoftwareEncoderWarning';
|
||||
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
|
||||
import {
|
||||
@@ -23,7 +22,6 @@ import {
|
||||
resolveScreenShareLayering,
|
||||
resolveScreenShareSenderCodec,
|
||||
resolveScreenShareTarget,
|
||||
SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS,
|
||||
SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
|
||||
type ScreenShareContext,
|
||||
type ScreenShareLayering,
|
||||
@@ -195,11 +193,13 @@ export async function releaseScreenShareCaptureCleanup(snapshot: ScreenShareCapt
|
||||
}
|
||||
}
|
||||
|
||||
function clampScreenShareEncoding(encoding: VideoEncoding, delivery: boolean): VideoEncoding {
|
||||
const ceiling = delivery ? SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS : SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS;
|
||||
function clampScreenShareEncoding(encoding: VideoEncoding): VideoEncoding {
|
||||
return {
|
||||
...encoding,
|
||||
maxBitrate: typeof encoding.maxBitrate === 'number' ? Math.min(encoding.maxBitrate, ceiling) : encoding.maxBitrate,
|
||||
maxBitrate:
|
||||
typeof encoding.maxBitrate === 'number'
|
||||
? Math.min(encoding.maxBitrate, SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS)
|
||||
: encoding.maxBitrate,
|
||||
priority: encoding.priority ?? 'high',
|
||||
};
|
||||
}
|
||||
@@ -208,15 +208,10 @@ export function resolveActiveScreenShareContext(): ScreenShareContext {
|
||||
return ActiveScreenShareSource.getShareContext() ?? 'display';
|
||||
}
|
||||
|
||||
function resolveScreenShareDeliveryArm(): boolean {
|
||||
return ActiveScreenShareSource.getTarget()?.delivery ?? ScreenShareDeliveryRollout.enabled;
|
||||
}
|
||||
|
||||
export function resolveConfiguredScreenShareTarget(
|
||||
context: ScreenShareContext,
|
||||
sourceDimensions: {width: number; height: number} | null,
|
||||
): ScreenShareTarget {
|
||||
const delivery = resolveScreenShareDeliveryArm();
|
||||
return resolveScreenShareTarget({
|
||||
mode: VoiceSettings.getStreamingMode(),
|
||||
storedResolution: VoiceSettings.getScreenshareResolution(),
|
||||
@@ -225,13 +220,8 @@ export function resolveConfiguredScreenShareTarget(
|
||||
context,
|
||||
sourceDimensions,
|
||||
hintSetting: VoiceSettings.getScreenShareContentHint(),
|
||||
delivery,
|
||||
...(delivery
|
||||
? {
|
||||
codec: getPreferredScreenShareCodec(),
|
||||
softwareEncoderClamp: ActiveScreenShareSource.isSoftwareEncoderClamped(),
|
||||
}
|
||||
: {}),
|
||||
codec: getPreferredScreenShareCodec(),
|
||||
softwareEncoderClamp: ActiveScreenShareSource.isSoftwareEncoderClamped(),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -242,7 +232,6 @@ export function resolveActiveScreenShareTarget(
|
||||
}
|
||||
|
||||
function recommitClampedScreenShareTarget(committed: ScreenShareTarget): ScreenShareTarget {
|
||||
if (committed.delivery !== true) return committed;
|
||||
if (committed.softwareEncoderClamped) return committed;
|
||||
const resolved = resolveActiveScreenShareTarget(committed.context);
|
||||
if (!resolved.softwareEncoderClamped) return committed;
|
||||
@@ -270,7 +259,7 @@ export function getStatsKind(
|
||||
|
||||
function resolveScreenShareEncoding(target: ScreenShareTarget, publishOptions?: TrackPublishOptions): VideoEncoding {
|
||||
if (publishOptions?.screenShareEncoding) {
|
||||
return clampScreenShareEncoding(publishOptions.screenShareEncoding, target.delivery === true);
|
||||
return clampScreenShareEncoding(publishOptions.screenShareEncoding);
|
||||
}
|
||||
return {maxBitrate: target.maxBitrate, maxFramerate: target.frameRate, priority: 'high'};
|
||||
}
|
||||
@@ -863,27 +852,22 @@ export function startScreenShareEncoderMonitor(options: ScreenShareEncoderMonito
|
||||
let framesEncoded: number | null = null;
|
||||
let sendSnapshot: ScreenShareSendSnapshot | null = null;
|
||||
let cpuLimitedTicks = 0;
|
||||
const delivery = ActiveScreenShareSource.getTarget()?.delivery === true;
|
||||
const tick = async (): Promise<void> => {
|
||||
const sender = options.track.sender;
|
||||
if (!sender) return;
|
||||
const stats = await sender.getStats();
|
||||
if (delivery) {
|
||||
syncScreenShareCaptureSize(sender);
|
||||
}
|
||||
syncScreenShareCaptureSize(sender);
|
||||
const encoded = countEncodedVideoFrames(stats);
|
||||
ActiveScreenShareSource.setEncoding(encoded !== null && framesEncoded !== null && encoded > framesEncoded);
|
||||
framesEncoded = encoded;
|
||||
if (delivery) {
|
||||
const snapshot = collectScreenShareSendSnapshot(stats);
|
||||
cpuLimitedTicks = snapshot.cpuLimited ? cpuLimitedTicks + 1 : 0;
|
||||
const limit = classifyScreenShareSendLimit(sendSnapshot, snapshot, cpuLimitedTicks);
|
||||
sendSnapshot = snapshot;
|
||||
if (limit !== ActiveScreenShareSource.getLimit()) {
|
||||
logger.info('Screen share send limit changed', {limit, codec: options.codec});
|
||||
}
|
||||
ActiveScreenShareSource.setLimit(limit);
|
||||
const snapshot = collectScreenShareSendSnapshot(stats);
|
||||
cpuLimitedTicks = snapshot.cpuLimited ? cpuLimitedTicks + 1 : 0;
|
||||
const limit = classifyScreenShareSendLimit(sendSnapshot, snapshot, cpuLimitedTicks);
|
||||
sendSnapshot = snapshot;
|
||||
if (limit !== ActiveScreenShareSource.getLimit()) {
|
||||
logger.info('Screen share send limit changed', {limit, codec: options.codec});
|
||||
}
|
||||
ActiveScreenShareSource.setLimit(limit);
|
||||
if (!verified) {
|
||||
const verification = verifyScreenShareEncoderStart(stats, options.codec, options.onEncodeFailure);
|
||||
if (verification === 'failed') return;
|
||||
@@ -907,8 +891,6 @@ export function startScreenShareEncoderMonitor(options: ScreenShareEncoderMonito
|
||||
cancelled = true;
|
||||
clearTimeout(timer);
|
||||
ActiveScreenShareSource.setEncoding(false);
|
||||
if (delivery) {
|
||||
ActiveScreenShareSource.setLimit(null);
|
||||
}
|
||||
ActiveScreenShareSource.setLimit(null);
|
||||
};
|
||||
}
|
||||
@@ -70,11 +70,6 @@ class ActiveScreenShareSource {
|
||||
this.encoding = false;
|
||||
return;
|
||||
}
|
||||
if (target.delivery !== true) {
|
||||
this.target = target;
|
||||
this.encoding = false;
|
||||
return;
|
||||
}
|
||||
const degradationPreference = this.frozenDegradationPreference ?? target.degradationPreference;
|
||||
this.frozenDegradationPreference = degradationPreference;
|
||||
this.softwareEncoderClamped = this.softwareEncoderClamped || target.softwareEncoderClamped;
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import ExperimentAssignments from '@app/features/experiment/state/ExperimentAssignments';
|
||||
import {Logger} from '@app/features/platform/utils/AppLogger';
|
||||
import {readScreenShareDeliveryAssignment} from '@fluxer/schema/src/domains/experiment/ExperimentSchemas';
|
||||
|
||||
const logger = new Logger('ScreenShareDeliveryRollout');
|
||||
|
||||
class ScreenShareDeliveryRolloutSelector {
|
||||
get enabled(): boolean {
|
||||
try {
|
||||
return readScreenShareDeliveryAssignment(ExperimentAssignments.response).enabled;
|
||||
} catch (err) {
|
||||
logger.warn('Failed to resolve screen share delivery assignment:', err);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const ScreenShareDeliveryRollout = new ScreenShareDeliveryRolloutSelector();
|
||||
|
||||
export default ScreenShareDeliveryRollout;
|
||||
@@ -3,7 +3,6 @@
|
||||
import AppStorage from '@app/features/platform/state/PersistentStorage';
|
||||
import {Logger} from '@app/features/platform/utils/AppLogger';
|
||||
import {makePersistent} from '@app/features/platform/utils/MobXPersistence';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import type {
|
||||
CodecPreference,
|
||||
ScreenShareContentHint,
|
||||
@@ -263,9 +262,6 @@ function applyStreamingModeDefaultMigrationV1(parsed: Record<string, unknown>):
|
||||
}
|
||||
|
||||
function applyScreenShareFrameRateFloorMigrationV1(parsed: Record<string, unknown>): boolean {
|
||||
if (!ScreenShareDeliveryRollout.enabled) {
|
||||
return false;
|
||||
}
|
||||
if (parsed.screenShareFrameRateFloorMigratedV1 === true) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import type {TrackPublishOptions} from 'livekit-client';
|
||||
|
||||
export const OPUS_MAX_AUDIO_BITRATE_BPS = 510000;
|
||||
@@ -40,9 +39,7 @@ export function buildMicrophonePublishOptions(
|
||||
|
||||
export const SCREEN_SHARE_AUDIO_PUBLISH_OPTIONS: TrackPublishOptions = {
|
||||
audioPreset: {
|
||||
get maxBitrate(): number {
|
||||
return ScreenShareDeliveryRollout.enabled ? SCREEN_SHARE_AUDIO_BITRATE_BPS : OPUS_MAX_AUDIO_BITRATE_BPS;
|
||||
},
|
||||
maxBitrate: SCREEN_SHARE_AUDIO_BITRATE_BPS,
|
||||
priority: 'high',
|
||||
},
|
||||
dtx: false,
|
||||
|
||||
@@ -9,7 +9,6 @@ import {
|
||||
isFirefoxBrowser,
|
||||
type NativePlatform,
|
||||
} from '@app/features/ui/utils/NativeUtils';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
|
||||
import {getGpuEncoderReportSync, type HardwareEncodeAnswer} from '@app/features/voice/utils/GpuEncoderCapabilities';
|
||||
import {
|
||||
@@ -26,10 +25,7 @@ import {
|
||||
type ScreenShareCodecRanking,
|
||||
} from '@app/features/voice/utils/ScreenShareCodecSelection';
|
||||
import {normaliseStreamingModeForContext} from '@app/features/voice/utils/ScreenShareOptions';
|
||||
import {
|
||||
getProbedVideoDecoderExclusionsSync,
|
||||
getVideoDecoderExclusionsSync,
|
||||
} from '@app/features/voice/utils/VideoDecoderCapabilities';
|
||||
import {getProbedVideoDecoderExclusionsSync} from '@app/features/voice/utils/VideoDecoderCapabilities';
|
||||
import type {TrackPublishDefaults, TrackPublishOptions} from 'livekit-client';
|
||||
import {BackupCodecPolicy, supportsVideoCodec, type VideoCodec, type VideoEncoding} from 'livekit-client';
|
||||
|
||||
@@ -98,7 +94,6 @@ let cachedReportNativeHardwareEncoderKey: object | null | undefined;
|
||||
let cachedReportHardwareAccelerationDisabled: boolean | undefined;
|
||||
let cachedReportAv1OptIn: boolean | undefined;
|
||||
let cachedReportHevcOptIn: boolean | undefined;
|
||||
let cachedReportDelivery: boolean | undefined;
|
||||
const runtimeEncodeFailureCodecs = new Set<VideoCodec>();
|
||||
const observedSoftwareEncodeCodecs = new Set<VideoCodec>();
|
||||
|
||||
@@ -214,21 +209,15 @@ function getScreenShareCodecPolicyUnsupported(
|
||||
return null;
|
||||
}
|
||||
|
||||
function hasPublishPathNativeHardwareEncoder(
|
||||
codec: VideoCodec,
|
||||
context: CodecPolicyContext,
|
||||
delivery: boolean,
|
||||
): boolean {
|
||||
function hasPublishPathNativeHardwareEncoder(codec: VideoCodec, context: CodecPolicyContext): boolean {
|
||||
if (!hasNativeHardwareEncoder(codec)) return false;
|
||||
const backend = getNativeHardwareEncoderCapabilitiesSync()?.backend;
|
||||
if (!delivery) return backend !== 'videotoolbox' && backend !== 'nvenc';
|
||||
if (backend === 'videotoolbox') return true;
|
||||
if (backend === 'nvenc') return context.platform !== 'linux';
|
||||
return false;
|
||||
}
|
||||
|
||||
function buildReport(): CodecCapabilityReport {
|
||||
const delivery = ScreenShareDeliveryRollout.enabled;
|
||||
const {caps, probedSuccessfully} = probeRawCapabilities();
|
||||
const context = buildScreenShareCodecPolicyContext();
|
||||
const gpuReport = getGpuEncoderReportSync();
|
||||
@@ -243,17 +232,11 @@ function buildReport(): CodecCapabilityReport {
|
||||
if (observedSoftwareEncodeCodecs.has(codec)) {
|
||||
return 'software';
|
||||
}
|
||||
if (!delivery) {
|
||||
if (hasPublishPathNativeHardwareEncoder(codec, context, delivery)) {
|
||||
return 'hardware';
|
||||
}
|
||||
return gpuReport ? gpuReport[codec] : 'unknown';
|
||||
}
|
||||
const measured = gpuReport ? gpuReport[codec] : 'unknown';
|
||||
if (measured !== 'unknown') {
|
||||
return measured;
|
||||
}
|
||||
return hasPublishPathNativeHardwareEncoder(codec, context, delivery) ? 'hardware' : 'unknown';
|
||||
return hasPublishPathNativeHardwareEncoder(codec, context) ? 'hardware' : 'unknown';
|
||||
}
|
||||
type DescribedUnsupported = Omit<CodecSupportInfo, 'hardwareAccelerated'>;
|
||||
function unsupported(codec: keyof CodecCapabilities, info: DescribedUnsupported): CodecSupportInfo {
|
||||
@@ -271,8 +254,8 @@ function buildReport(): CodecCapabilityReport {
|
||||
detail: 'This codec failed while publishing during the current session.',
|
||||
});
|
||||
}
|
||||
const supportedByNativeHardware = hasPublishPathNativeHardwareEncoder(codec, context, delivery);
|
||||
if (caps[codec] || (!delivery && supportedByNativeHardware)) {
|
||||
const supportedByNativeHardware = hasPublishPathNativeHardwareEncoder(codec, context);
|
||||
if (caps[codec]) {
|
||||
return {
|
||||
supported: true,
|
||||
reason: 'supported',
|
||||
@@ -348,10 +331,8 @@ export function getCodecCapabilityReport(): CodecCapabilityReport {
|
||||
const hardwareAccelerationDisabled = isDesktopHardwareAccelerationDisabled();
|
||||
const av1OptIn = VoiceSettings.getScreenShareAv1OptIn();
|
||||
const hevcOptIn = VoiceSettings.getScreenShareHevcOptIn();
|
||||
const delivery = ScreenShareDeliveryRollout.enabled;
|
||||
if (
|
||||
cachedReport &&
|
||||
cachedReportDelivery === delivery &&
|
||||
cachedReportGpuKey === currentGpu &&
|
||||
cachedReportNativeHardwareEncoderKey === currentNativeHardwareEncoder &&
|
||||
cachedReportHardwareAccelerationDisabled === hardwareAccelerationDisabled &&
|
||||
@@ -365,7 +346,6 @@ export function getCodecCapabilityReport(): CodecCapabilityReport {
|
||||
cachedReportHardwareAccelerationDisabled = hardwareAccelerationDisabled;
|
||||
cachedReportAv1OptIn = av1OptIn;
|
||||
cachedReportHevcOptIn = hevcOptIn;
|
||||
cachedReportDelivery = delivery;
|
||||
return cachedReport;
|
||||
}
|
||||
|
||||
@@ -538,9 +518,7 @@ export function getVideoPublishCodecDenial(codec: VideoCodec): VideoPublishCodec
|
||||
if (!supportsVideoCodec(codec)) return 'sender-cannot-encode';
|
||||
if (getScreenShareCodecPolicyUnsupported(codec, buildScreenShareCodecPolicyContext())) return 'policy';
|
||||
if (runtimeEncodeFailureCodecs.has(codec)) return 'runtime-failed';
|
||||
const decoderExclusions = ScreenShareDeliveryRollout.enabled
|
||||
? getProbedVideoDecoderExclusionsSync()
|
||||
: getVideoDecoderExclusionsSync();
|
||||
const decoderExclusions = getProbedVideoDecoderExclusionsSync();
|
||||
if (decoderExclusions?.includes(codec) === true) return 'decoder-excluded';
|
||||
return null;
|
||||
}
|
||||
@@ -701,7 +679,6 @@ export function resetCachedCodecCapabilities(): void {
|
||||
cachedReportHardwareAccelerationDisabled = undefined;
|
||||
cachedReportAv1OptIn = undefined;
|
||||
cachedReportHevcOptIn = undefined;
|
||||
cachedReportDelivery = undefined;
|
||||
runtimeEncodeFailureCodecs.clear();
|
||||
observedSoftwareEncodeCodecs.clear();
|
||||
resetNativeHardwareEncoderCapabilities();
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
import {Logger} from '@app/features/platform/utils/AppLogger';
|
||||
import {getElectronAPI, isDesktop} from '@app/features/ui/utils/NativeUtils';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import type {GpuDeviceInfo, GpuInfo} from '@app/types/electron.d';
|
||||
import type {VideoCodec} from 'livekit-client';
|
||||
|
||||
@@ -238,12 +237,6 @@ export const H264_ENCODE_PROBE_CONTENT_TYPES: ReadonlyArray<string> = H264_PROBE
|
||||
(profileLevelId) => `video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=${profileLevelId}`,
|
||||
);
|
||||
|
||||
const CONTROL_H264_PROBE_PROFILE_LEVEL_IDS: ReadonlyArray<string> = ['640028', '4d0028', '420028', '42e028'];
|
||||
|
||||
const CONTROL_H264_ENCODE_PROBE_CONTENT_TYPES: ReadonlyArray<string> = CONTROL_H264_PROBE_PROFILE_LEVEL_IDS.map(
|
||||
(profileLevelId) => `video/H264;level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=${profileLevelId}`,
|
||||
);
|
||||
|
||||
export const WEBRTC_ENCODE_PROBE_CONTENT_TYPES: Record<VideoCodec, ReadonlyArray<string>> = {
|
||||
av1: ['video/AV1'],
|
||||
h265: ['video/H265'],
|
||||
@@ -415,24 +408,17 @@ export async function probeWebRtcEncodeEfficiency(
|
||||
): Promise<Record<VideoCodec, HardwareEncodeAnswer> | null> {
|
||||
const mediaCapabilities = getMediaCapabilities();
|
||||
if (!mediaCapabilities) return null;
|
||||
const delivery = ScreenShareDeliveryRollout.enabled;
|
||||
const video = resolveEncodeProbeVideoConfig(config);
|
||||
const codecs: ReadonlyArray<VideoCodec> = ['av1', 'h265', 'h264', 'vp9', 'vp8'];
|
||||
const answers = await Promise.all(
|
||||
codecs.map((codec) =>
|
||||
probeContentTypesEncodeEfficiency(
|
||||
mediaCapabilities,
|
||||
codec === 'h264' && !delivery
|
||||
? CONTROL_H264_ENCODE_PROBE_CONTENT_TYPES
|
||||
: WEBRTC_ENCODE_PROBE_CONTENT_TYPES[codec],
|
||||
video,
|
||||
),
|
||||
probeContentTypesEncodeEfficiency(mediaCapabilities, WEBRTC_ENCODE_PROBE_CONTENT_TYPES[codec], video),
|
||||
),
|
||||
);
|
||||
const result = {} as Record<VideoCodec, HardwareEncodeAnswer>;
|
||||
codecs.forEach((codec, index) => {
|
||||
const codecAnswers = answers[index] ?? [];
|
||||
if (delivery && codec === 'h264') recordH264HardwareProfileProbe(codecAnswers, video);
|
||||
if (codec === 'h264') recordH264HardwareProfileProbe(codecAnswers, video);
|
||||
result[codec] = collapseEncodeProbeAnswers(codecAnswers);
|
||||
});
|
||||
return result;
|
||||
@@ -460,7 +446,6 @@ let pendingPromise: Promise<HardwareEncodeReport | null> | null = null;
|
||||
|
||||
function fetchReport(): Promise<HardwareEncodeReport | null> {
|
||||
if (!isDesktop()) {
|
||||
if (!ScreenShareDeliveryRollout.enabled) return Promise.resolve(null);
|
||||
return probeH264HardwareProfiles().then(() => null);
|
||||
}
|
||||
const electron = getElectronAPI();
|
||||
|
||||
@@ -9,12 +9,7 @@ import {
|
||||
markScreenShareDecodeFailure,
|
||||
resetVideoDecoderExclusions,
|
||||
} from '@app/features/voice/utils/VideoDecoderCapabilities';
|
||||
import {afterEach, describe, expect, it, vi} from 'vitest';
|
||||
|
||||
vi.mock('@app/features/voice/state/ScreenShareDeliveryRollout', () => ({
|
||||
ScreenShareDeliveryRollout: {enabled: true},
|
||||
default: {enabled: true},
|
||||
}));
|
||||
import {afterEach, describe, expect, it} from 'vitest';
|
||||
|
||||
function buildStats(inbound: Record<string, unknown>): RTCStatsReport {
|
||||
return new Map<string, unknown>([
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {Logger} from '@app/features/platform/utils/AppLogger';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import SoftwareEncoderWarning from '@app/features/voice/state/SoftwareEncoderWarning';
|
||||
import {classifyVideoDecoderAcceleration} from '@app/features/voice/utils/VideoAccelerationClassification';
|
||||
import type {VideoCodec} from 'livekit-client';
|
||||
|
||||
const logger = new Logger('ScreenShareCodecDiagnostics');
|
||||
const DECODE_SAMPLE_INTERVAL_MS = 5000;
|
||||
const DECODER_VERIFICATION_DELAY_MS = 5000;
|
||||
const UNKNOWN_DECODER_IMPLEMENTATION = 'software decoder';
|
||||
|
||||
interface CodecStatsEntry {
|
||||
@@ -144,97 +142,6 @@ function collectInboundVideoStats(stats: RTCStatsReport): {
|
||||
return {codecs, reports};
|
||||
}
|
||||
|
||||
function findSoftwareVideoDecoderInStats(stats: RTCStatsReport): SoftwareVideoDecoderInfo | null {
|
||||
const {codecs, reports} = collectInboundVideoStats(stats);
|
||||
for (const report of reports) {
|
||||
if (getStatsKind(report, codecs) !== 'video') continue;
|
||||
const implementation =
|
||||
typeof report.decoderImplementation === 'string' && report.decoderImplementation.length > 0
|
||||
? report.decoderImplementation
|
||||
: null;
|
||||
const powerEfficientDecoder =
|
||||
typeof report.powerEfficientDecoder === 'boolean' ? report.powerEfficientDecoder : null;
|
||||
if (!isSoftwareVideoStats(implementation, powerEfficientDecoder)) continue;
|
||||
return {
|
||||
codec: getCodecLabel(report.codecId ? codecs.get(report.codecId)?.mimeType : undefined),
|
||||
implementation: implementation ?? UNKNOWN_DECODER_IMPLEMENTATION,
|
||||
powerEfficientDecoder,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function findStalledVideoDecoderInStats(stats: RTCStatsReport): StalledVideoDecoderInfo | null {
|
||||
const {codecs, reports} = collectInboundVideoStats(stats);
|
||||
for (const report of reports) {
|
||||
if (getStatsKind(report, codecs) !== 'video') continue;
|
||||
const framesDecoded = finiteNumber(report.framesDecoded);
|
||||
if (framesDecoded === null || framesDecoded > 0) continue;
|
||||
const framesReceived = finiteNumber(report.framesReceived);
|
||||
if (framesReceived === null || framesReceived < 1) continue;
|
||||
const mimeType = report.codecId ? codecs.get(report.codecId)?.mimeType : undefined;
|
||||
const codec = getVideoCodecFromMimeType(mimeType);
|
||||
if (!codec) continue;
|
||||
return {
|
||||
codec,
|
||||
mimeType,
|
||||
packetsReceived: finiteNumber(report.packetsReceived) ?? 0,
|
||||
bytesReceived: finiteNumber(report.bytesReceived) ?? 0,
|
||||
framesDecoded,
|
||||
framesReceived,
|
||||
framesDropped: finiteNumber(report.framesDropped),
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function scheduleScreenShareDecoderVerification(
|
||||
getStats: () => Promise<RTCStatsReport | undefined>,
|
||||
onDecodeFailure?: (failure: StalledVideoDecoderInfo) => void,
|
||||
): () => void {
|
||||
let cancelled = false;
|
||||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||
const scheduleConfirmation = (first: StalledVideoDecoderInfo): void => {
|
||||
timer = setTimeout(async () => {
|
||||
timer = null;
|
||||
try {
|
||||
const stats = await getStats();
|
||||
const confirmed = confirmDecodeStall(first, stats ? findStalledVideoDecoderInStats(stats) : null);
|
||||
if (cancelled || !confirmed) return;
|
||||
logger.warn('Screen share video decode is stalled', confirmed);
|
||||
onDecodeFailure?.(confirmed);
|
||||
} catch (error) {
|
||||
logger.debug('Failed to confirm the screen share decode stall', {error});
|
||||
}
|
||||
}, DECODER_VERIFICATION_DELAY_MS);
|
||||
};
|
||||
timer = setTimeout(async () => {
|
||||
timer = null;
|
||||
let firstStall: StalledVideoDecoderInfo | null = null;
|
||||
try {
|
||||
const stats = await getStats();
|
||||
if (!stats) return;
|
||||
firstStall = findStalledVideoDecoderInStats(stats);
|
||||
const decoder = findSoftwareVideoDecoderInStats(stats);
|
||||
if (!decoder) return;
|
||||
logger.warn('Screen share is using a software decoder', decoder);
|
||||
SoftwareEncoderWarning.triggerDecoderWarning(decoder.codec, decoder.implementation);
|
||||
} catch (error) {
|
||||
logger.debug('Failed to verify screen share decoder', {error});
|
||||
} finally {
|
||||
if (!cancelled && firstStall) {
|
||||
scheduleConfirmation(firstStall);
|
||||
}
|
||||
}
|
||||
}, DECODER_VERIFICATION_DELAY_MS);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
if (!timer) return;
|
||||
clearTimeout(timer);
|
||||
timer = null;
|
||||
};
|
||||
}
|
||||
|
||||
export function findInboundVideoDecodeSample(stats: RTCStatsReport): InboundVideoDecodeSample | null {
|
||||
const {codecs, reports} = collectInboundVideoStats(stats);
|
||||
for (const report of reports) {
|
||||
@@ -338,7 +245,6 @@ export function monitorScreenShareDecodeHealth(
|
||||
getStats: () => Promise<RTCStatsReport | undefined>,
|
||||
onDecodeStall?: (failure: StalledVideoDecoderInfo) => void,
|
||||
): () => void {
|
||||
if (!ScreenShareDeliveryRollout.enabled) return scheduleScreenShareDecoderVerification(getStats, onDecodeStall);
|
||||
let cancelled = false;
|
||||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||
let previousSample: InboundVideoDecodeSample | null = null;
|
||||
|
||||
@@ -8,14 +8,7 @@ import {
|
||||
resolveScreenShareLayering,
|
||||
resolveScreenShareTarget,
|
||||
} from '@app/features/voice/utils/ScreenShareOptions';
|
||||
import {afterEach, describe, expect, it, vi} from 'vitest';
|
||||
|
||||
const rollout = vi.hoisted(() => ({enabled: false}));
|
||||
|
||||
vi.mock('@app/features/voice/state/ScreenShareDeliveryRollout', () => ({
|
||||
ScreenShareDeliveryRollout: rollout,
|
||||
default: rollout,
|
||||
}));
|
||||
import {describe, expect, it, vi} from 'vitest';
|
||||
|
||||
vi.mock('@app/features/voice/utils/NativeAudioCaptureBridge', () => ({
|
||||
rememberCapturedDisplayAudioTrack: () => undefined,
|
||||
@@ -29,10 +22,6 @@ const {getDisplayMediaOptions} = await import(
|
||||
'@app/features/voice/engine/voice_screen_share_manager/DisplayMediaCapture'
|
||||
);
|
||||
|
||||
afterEach(() => {
|
||||
rollout.enabled = false;
|
||||
});
|
||||
|
||||
function collectKeys(value: unknown, keys: Set<string>): Set<string> {
|
||||
if (typeof value !== 'object' || value === null) return keys;
|
||||
for (const [key, entry] of Object.entries(value)) {
|
||||
@@ -87,29 +76,19 @@ describe('display capture constraints', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('the screen share delivery experiment', () => {
|
||||
it('keeps the source preset and the 90 and 120 FPS rungs off the experiment', () => {
|
||||
expect(targetOf()).toMatchObject({resolution: 'source', frameRate: 15});
|
||||
expect(resolveScreenShareFrameRate(120)).toBe(120);
|
||||
expect(resolveScreenShareFrameRate(90)).toBe(90);
|
||||
expect(resolveScreenShareFrameRate(60)).toBe(60);
|
||||
});
|
||||
|
||||
it('moves the preset to 1080p30 and lands the faster rungs on 60 FPS on the experiment', () => {
|
||||
rollout.enabled = true;
|
||||
describe('screen share quality', () => {
|
||||
it('uses the 1080p30 preset and lands the faster rungs on 60 FPS', () => {
|
||||
expect(targetOf()).toMatchObject({resolution: 'high', frameRate: 30});
|
||||
expect(resolveScreenShareFrameRate(120)).toBe(60);
|
||||
expect(resolveScreenShareFrameRate(90)).toBe(60);
|
||||
expect(resolveScreenShareFrameRate(60)).toBe(60);
|
||||
});
|
||||
|
||||
it('reads the rung table off the experiment and the pixel budget on it', () => {
|
||||
expect(getScreenShareBitrateBps('source', 60)).toBe(6_000_000);
|
||||
rollout.enabled = true;
|
||||
it('reads the bitrate off the pixel budget', () => {
|
||||
expect(getScreenShareBitrateBps('source', 60)).toBe(9_000_000);
|
||||
});
|
||||
|
||||
it('publishes the stored frame rate and the rung bitrate off the experiment', () => {
|
||||
it('publishes the resolved frame rate and the pixel budget bitrate', () => {
|
||||
const {publishOptions} = buildScreenShareOptions({
|
||||
resolution: 'source',
|
||||
frameRate: 90,
|
||||
@@ -118,26 +97,18 @@ describe('the screen share delivery experiment', () => {
|
||||
sourceDimensions: {width: 3840, height: 2160},
|
||||
});
|
||||
expect(publishOptions.screenShareEncoding).toEqual({
|
||||
maxBitrate: 6_000_000,
|
||||
maxFramerate: 90,
|
||||
maxBitrate: 9_000_000,
|
||||
maxFramerate: 60,
|
||||
priority: 'high',
|
||||
});
|
||||
expect(publishOptions.degradationPreference).toBe('maintain-resolution');
|
||||
});
|
||||
|
||||
it('holds the motion hint for every surface off the experiment and only for a camera on it', () => {
|
||||
expect(targetOf({mode: 'gaming'}).contentHint).toBe('motion');
|
||||
rollout.enabled = true;
|
||||
it('holds the motion hint only for a camera', () => {
|
||||
expect(targetOf({mode: 'gaming'}).contentHint).toBeUndefined();
|
||||
});
|
||||
|
||||
it('ignores the software H.264 clamp off the experiment', () => {
|
||||
expect(targetOf({mode: 'gaming', softwareEncoderClamp: true})).toMatchObject({
|
||||
resolution: 'ultra',
|
||||
frameRate: 60,
|
||||
softwareEncoderClamped: false,
|
||||
});
|
||||
rollout.enabled = true;
|
||||
it('applies the software H.264 clamp', () => {
|
||||
expect(targetOf({mode: 'gaming', softwareEncoderClamp: true})).toMatchObject({
|
||||
resolution: 'medium',
|
||||
frameRate: 30,
|
||||
@@ -147,15 +118,7 @@ describe('the screen share delivery experiment', () => {
|
||||
});
|
||||
|
||||
describe('screen share degradation preference', () => {
|
||||
it('holds the resolution for every share off the experiment', () => {
|
||||
expect(
|
||||
resolveScreenShareDegradationPreference({
|
||||
context: 'display',
|
||||
rung: 'medium',
|
||||
contentHint: undefined,
|
||||
maxBitrate: 3_000_000,
|
||||
}),
|
||||
).toBe('maintain-resolution');
|
||||
it('keeps device shares balanced', () => {
|
||||
expect(
|
||||
resolveScreenShareDegradationPreference({
|
||||
context: 'device',
|
||||
@@ -167,7 +130,6 @@ describe('screen share degradation preference', () => {
|
||||
});
|
||||
|
||||
it('refuses maintain-framerate below the initial frame dropper cliff', () => {
|
||||
rollout.enabled = true;
|
||||
expect(
|
||||
resolveScreenShareDegradationPreference({
|
||||
context: 'display',
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import type {ScreenshareResolution, StreamingMode} from '@app/features/voice/state/VoiceSettings';
|
||||
import type {
|
||||
ScreenShareContentHint,
|
||||
@@ -23,8 +22,7 @@ const DIMENSIONS: Record<
|
||||
ultra: {width: 2560, height: 1440},
|
||||
source: {width: 3840, height: 2160},
|
||||
};
|
||||
export const SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS = 6_000_000;
|
||||
export const SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS = 9_000_000;
|
||||
export const SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS = 9_000_000;
|
||||
export const SUPPORTED_SCREEN_SHARE_FRAME_RATES = [15, 30, 60, 90, 120] as const;
|
||||
|
||||
export type SupportedScreenShareFrameRate = (typeof SUPPORTED_SCREEN_SHARE_FRAME_RATES)[number];
|
||||
@@ -52,15 +50,7 @@ const BITRATE_RUNGS = [
|
||||
'source',
|
||||
] as const satisfies ReadonlyArray<ScreenshareResolution>;
|
||||
|
||||
function isScreenShareDeliveryEnabled(delivery: boolean | undefined): boolean {
|
||||
return delivery ?? ScreenShareDeliveryRollout.enabled;
|
||||
}
|
||||
|
||||
export function resolveScreenShareFrameRate(frameRate: number, delivery?: boolean): SupportedScreenShareFrameRate {
|
||||
if (!isScreenShareDeliveryEnabled(delivery)) {
|
||||
if (frameRate >= 120) return 120;
|
||||
if (frameRate >= 90) return 90;
|
||||
}
|
||||
export function resolveScreenShareFrameRate(frameRate: number): SupportedScreenShareFrameRate {
|
||||
if (frameRate >= 60) return 60;
|
||||
if (frameRate >= 30) return 30;
|
||||
return 15;
|
||||
@@ -83,12 +73,10 @@ function computeScreenShareBitrateBps(
|
||||
pixels: number,
|
||||
rung: ScreenshareResolution,
|
||||
frameRate: SupportedScreenShareFrameRate,
|
||||
delivery: boolean,
|
||||
): number {
|
||||
const rungFloor = BITRATE_KBPS[rung][frameRate] * 1000;
|
||||
if (!delivery) return rungFloor;
|
||||
const computed = Math.round(SCREEN_SHARE_BITS_PER_PIXEL_PER_FRAME * pixels * frameRate);
|
||||
return Math.min(SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS, Math.max(rungFloor, computed));
|
||||
return Math.min(SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS, Math.max(rungFloor, computed));
|
||||
}
|
||||
|
||||
export function getScreenShareBitrateBps(
|
||||
@@ -98,15 +86,9 @@ export function getScreenShareBitrateBps(
|
||||
width: number;
|
||||
height: number;
|
||||
} | null,
|
||||
delivery?: boolean,
|
||||
): number {
|
||||
const fit = resolveEffectiveScreenShareDimensions(resolution, sourceDimensions);
|
||||
return computeScreenShareBitrateBps(
|
||||
fit.width * fit.height,
|
||||
fit.rung,
|
||||
frameRate,
|
||||
isScreenShareDeliveryEnabled(delivery),
|
||||
);
|
||||
return computeScreenShareBitrateBps(fit.width * fit.height, fit.rung, frameRate);
|
||||
}
|
||||
|
||||
export const STREAMING_MODE_PRESETS: Record<
|
||||
@@ -115,16 +97,6 @@ export const STREAMING_MODE_PRESETS: Record<
|
||||
resolution: ScreenshareResolution;
|
||||
frameRate: SupportedScreenShareFrameRate;
|
||||
}
|
||||
> = {
|
||||
gaming: {resolution: 'ultra', frameRate: 60},
|
||||
screenshare: {resolution: 'source', frameRate: 15},
|
||||
};
|
||||
const SCREEN_SHARE_DELIVERY_STREAMING_MODE_PRESETS: Record<
|
||||
Exclude<StreamingMode, 'custom'>,
|
||||
{
|
||||
resolution: ScreenshareResolution;
|
||||
frameRate: SupportedScreenShareFrameRate;
|
||||
}
|
||||
> = {
|
||||
gaming: {resolution: 'ultra', frameRate: 60},
|
||||
screenshare: {resolution: 'high', frameRate: 30},
|
||||
@@ -152,7 +124,6 @@ export interface ScreenShareBuildConfig {
|
||||
includeAudio: boolean;
|
||||
contentHint?: ScreenShareCaptureOptions['contentHint'];
|
||||
degradationPreference?: NonNullable<TrackPublishOptions['degradationPreference']>;
|
||||
delivery?: boolean;
|
||||
sourceDimensions?: {
|
||||
width: number;
|
||||
height: number;
|
||||
@@ -208,10 +179,9 @@ export function resolveEffectiveScreenShareDimensions(
|
||||
}
|
||||
|
||||
export function buildScreenShareOptions(config: ScreenShareBuildConfig): BuiltScreenShareOptions {
|
||||
const delivery = isScreenShareDeliveryEnabled(config.delivery);
|
||||
const {width, height, rung} = resolveEffectiveScreenShareDimensions(config.resolution, config.sourceDimensions);
|
||||
const resolvedFrameRate = resolveScreenShareFrameRate(config.frameRate, delivery);
|
||||
const maxBitrate = computeScreenShareBitrateBps(width * height, rung, resolvedFrameRate, delivery);
|
||||
const resolvedFrameRate = resolveScreenShareFrameRate(config.frameRate);
|
||||
const maxBitrate = computeScreenShareBitrateBps(width * height, rung, resolvedFrameRate);
|
||||
const video: ScreenShareVideoOptions = {
|
||||
cursor: resolveScreenShareCursorCapture(config.preferredDisplaySurface),
|
||||
...(config.preferredDisplaySurface ? {displaySurface: config.preferredDisplaySurface} : {}),
|
||||
@@ -235,7 +205,6 @@ export function buildScreenShareOptions(config: ScreenShareBuildConfig): BuiltSc
|
||||
contentHint: config.contentHint,
|
||||
maxBitrate,
|
||||
degradationPreference: config.degradationPreference,
|
||||
delivery,
|
||||
}),
|
||||
screenShareEncoding: {maxBitrate, maxFramerate: resolvedFrameRate, priority: 'high'},
|
||||
},
|
||||
@@ -270,17 +239,14 @@ export function resolveStreamingModeSettings(
|
||||
customResolution: ScreenshareResolution,
|
||||
customFrameRate: number,
|
||||
hasHigherQuality: boolean,
|
||||
delivery?: boolean,
|
||||
): {
|
||||
resolution: ScreenshareResolution;
|
||||
frameRate: SupportedScreenShareFrameRate;
|
||||
} {
|
||||
const enabled = isScreenShareDeliveryEnabled(delivery);
|
||||
const presets = enabled ? SCREEN_SHARE_DELIVERY_STREAMING_MODE_PRESETS : STREAMING_MODE_PRESETS;
|
||||
const resolved =
|
||||
mode === 'custom'
|
||||
? {resolution: customResolution, frameRate: resolveScreenShareFrameRate(customFrameRate, enabled)}
|
||||
: (hasHigherQuality ? presets : FREE_STREAMING_MODE_PRESETS)[mode];
|
||||
? {resolution: customResolution, frameRate: resolveScreenShareFrameRate(customFrameRate)}
|
||||
: (hasHigherQuality ? STREAMING_MODE_PRESETS : FREE_STREAMING_MODE_PRESETS)[mode];
|
||||
if (hasHigherQuality) {
|
||||
return resolved;
|
||||
}
|
||||
@@ -319,7 +285,6 @@ export interface ScreenShareQualityInput {
|
||||
storedFrameRate: number;
|
||||
entitled: boolean;
|
||||
context: ScreenShareContext;
|
||||
delivery?: boolean;
|
||||
}
|
||||
|
||||
export interface ScreenShareTargetInput extends ScreenShareQualityInput {
|
||||
@@ -341,7 +306,6 @@ export interface ScreenShareTarget {
|
||||
contentHint: ScreenShareCaptureOptions['contentHint'];
|
||||
degradationPreference: NonNullable<TrackPublishOptions['degradationPreference']>;
|
||||
softwareEncoderClamped: boolean;
|
||||
delivery?: boolean;
|
||||
presetOwned: boolean;
|
||||
tierLimited: boolean;
|
||||
deviceMapped: boolean;
|
||||
@@ -353,15 +317,11 @@ export interface ScreenShareDegradationInput {
|
||||
contentHint: ScreenShareCaptureOptions['contentHint'];
|
||||
maxBitrate: number;
|
||||
degradationPreference?: NonNullable<TrackPublishOptions['degradationPreference']>;
|
||||
delivery?: boolean;
|
||||
}
|
||||
|
||||
export function resolveScreenShareDegradationPreference(
|
||||
target: ScreenShareDegradationInput,
|
||||
): NonNullable<TrackPublishOptions['degradationPreference']> {
|
||||
if (!isScreenShareDeliveryEnabled(target.delivery)) {
|
||||
return target.context === 'device' ? 'balanced' : 'maintain-resolution';
|
||||
}
|
||||
if (target.degradationPreference) return target.degradationPreference;
|
||||
if (target.context === 'device') return 'balanced';
|
||||
if (target.rung === 'source') return 'maintain-resolution';
|
||||
@@ -393,17 +353,14 @@ function clampToSoftwareH264Budget(quality: {
|
||||
return {mode: quality.mode, resolution: cappedResolution, frameRate: cappedFrameRate};
|
||||
}
|
||||
|
||||
function resolveEffectiveScreenShareQuality(
|
||||
input: ScreenShareQualityInput,
|
||||
delivery: boolean,
|
||||
): {
|
||||
function resolveEffectiveScreenShareQuality(input: ScreenShareQualityInput): {
|
||||
mode: StreamingMode;
|
||||
resolution: ScreenshareResolution;
|
||||
frameRate: SupportedScreenShareFrameRate;
|
||||
} {
|
||||
const mode = normaliseStreamingModeForContext(input.mode, input.context);
|
||||
const resolution = normaliseResolutionForContext(input.storedResolution, input.context, input.entitled);
|
||||
const settings = resolveStreamingModeSettings(mode, resolution, input.storedFrameRate, input.entitled, delivery);
|
||||
const settings = resolveStreamingModeSettings(mode, resolution, input.storedFrameRate, input.entitled);
|
||||
return {mode, resolution: settings.resolution, frameRate: settings.frameRate};
|
||||
}
|
||||
|
||||
@@ -420,23 +377,20 @@ function resolveScreenShareContentHint(
|
||||
mode: StreamingMode,
|
||||
hintSetting: ScreenShareContentHint,
|
||||
context: ScreenShareContext,
|
||||
delivery: boolean,
|
||||
): ScreenShareCaptureOptions['contentHint'] {
|
||||
const hint = resolveScreenShareContentHintForMode(mode, hintSetting);
|
||||
if (!delivery) return hint;
|
||||
return hint === 'motion' && context !== 'device' ? undefined : hint;
|
||||
}
|
||||
|
||||
export function resolveScreenShareTarget(input: ScreenShareTargetInput): ScreenShareTarget {
|
||||
const delivery = isScreenShareDeliveryEnabled(input.delivery);
|
||||
const clamped = delivery && (input.softwareEncoderClamp === true || shouldClampToSoftwareH264(input.codec));
|
||||
const configured = resolveEffectiveScreenShareQuality(input, delivery);
|
||||
const clamped = input.softwareEncoderClamp === true || shouldClampToSoftwareH264(input.codec);
|
||||
const configured = resolveEffectiveScreenShareQuality(input);
|
||||
const effective = clamped ? clampToSoftwareH264Budget(configured) : configured;
|
||||
const configuredOnDisplay = resolveEffectiveScreenShareQuality({...input, context: 'display'}, delivery);
|
||||
const configuredOnDisplay = resolveEffectiveScreenShareQuality({...input, context: 'display'});
|
||||
const onDisplay = clamped ? clampToSoftwareH264Budget(configuredOnDisplay) : configuredOnDisplay;
|
||||
const fit = resolveEffectiveScreenShareDimensions(effective.resolution, input.sourceDimensions);
|
||||
const maxBitrate = computeScreenShareBitrateBps(fit.width * fit.height, fit.rung, effective.frameRate, delivery);
|
||||
const contentHint = resolveScreenShareContentHint(effective.mode, input.hintSetting, input.context, delivery);
|
||||
const maxBitrate = computeScreenShareBitrateBps(fit.width * fit.height, fit.rung, effective.frameRate);
|
||||
const contentHint = resolveScreenShareContentHint(effective.mode, input.hintSetting, input.context);
|
||||
return {
|
||||
mode: effective.mode,
|
||||
resolution: effective.resolution,
|
||||
@@ -452,16 +406,14 @@ export function resolveScreenShareTarget(input: ScreenShareTargetInput): ScreenS
|
||||
rung: fit.rung,
|
||||
contentHint,
|
||||
maxBitrate,
|
||||
delivery,
|
||||
}),
|
||||
softwareEncoderClamped: clamped,
|
||||
delivery,
|
||||
presetOwned: effective.mode !== 'custom',
|
||||
tierLimited:
|
||||
!input.entitled &&
|
||||
input.mode === 'custom' &&
|
||||
(PREMIUM_SCREEN_SHARE_RESOLUTIONS.includes(input.storedResolution) ||
|
||||
resolveScreenShareFrameRate(input.storedFrameRate, delivery) > FREE_TIER_MAX_FRAME_RATE),
|
||||
resolveScreenShareFrameRate(input.storedFrameRate) > FREE_TIER_MAX_FRAME_RATE),
|
||||
deviceMapped:
|
||||
effective.mode !== onDisplay.mode ||
|
||||
effective.resolution !== onDisplay.resolution ||
|
||||
@@ -570,13 +522,12 @@ export function resolveScreenShareSenderCodec(
|
||||
}
|
||||
|
||||
export function buildScreenShareSenderParameters(input: ScreenShareSenderParametersInput): ScreenShareSenderParameters {
|
||||
const delivery = input.target.delivery === true;
|
||||
const degradationPreference = resolveScreenShareDegradationPreference(input.target);
|
||||
const targetPixels = input.target.width * input.target.height;
|
||||
const capturePixels = input.capture ? input.capture.width * input.capture.height : null;
|
||||
const baseScale = Math.min(...input.encodings.map((encoding) => encoding.scaleResolutionDownBy ?? 1));
|
||||
const targetScale =
|
||||
capturePixels === null || (delivery && degradationPreference === 'maintain-framerate')
|
||||
capturePixels === null || degradationPreference === 'maintain-framerate'
|
||||
? 1
|
||||
: Math.max(1, Math.sqrt(capturePixels / targetPixels));
|
||||
const sentPixels = capturePixels === null ? null : capturePixels / (targetScale * targetScale);
|
||||
@@ -584,12 +535,7 @@ export function buildScreenShareSenderParameters(input: ScreenShareSenderParamet
|
||||
sentPixels !== null && sentPixels < targetPixels * SENT_PIXEL_FILL_RATIO
|
||||
? Math.min(
|
||||
input.target.maxBitrate,
|
||||
computeScreenShareBitrateBps(
|
||||
sentPixels,
|
||||
resolveBitrateRungForPixels(sentPixels),
|
||||
input.target.frameRate,
|
||||
delivery,
|
||||
),
|
||||
computeScreenShareBitrateBps(sentPixels, resolveBitrateRungForPixels(sentPixels), input.target.frameRate),
|
||||
)
|
||||
: input.target.maxBitrate;
|
||||
const bitrates = distributeScreenShareBitrate(input.encodings, maxBitrate);
|
||||
@@ -628,9 +574,8 @@ export function resolveScreenShareQualityPick(
|
||||
input: ScreenShareQualityInput,
|
||||
pick: ScreenShareQualityPick,
|
||||
): ScreenShareQualityPatch | null {
|
||||
const delivery = isScreenShareDeliveryEnabled(input.delivery);
|
||||
const effective = resolveEffectiveScreenShareQuality(input, delivery);
|
||||
const stored = resolveEffectiveScreenShareQuality({...input, mode: 'custom'}, delivery);
|
||||
const effective = resolveEffectiveScreenShareQuality(input);
|
||||
const stored = resolveEffectiveScreenShareQuality({...input, mode: 'custom'});
|
||||
if (pick.axis === 'resolution') {
|
||||
if (normaliseResolutionForContext(pick.resolution, input.context, input.entitled) !== pick.resolution) return null;
|
||||
if (pick.resolution === effective.resolution) return null;
|
||||
|
||||
@@ -8,7 +8,6 @@ import {resolveConfiguredScreenShareTarget} from '@app/features/voice/engine/voi
|
||||
import ActiveScreenShareSource from '@app/features/voice/state/ActiveScreenShareSource';
|
||||
import {clearDesktopSourceIntent, setDesktopSourceIntent} from '@app/features/voice/state/DesktopSourceIntent';
|
||||
import LocalVoiceState from '@app/features/voice/state/LocalVoiceState';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
|
||||
import {
|
||||
filterRoutableLinuxAudioSources,
|
||||
@@ -806,7 +805,6 @@ export async function switchConfiguredDisplayScreenShare(
|
||||
}
|
||||
|
||||
export async function restartActiveScreenShareCapture(): Promise<boolean> {
|
||||
if (!ScreenShareDeliveryRollout.enabled) return false;
|
||||
if (!didScreenShareStart()) return false;
|
||||
const publishedSource = ActiveScreenShareSource.getPublishedSource();
|
||||
const sourceId = ActiveScreenShareSource.getSourceId();
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {Logger} from '@app/features/platform/utils/AppLogger';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import type {VideoCodec} from 'livekit-client';
|
||||
|
||||
const logger = new Logger('VideoDecoderCapabilities');
|
||||
@@ -150,7 +149,7 @@ export function getVideoDecoderExclusionsSync(): Array<VideoCodec> | null {
|
||||
}
|
||||
|
||||
export function markScreenShareDecodeFailure(codec: VideoCodec, reason: string): boolean {
|
||||
if (ScreenShareDeliveryRollout.enabled && isBaselineWebRtcDecodeCodec(codec)) {
|
||||
if (isBaselineWebRtcDecodeCodec(codec)) {
|
||||
logger.warn('Keeping a baseline codec advertised despite a local screen share decode stall', {codec, reason});
|
||||
return false;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user