mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-18 21:45:31 +09:00
feat(voice): ship the screen share delivery rework to everyone (#2931)
This commit is contained in:
1 parent
88038a1d5b
commit
a0825e77c4
53 files changed
+171
-1806
No files matched your search
@@ -11,7 +11,6 @@ import {
|
||||
getLocalScreenShareVideoPublications,
|
||||
isLiveLocalTrackPublication,
|
||||
} from '@app/features/voice/engine/VoiceTrackPublicationUtils';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
|
||||
import {
|
||||
buildScreenShareCodecProfile,
|
||||
@@ -357,7 +356,6 @@ class ScreenShareCodecNegotiation {
|
||||
encoderMode: ScreenShareEncoderMode;
|
||||
order: ReadonlyArray<VideoCodec>;
|
||||
} | null = null;
|
||||
private frozenDelivery: {trackSid: string; enabled: boolean} | null = null;
|
||||
private lastCodecChangeAt = 0;
|
||||
private suppressionTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private publishedTrackSid: string | null = null;
|
||||
@@ -452,22 +450,9 @@ class ScreenShareCodecNegotiation {
|
||||
this.localCodecs = buildLocalCodecAdvertisements(this.resolveCodecPreferenceOrder());
|
||||
}
|
||||
|
||||
private resolveScreenShareDelivery(room: Room | null = this.room): boolean {
|
||||
const trackSid = this.getLocalScreenSharePublication(room)?.trackSid ?? null;
|
||||
if (trackSid === null) {
|
||||
this.frozenDelivery = null;
|
||||
return ScreenShareDeliveryRollout.enabled;
|
||||
}
|
||||
if (this.frozenDelivery?.trackSid === trackSid) return this.frozenDelivery.enabled;
|
||||
const enabled = ScreenShareDeliveryRollout.enabled;
|
||||
this.frozenDelivery = {trackSid, enabled};
|
||||
return enabled;
|
||||
}
|
||||
|
||||
private resolveCodecPreferenceOrder(
|
||||
preference: CodecPreference = VoiceSettings.getPreferredScreenShareCodec(),
|
||||
): ReadonlyArray<VideoCodec> {
|
||||
if (!this.resolveScreenShareDelivery()) return getScreenShareCodecPreferenceOrder(preference);
|
||||
const encoderMode = VoiceSettings.getScreenShareEncoderMode();
|
||||
const frozen = this.frozenCodecPreference;
|
||||
if (
|
||||
@@ -523,7 +508,7 @@ class ScreenShareCodecNegotiation {
|
||||
unknownParticipants: number;
|
||||
} {
|
||||
const now = Date.now();
|
||||
const viewerIdentities = this.resolveScreenShareDelivery(room) ? this.getScreenShareViewerIdentities(room) : null;
|
||||
const viewerIdentities = this.getScreenShareViewerIdentities(room);
|
||||
const knownRemoteCodecs: Array<Array<FluxerCodecAdvertisement>> = [];
|
||||
const participants: Array<{identity: string; firstSeenAt: number}> = [];
|
||||
for (const participant of room?.remoteParticipants.values() ?? []) {
|
||||
@@ -640,7 +625,6 @@ class ScreenShareCodecNegotiation {
|
||||
this.selectedCodec = null;
|
||||
this.localCodecs = [];
|
||||
this.frozenCodecPreference = null;
|
||||
this.frozenDelivery = null;
|
||||
this.lastCodecChangeAt = 0;
|
||||
this.publishedTrackSid = null;
|
||||
this.clearSuppressionTimer();
|
||||
@@ -749,7 +733,6 @@ class ScreenShareCodecNegotiation {
|
||||
const {knownRemoteCodecs, unknownParticipants} = this.getRemoteCodecInputs(room);
|
||||
this.scheduleGraceReevaluations(room, bindingRevision);
|
||||
const previousCodec = this.selectedCodec;
|
||||
const delivery = this.resolveScreenShareDelivery(room);
|
||||
this.negotiationSnapshot = transitionScreenShareCodecNegotiationSnapshot(this.negotiationSnapshot, {
|
||||
type: 'negotiation.evaluate',
|
||||
localCodecs: this.localCodecs,
|
||||
@@ -757,7 +740,7 @@ class ScreenShareCodecNegotiation {
|
||||
unknownParticipants,
|
||||
reason,
|
||||
codecPreference: this.resolveCodecPreferenceOrder(),
|
||||
publishedCodec: delivery ? this.observePublishedScreenShareCodec(room) : null,
|
||||
publishedCodec: this.observePublishedScreenShareCodec(room),
|
||||
});
|
||||
const selection = this.negotiationSnapshot.context.selection;
|
||||
if (!selection) return null;
|
||||
@@ -765,11 +748,7 @@ class ScreenShareCodecNegotiation {
|
||||
this.selectedCodec = selection.codec;
|
||||
return selection;
|
||||
}
|
||||
if (
|
||||
delivery &&
|
||||
previousCodec !== null &&
|
||||
Date.now() - this.lastCodecChangeAt < SCREEN_SHARE_CODEC_CHANGE_SUPPRESSION_MS
|
||||
) {
|
||||
if (previousCodec !== null && Date.now() - this.lastCodecChangeAt < SCREEN_SHARE_CODEC_CHANGE_SUPPRESSION_MS) {
|
||||
logger.debug('Suppressed a screen share codec change inside the change window', {
|
||||
codec: selection.codec,
|
||||
previousCodec,
|
||||
@@ -779,7 +758,7 @@ class ScreenShareCodecNegotiation {
|
||||
return selection;
|
||||
}
|
||||
this.selectedCodec = selection.codec;
|
||||
if (delivery) this.armCodecChangeSuppression();
|
||||
this.armCodecChangeSuppression();
|
||||
this.mediaSessionId = createId('media');
|
||||
logger.info('Selected screen share codec from XState capability intersection', selection);
|
||||
await this.publishSessionUpdate(room, selection);
|
||||
|
||||
@@ -9,12 +9,7 @@ import {
|
||||
type VoiceGatewayStateEvent,
|
||||
type VoiceGatewayStateSnapshot,
|
||||
} from '@app/features/voice/engine/VoiceGatewayStateMachine';
|
||||
import {describe, expect, it, vi} from 'vitest';
|
||||
|
||||
vi.mock('@app/features/voice/state/ScreenShareDeliveryRollout', () => ({
|
||||
ScreenShareDeliveryRollout: {enabled: false},
|
||||
default: {enabled: false},
|
||||
}));
|
||||
import {describe, expect, it} from 'vitest';
|
||||
|
||||
function voiceState(overrides: Partial<VoiceState> = {}): VoiceState {
|
||||
return {
|
||||
|
||||
@@ -19,12 +19,7 @@ import {
|
||||
type VoiceMediaGraphStreamTileState,
|
||||
} from '@app/features/voice/engine/VoiceMediaGraphTileState';
|
||||
import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
|
||||
import {describe, expect, it, vi} from 'vitest';
|
||||
|
||||
vi.mock('@app/features/voice/state/ScreenShareDeliveryRollout', () => ({
|
||||
ScreenShareDeliveryRollout: {enabled: false},
|
||||
default: {enabled: false},
|
||||
}));
|
||||
import {describe, expect, it} from 'vitest';
|
||||
|
||||
const SIMULATION_SEEDS: ReadonlyArray<number> = Array.from({length: 20}, (_, index) => 1 + index * 7919);
|
||||
const SIMULATION_STEP_COUNT = 1000;
|
||||
|
||||
@@ -11,7 +11,6 @@ import {
|
||||
voiceMediaGraphStatsTrackKey,
|
||||
} from '@app/features/voice/engine/VoiceMediaGraphStatsObservations';
|
||||
import type {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import type {VoiceEngineV2PerTrackStats} from '@fluxer/voice_engine_v2';
|
||||
|
||||
const VOICE_MEDIA_GRAPH_STATS_TRACK_LIMIT = 1024;
|
||||
@@ -77,9 +76,7 @@ function perTrackStatToInfo(track: VoiceEngineV2PerTrackStats): VoiceMediaGraphP
|
||||
if (track.kind !== 'video') return null;
|
||||
const width = track.frameWidth ?? track.sourceFrameWidth;
|
||||
const height = track.frameHeight ?? track.sourceFrameHeight;
|
||||
const fps = ScreenShareDeliveryRollout.enabled
|
||||
? (track.effectiveFramesPerSecond ?? track.framesPerSecond)
|
||||
: (track.effectiveFramesPerSecond ?? track.framesPerSecond ?? track.sourceFramesPerSecond);
|
||||
const fps = track.effectiveFramesPerSecond ?? track.framesPerSecond;
|
||||
const info: VoiceMediaGraphPartialTrackInfo = {};
|
||||
if (isPositiveDimension(width) && isPositiveDimension(height)) {
|
||||
info.width = width;
|
||||
@@ -193,9 +190,7 @@ function observationToPartialTrackInfo(
|
||||
): VoiceMediaGraphPartialTrackInfo | null {
|
||||
const width = observation.width ?? observation.sourceWidth ?? undefined;
|
||||
const height = observation.height ?? observation.sourceHeight ?? undefined;
|
||||
const fps = ScreenShareDeliveryRollout.enabled
|
||||
? (observation.fps ?? undefined)
|
||||
: (observation.fps ?? observation.sourceFps ?? undefined);
|
||||
const fps = observation.fps ?? undefined;
|
||||
const info: VoiceMediaGraphPartialTrackInfo = {};
|
||||
if (isPositiveDimension(width) && isPositiveDimension(height)) {
|
||||
info.width = width;
|
||||
|
||||
@@ -16,7 +16,6 @@ import {
|
||||
} from '@app/features/voice/engine/VoiceStreamWatchState';
|
||||
import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
|
||||
import ParticipantVolume from '@app/features/voice/state/ParticipantVolume';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import {ScreenShareWatchErrorCode, ScreenShareWatchFailures} from '@app/features/voice/state/ScreenShareWatchFailures';
|
||||
import {monitorScreenShareDecodeHealth} from '@app/features/voice/utils/ScreenShareCodecDiagnostics';
|
||||
import {markScreenShareDecodeFailure} from '@app/features/voice/utils/VideoDecoderCapabilities';
|
||||
@@ -108,30 +107,6 @@ interface ParticipantSpeakingDisposer {
|
||||
dispose: () => void;
|
||||
}
|
||||
|
||||
type LatencyTunedReceiver = RTCRtpReceiver & {
|
||||
jitterBufferTarget?: number;
|
||||
playoutDelayHint?: number;
|
||||
};
|
||||
|
||||
function getRemoteTrackReceiver(track: RemoteTrack): LatencyTunedReceiver | undefined {
|
||||
return (track as RemoteTrack & {receiver?: LatencyTunedReceiver}).receiver;
|
||||
}
|
||||
|
||||
function applyInteractiveReceiverBuffer(track: RemoteTrack, pub: RemoteTrackPublication): void {
|
||||
const receiver = getRemoteTrackReceiver(track);
|
||||
if (!receiver) return;
|
||||
try {
|
||||
if (pub.kind === Track.Kind.Video && pub.source === Track.Source.ScreenShare) {
|
||||
receiver.jitterBufferTarget = 80;
|
||||
receiver.playoutDelayHint = 0.04;
|
||||
} else if (pub.kind === Track.Kind.Audio) {
|
||||
receiver.jitterBufferTarget = 60;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.debug('Failed to apply interactive receiver buffer target', {error, source: pub.source, kind: pub.kind});
|
||||
}
|
||||
}
|
||||
|
||||
export function bindRoomEvents(
|
||||
room: Room,
|
||||
attemptId: number,
|
||||
@@ -141,7 +116,6 @@ export function bindRoomEvents(
|
||||
dependencies: RoomEventDependencies,
|
||||
): void {
|
||||
const guard = dependencies.connection.createGuardedHandler;
|
||||
const screenShareDeliveryEnabled = ScreenShareDeliveryRollout.enabled;
|
||||
const participantSpeakingDisposers = new Map<string, ParticipantSpeakingDisposer>();
|
||||
const screenShareDecodeMonitorCancels = new Map<string, () => void>();
|
||||
const remoteTrackLifecycleDisposers = new Map<string, () => void>();
|
||||
@@ -443,9 +417,6 @@ export function bindRoomEvents(
|
||||
trackSid: pub.trackSid,
|
||||
});
|
||||
}
|
||||
if (!screenShareDeliveryEnabled) {
|
||||
applyInteractiveReceiverBuffer(track, pub);
|
||||
}
|
||||
monitorDecodeHealth(track, pub);
|
||||
dependencies.remoteSpeaking.attachIfApplicable(participant, pub, track);
|
||||
bindRemoteTrackLifecycleIfApplicable(participant, pub, track);
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import assert from 'node:assert/strict';
|
||||
import {isElectronPlatform} from '@app/features/platform/types/Platform';
|
||||
import {Logger} from '@app/features/platform/utils/AppLogger';
|
||||
import {Store} from '@app/features/voice/engine/Store';
|
||||
import {sendVoiceStateDisconnect} from '@app/features/voice/engine/VoiceChannelConnector';
|
||||
@@ -37,7 +36,6 @@ import {
|
||||
isReadyToRepublishTrack,
|
||||
} from '@app/features/voice/engine/v2/VoiceEngineV2AppAdapterAssertions';
|
||||
import {VoiceEngineV2AppReconnectPolicy} from '@app/features/voice/engine/v2/VoiceEngineV2AppReconnectPolicy';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import VoiceRegionTeleport from '@app/features/voice/state/VoiceRegionTeleport';
|
||||
import {
|
||||
findVideoPublishCodecPolicyViolation,
|
||||
@@ -45,10 +43,7 @@ import {
|
||||
} from '@app/features/voice/utils/CodecCapabilityDetector';
|
||||
import {getH264HardwareProfilesSync} from '@app/features/voice/utils/GpuEncoderCapabilities';
|
||||
import {setNoiseSuppressionScopeGuildId} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionSelection';
|
||||
import {
|
||||
SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS,
|
||||
SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
|
||||
} from '@app/features/voice/utils/ScreenShareOptions';
|
||||
import {SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS} from '@app/features/voice/utils/ScreenShareOptions';
|
||||
import {
|
||||
clearScreenShareDecodeFailures,
|
||||
getVideoDecoderExclusionsSync,
|
||||
@@ -151,9 +146,7 @@ function createWebAudioMixOption(): RoomOptions['webAudioMix'] {
|
||||
function createRoomPublishDefaults(): RoomOptions['publishDefaults'] {
|
||||
return {
|
||||
screenShareEncoding: {
|
||||
maxBitrate: ScreenShareDeliveryRollout.enabled
|
||||
? SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS
|
||||
: SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
|
||||
maxBitrate: SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
|
||||
maxFramerate: 30,
|
||||
priority: 'high',
|
||||
},
|
||||
@@ -164,7 +157,6 @@ function createRoomPublishDefaults(): RoomOptions['publishDefaults'] {
|
||||
function createRoomOptions(
|
||||
e2eeKey: string | null,
|
||||
subscriberVideoCodecExclusions: RoomOptions['subscriberVideoCodecExclusions'],
|
||||
screenShareDelivery: boolean,
|
||||
): {
|
||||
roomOptions: RoomOptions;
|
||||
e2eeKeyProvider: ExternalE2EEKeyProvider | null;
|
||||
@@ -176,8 +168,7 @@ function createRoomOptions(
|
||||
webAudioMix: createWebAudioMixOption(),
|
||||
publishDefaults: createRoomPublishDefaults(),
|
||||
subscriberVideoCodecExclusions,
|
||||
screenShareDelivery,
|
||||
h264HardwareProfiles: screenShareDelivery ? getH264HardwareProfilesSync()?.profiles : undefined,
|
||||
h264HardwareProfiles: getH264HardwareProfilesSync()?.profiles,
|
||||
};
|
||||
let e2eeKeyProvider: ExternalE2EEKeyProvider | null = null;
|
||||
let e2eeWorker: Worker | null = null;
|
||||
@@ -196,19 +187,11 @@ function createRoomOptions(
|
||||
return {roomOptions, e2eeKeyProvider, e2eeWorker};
|
||||
}
|
||||
|
||||
function createRoomConnectOptions(screenShareDelivery: boolean): RoomConnectOptions {
|
||||
function createRoomConnectOptions(): RoomConnectOptions {
|
||||
const connectOptions: RoomConnectOptions = {
|
||||
autoSubscribe: false,
|
||||
};
|
||||
assert.equal(connectOptions.autoSubscribe, false, 'LiveKit connect options must not auto-subscribe');
|
||||
if (!screenShareDelivery && isElectronPlatform()) {
|
||||
connectOptions.rtcConfig = {iceTransportPolicy: 'relay'};
|
||||
assert.equal(
|
||||
connectOptions.rtcConfig.iceTransportPolicy,
|
||||
'relay',
|
||||
'Electron LiveKit connects must force relay ICE',
|
||||
);
|
||||
}
|
||||
return connectOptions;
|
||||
}
|
||||
|
||||
@@ -564,12 +547,7 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
|
||||
logger.warn('Aborting LiveKit room creation after codec probing because attempt is stale', {attemptId});
|
||||
return;
|
||||
}
|
||||
const screenShareDelivery = ScreenShareDeliveryRollout.enabled;
|
||||
const {roomOptions, e2eeKeyProvider, e2eeWorker} = createRoomOptions(
|
||||
e2eeKey,
|
||||
subscriberVideoCodecExclusions,
|
||||
screenShareDelivery,
|
||||
);
|
||||
const {roomOptions, e2eeKeyProvider, e2eeWorker} = createRoomOptions(e2eeKey, subscriberVideoCodecExclusions);
|
||||
const room = new LiveKitRoom(roomOptions);
|
||||
ownE2EEWorker(room, e2eeWorker);
|
||||
let roomClosed = false;
|
||||
@@ -602,7 +580,7 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
|
||||
return;
|
||||
}
|
||||
logger.info('Attempting to connect to LiveKit', {endpoint, guildId, channelId: resolvedChannelId});
|
||||
const connectOptions = createRoomConnectOptions(screenShareDelivery);
|
||||
const connectOptions = createRoomConnectOptions();
|
||||
room
|
||||
.connect(endpoint, token, connectOptions)
|
||||
.then(() => {
|
||||
@@ -704,14 +682,12 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
|
||||
const connectionId = raw.connection_id ?? null;
|
||||
this.abortHotSwap();
|
||||
const cachedExclusions = getVideoDecoderExclusionsSync();
|
||||
const screenShareDelivery = ScreenShareDeliveryRollout.enabled;
|
||||
const roomOptions: RoomOptions = {
|
||||
adaptiveStream: false,
|
||||
dynacast: true,
|
||||
webAudioMix: createWebAudioMixOption(),
|
||||
publishDefaults: createRoomPublishDefaults(),
|
||||
subscriberVideoCodecExclusions: cachedExclusions && cachedExclusions.length > 0 ? cachedExclusions : undefined,
|
||||
screenShareDelivery,
|
||||
};
|
||||
if (!this.isLatestConnectionAttempt(attemptId) || this.connectionState.room !== existingRoom) {
|
||||
logger.warn('Region hot-swap: aborted before room creation because attempt is stale', {attemptId});
|
||||
@@ -728,7 +704,7 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
|
||||
this.abortHotSwap();
|
||||
}
|
||||
});
|
||||
const connectOptions = createRoomConnectOptions(screenShareDelivery);
|
||||
const connectOptions = createRoomConnectOptions();
|
||||
logger.info('Region hot-swap: connecting to new endpoint', {endpoint, guildId, channelId});
|
||||
newRoom
|
||||
.connect(endpoint, token, connectOptions)
|
||||
|
||||
+3
-12
@@ -63,7 +63,6 @@ import ActiveScreenShareSource, {
|
||||
type PublishedScreenShareSource,
|
||||
} from '@app/features/voice/state/ActiveScreenShareSource';
|
||||
import LocalVoiceState from '@app/features/voice/state/LocalVoiceState';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
|
||||
import {
|
||||
prepareHighFidelityScreenShareAudioTrack,
|
||||
@@ -183,7 +182,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
encoderVerificationTimer: (() => void) | null = null;
|
||||
private screenShareTrackingHolds = 0;
|
||||
private screenShareCapturePaused = false;
|
||||
private screenShareDeliveryArmed = false;
|
||||
private screenShareTrackingGeneration = 0;
|
||||
private readonly reconciledCodecPairs = new Set<string>();
|
||||
private readonly verifiedCodecCorrectionsByTrack = new WeakMap<MediaStreamTrack, number>();
|
||||
@@ -679,7 +677,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
preferredTrack?: LocalVideoTrack,
|
||||
): void {
|
||||
this.cleanupActiveScreenShareEndListenerInternal();
|
||||
this.screenShareDeliveryArmed = ScreenShareDeliveryRollout.enabled;
|
||||
const publication = preferredTrack ? undefined : participant.getTrackPublication(Track.Source.ScreenShare);
|
||||
const videoTrack = preferredTrack ?? publication?.videoTrack;
|
||||
const mediaStreamTrack = videoTrack?.mediaStreamTrack;
|
||||
@@ -730,15 +727,11 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
};
|
||||
const onMuted = (): void => reportCapturePaused(true);
|
||||
const onUnmuted = (): void => reportCapturePaused(false);
|
||||
const tracksCapturePauses = this.screenShareDeliveryArmed;
|
||||
mediaStreamTrack.addEventListener('ended', onEnded);
|
||||
if (tracksCapturePauses) {
|
||||
mediaStreamTrack.addEventListener('mute', onMuted);
|
||||
mediaStreamTrack.addEventListener('unmute', onUnmuted);
|
||||
}
|
||||
mediaStreamTrack.addEventListener('mute', onMuted);
|
||||
mediaStreamTrack.addEventListener('unmute', onUnmuted);
|
||||
this.activeScreenShareEndListener = () => {
|
||||
mediaStreamTrack.removeEventListener('ended', onEnded);
|
||||
if (!tracksCapturePauses) return;
|
||||
mediaStreamTrack.removeEventListener('mute', onMuted);
|
||||
mediaStreamTrack.removeEventListener('unmute', onUnmuted);
|
||||
};
|
||||
@@ -1152,7 +1145,7 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
nextConstraints.frameRate = {ideal: resolution.frameRate, max: resolution.frameRate};
|
||||
}
|
||||
if (JSON.stringify(currentConstraints) === JSON.stringify(nextConstraints)) return 'unchanged';
|
||||
if (this.screenShareDeliveryArmed && screenShareSourceHasClonedTracks(screenShareTrack)) {
|
||||
if (screenShareSourceHasClonedTracks(screenShareTrack)) {
|
||||
logger.warn('Screen share capture geometry needs a restart while a backup codec holds a clone of the source', {
|
||||
resolution,
|
||||
});
|
||||
@@ -1202,7 +1195,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
this.ensureScreenShareKeepAliveSinkInternal(participant);
|
||||
updateLocalParticipantFromRoom(room);
|
||||
this.syncLocalScreenShareAudioStateInternal(participant, participant.isScreenShareEnabled);
|
||||
if (!this.screenShareDeliveryArmed) return true;
|
||||
return geometryUpdate !== 'needs-capture-restart' && geometryUpdate !== 'failed';
|
||||
}
|
||||
|
||||
@@ -1225,7 +1217,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
|
||||
}
|
||||
|
||||
resetStreamTracking(): void {
|
||||
this.screenShareDeliveryArmed = false;
|
||||
this.setScreenShareCapturePausedInternal(false);
|
||||
this.clearScreenShareKeepAliveSinkInternal();
|
||||
this.setStreamingPriorityInternal(false);
|
||||
|
||||
@@ -32,11 +32,6 @@ const voiceStates: Record<string, Record<string, Record<string, unknown>>> = {};
|
||||
const settingsUpdate = vi.fn();
|
||||
const openPremiumModal = vi.fn();
|
||||
|
||||
vi.mock('@app/features/voice/state/ScreenShareDeliveryRollout', () => ({
|
||||
ScreenShareDeliveryRollout: {enabled: true},
|
||||
default: {enabled: true},
|
||||
}));
|
||||
|
||||
vi.mock('@app/features/voice/utils/GpuEncoderCapabilities', () => ({
|
||||
getGpuEncoderReportSync: () => gpuReport,
|
||||
getH264HardwareProfilesSync: () => h264HardwareProfiles,
|
||||
|
||||
-48
@@ -6,13 +6,11 @@ import {
|
||||
stopMediaTrack,
|
||||
stopUnselectedStreamTracks,
|
||||
} from '@app/features/voice/engine/voice_screen_share_manager/shared';
|
||||
import ActiveScreenShareSource from '@app/features/voice/state/ActiveScreenShareSource';
|
||||
import {rememberCapturedDisplayAudioTrack} from '@app/features/voice/utils/NativeAudioCaptureBridge';
|
||||
import {ScreenShareAudioCaptureError} from '@app/features/voice/utils/ScreenShareAudioCaptureError';
|
||||
import type {ScreenShareCaptureOptions} from 'livekit-client';
|
||||
|
||||
type DisplayMediaVideoConstraints = MediaTrackConstraints & {
|
||||
colorSpace?: string;
|
||||
cursor?: 'always' | 'motion' | 'never';
|
||||
displaySurface?: 'browser' | 'monitor' | 'window';
|
||||
};
|
||||
@@ -20,37 +18,12 @@ type DisplayMediaAudioConstraints = MediaTrackConstraints & {
|
||||
restrictOwnAudio?: boolean;
|
||||
suppressLocalAudioPlayback?: boolean;
|
||||
};
|
||||
type DisplayMediaTrackSettings = MediaTrackSettings & {
|
||||
cursor?: 'always' | 'motion' | 'never';
|
||||
displaySurface?: 'browser' | 'monitor' | 'window';
|
||||
};
|
||||
function resolveDisplayMediaCursorCapture(
|
||||
displaySurface: DisplayMediaVideoConstraints['displaySurface'],
|
||||
): 'always' | 'motion' | 'never' {
|
||||
return displaySurface === 'window' ? 'never' : 'always';
|
||||
}
|
||||
|
||||
function getRequestedDisplayMediaVideoConstraints(
|
||||
options: ScreenShareCaptureOptions | undefined,
|
||||
): DisplayMediaVideoConstraints | null {
|
||||
if (typeof options?.video !== 'object' || !options.video) return null;
|
||||
return options.video as DisplayMediaVideoConstraints;
|
||||
}
|
||||
|
||||
function resolveCapturedDisplayMediaCursorCapture(
|
||||
track: Pick<MediaStreamTrack, 'getSettings'>,
|
||||
options?: ScreenShareCaptureOptions,
|
||||
): 'always' | 'motion' | 'never' {
|
||||
const requestedVideo = getRequestedDisplayMediaVideoConstraints(options);
|
||||
const requestedCursor = requestedVideo?.cursor;
|
||||
const requestedDisplaySurface = requestedVideo?.displaySurface;
|
||||
if (requestedCursor && requestedCursor !== resolveDisplayMediaCursorCapture(requestedDisplaySurface)) {
|
||||
return requestedCursor;
|
||||
}
|
||||
const settings = track.getSettings() as DisplayMediaTrackSettings;
|
||||
return resolveDisplayMediaCursorCapture(settings.displaySurface ?? requestedDisplaySurface);
|
||||
}
|
||||
|
||||
export function getDisplayMediaOptions(options?: ScreenShareCaptureOptions): DisplayMediaStreamOptions {
|
||||
let videoConstraints: MediaTrackConstraints | boolean = options?.video ?? true;
|
||||
const resolution = options?.resolution;
|
||||
@@ -95,21 +68,6 @@ export function getDisplayMediaOptions(options?: ScreenShareCaptureOptions): Dis
|
||||
} as DisplayMediaStreamOptions;
|
||||
}
|
||||
|
||||
function buildCapturedDisplayMediaConstraints(
|
||||
displayMediaOptions: DisplayMediaStreamOptions,
|
||||
cursor: 'always' | 'motion' | 'never',
|
||||
): MediaTrackConstraints {
|
||||
const requestedVideo =
|
||||
typeof displayMediaOptions.video === 'object' && displayMediaOptions.video
|
||||
? (displayMediaOptions.video as DisplayMediaVideoConstraints)
|
||||
: undefined;
|
||||
const constraints: DisplayMediaVideoConstraints = {colorSpace: 'rec709', cursor};
|
||||
if (requestedVideo?.width !== undefined) constraints.width = requestedVideo.width;
|
||||
if (requestedVideo?.height !== undefined) constraints.height = requestedVideo.height;
|
||||
if (requestedVideo?.frameRate !== undefined) constraints.frameRate = requestedVideo.frameRate;
|
||||
return constraints;
|
||||
}
|
||||
|
||||
export async function createDisplayScreenShareTracks(
|
||||
options?: ScreenShareCaptureOptions,
|
||||
captureContext?: DisplayScreenShareCaptureContext,
|
||||
@@ -127,12 +85,6 @@ export async function createDisplayScreenShareTracks(
|
||||
if (options?.contentHint) {
|
||||
videoTrack.contentHint = options.contentHint;
|
||||
}
|
||||
if (ActiveScreenShareSource.getTarget()?.delivery !== true) {
|
||||
const cursor = resolveCapturedDisplayMediaCursorCapture(videoTrack, options);
|
||||
await videoTrack
|
||||
.applyConstraints(buildCapturedDisplayMediaConstraints(displayMediaOptions, cursor))
|
||||
.catch(() => undefined);
|
||||
}
|
||||
const capturedAudioTrack = stream.getAudioTracks()[0];
|
||||
const audioTrack = capturedAudioTrack?.readyState === 'live' ? capturedAudioTrack : undefined;
|
||||
rememberCapturedDisplayAudioTrack(capturedAudioTrack, Boolean(capturedAudioTrack) && !audioTrack);
|
||||
|
||||
@@ -4,7 +4,6 @@ import {getDesktopTroubleshootingSettings} from '@app/features/devtools/utils/De
|
||||
import {Logger} from '@app/features/platform/utils/AppLogger';
|
||||
import ScreenShareCodecNegotiation from '@app/features/voice/engine/ScreenShareCodecNegotiation';
|
||||
import ActiveScreenShareSource from '@app/features/voice/state/ActiveScreenShareSource';
|
||||
import ScreenShareDeliveryRollout from '@app/features/voice/state/ScreenShareDeliveryRollout';
|
||||
import SoftwareEncoderWarning from '@app/features/voice/state/SoftwareEncoderWarning';
|
||||
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
|
||||
import {
|
||||
@@ -23,7 +22,6 @@ import {
|
||||
resolveScreenShareLayering,
|
||||
resolveScreenShareSenderCodec,
|
||||
resolveScreenShareTarget,
|
||||
SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS,
|
||||
SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS,
|
||||
type ScreenShareContext,
|
||||
type ScreenShareLayering,
|
||||
@@ -195,11 +193,13 @@ export async function releaseScreenShareCaptureCleanup(snapshot: ScreenShareCapt
|
||||
}
|
||||
}
|
||||
|
||||
function clampScreenShareEncoding(encoding: VideoEncoding, delivery: boolean): VideoEncoding {
|
||||
const ceiling = delivery ? SCREEN_SHARE_DELIVERY_MAX_VIDEO_BITRATE_BPS : SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS;
|
||||
function clampScreenShareEncoding(encoding: VideoEncoding): VideoEncoding {
|
||||
return {
|
||||
...encoding,
|
||||
maxBitrate: typeof encoding.maxBitrate === 'number' ? Math.min(encoding.maxBitrate, ceiling) : encoding.maxBitrate,
|
||||
maxBitrate:
|
||||
typeof encoding.maxBitrate === 'number'
|
||||
? Math.min(encoding.maxBitrate, SCREEN_SHARE_MAX_VIDEO_BITRATE_BPS)
|
||||
: encoding.maxBitrate,
|
||||
priority: encoding.priority ?? 'high',
|
||||
};
|
||||
}
|
||||
@@ -208,15 +208,10 @@ export function resolveActiveScreenShareContext(): ScreenShareContext {
|
||||
return ActiveScreenShareSource.getShareContext() ?? 'display';
|
||||
}
|
||||
|
||||
function resolveScreenShareDeliveryArm(): boolean {
|
||||
return ActiveScreenShareSource.getTarget()?.delivery ?? ScreenShareDeliveryRollout.enabled;
|
||||
}
|
||||
|
||||
export function resolveConfiguredScreenShareTarget(
|
||||
context: ScreenShareContext,
|
||||
sourceDimensions: {width: number; height: number} | null,
|
||||
): ScreenShareTarget {
|
||||
const delivery = resolveScreenShareDeliveryArm();
|
||||
return resolveScreenShareTarget({
|
||||
mode: VoiceSettings.getStreamingMode(),
|
||||
storedResolution: VoiceSettings.getScreenshareResolution(),
|
||||
@@ -225,13 +220,8 @@ export function resolveConfiguredScreenShareTarget(
|
||||
context,
|
||||
sourceDimensions,
|
||||
hintSetting: VoiceSettings.getScreenShareContentHint(),
|
||||
delivery,
|
||||
...(delivery
|
||||
? {
|
||||
codec: getPreferredScreenShareCodec(),
|
||||
softwareEncoderClamp: ActiveScreenShareSource.isSoftwareEncoderClamped(),
|
||||
}
|
||||
: {}),
|
||||
codec: getPreferredScreenShareCodec(),
|
||||
softwareEncoderClamp: ActiveScreenShareSource.isSoftwareEncoderClamped(),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -242,7 +232,6 @@ export function resolveActiveScreenShareTarget(
|
||||
}
|
||||
|
||||
function recommitClampedScreenShareTarget(committed: ScreenShareTarget): ScreenShareTarget {
|
||||
if (committed.delivery !== true) return committed;
|
||||
if (committed.softwareEncoderClamped) return committed;
|
||||
const resolved = resolveActiveScreenShareTarget(committed.context);
|
||||
if (!resolved.softwareEncoderClamped) return committed;
|
||||
@@ -270,7 +259,7 @@ export function getStatsKind(
|
||||
|
||||
function resolveScreenShareEncoding(target: ScreenShareTarget, publishOptions?: TrackPublishOptions): VideoEncoding {
|
||||
if (publishOptions?.screenShareEncoding) {
|
||||
return clampScreenShareEncoding(publishOptions.screenShareEncoding, target.delivery === true);
|
||||
return clampScreenShareEncoding(publishOptions.screenShareEncoding);
|
||||
}
|
||||
return {maxBitrate: target.maxBitrate, maxFramerate: target.frameRate, priority: 'high'};
|
||||
}
|
||||
@@ -863,27 +852,22 @@ export function startScreenShareEncoderMonitor(options: ScreenShareEncoderMonito
|
||||
let framesEncoded: number | null = null;
|
||||
let sendSnapshot: ScreenShareSendSnapshot | null = null;
|
||||
let cpuLimitedTicks = 0;
|
||||
const delivery = ActiveScreenShareSource.getTarget()?.delivery === true;
|
||||
const tick = async (): Promise<void> => {
|
||||
const sender = options.track.sender;
|
||||
if (!sender) return;
|
||||
const stats = await sender.getStats();
|
||||
if (delivery) {
|
||||
syncScreenShareCaptureSize(sender);
|
||||
}
|
||||
syncScreenShareCaptureSize(sender);
|
||||
const encoded = countEncodedVideoFrames(stats);
|
||||
ActiveScreenShareSource.setEncoding(encoded !== null && framesEncoded !== null && encoded > framesEncoded);
|
||||
framesEncoded = encoded;
|
||||
if (delivery) {
|
||||
const snapshot = collectScreenShareSendSnapshot(stats);
|
||||
cpuLimitedTicks = snapshot.cpuLimited ? cpuLimitedTicks + 1 : 0;
|
||||
const limit = classifyScreenShareSendLimit(sendSnapshot, snapshot, cpuLimitedTicks);
|
||||
sendSnapshot = snapshot;
|
||||
if (limit !== ActiveScreenShareSource.getLimit()) {
|
||||
logger.info('Screen share send limit changed', {limit, codec: options.codec});
|
||||
}
|
||||
ActiveScreenShareSource.setLimit(limit);
|
||||
const snapshot = collectScreenShareSendSnapshot(stats);
|
||||
cpuLimitedTicks = snapshot.cpuLimited ? cpuLimitedTicks + 1 : 0;
|
||||
const limit = classifyScreenShareSendLimit(sendSnapshot, snapshot, cpuLimitedTicks);
|
||||
sendSnapshot = snapshot;
|
||||
if (limit !== ActiveScreenShareSource.getLimit()) {
|
||||
logger.info('Screen share send limit changed', {limit, codec: options.codec});
|
||||
}
|
||||
ActiveScreenShareSource.setLimit(limit);
|
||||
if (!verified) {
|
||||
const verification = verifyScreenShareEncoderStart(stats, options.codec, options.onEncodeFailure);
|
||||
if (verification === 'failed') return;
|
||||
@@ -907,8 +891,6 @@ export function startScreenShareEncoderMonitor(options: ScreenShareEncoderMonito
|
||||
cancelled = true;
|
||||
clearTimeout(timer);
|
||||
ActiveScreenShareSource.setEncoding(false);
|
||||
if (delivery) {
|
||||
ActiveScreenShareSource.setLimit(null);
|
||||
}
|
||||
ActiveScreenShareSource.setLimit(null);
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user