refactor(voice): make screen-share source swaps atomic (#2385)

This commit is contained in:
Hampus authored and GitHub committed 2026-09-02 17:33:18 +02:00
1 parent 62577b25bb
commit 8d34c6bcaa
43 files changed
+1168 -240

No files matched your search

@@ -141,6 +141,7 @@ import {VoiceEngineV2AppSourceLifecycleBridge} from '@app/features/voice/engine/
import {VoiceEngineV2AppStatsHostAdapter} from '@app/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter';
import VoiceEngineV2AppSubscriptionAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppSubscriptionAdapter';
import voiceEngineV2AppVoiceStateAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppVoiceStateAdapter';
import type {DisplayScreenShareCaptureContext} from '@app/features/voice/engine/voice_screen_share_manager/shared';
import type {VoiceStateAckPayload} from '@app/features/voice/events/VoiceStateAck';
import CallMediaPrefs from '@app/features/voice/state/CallMediaPrefs';
import {type ChannelE2EEStatus, computeChannelE2EEStatus} from '@app/features/voice/state/ChannelE2EEStatus';
@@ -2238,11 +2239,13 @@ class MediaEngineFacade extends Store {
async replaceActiveDisplayScreenShare(
options?: ScreenShareCaptureOptions,
publishOptions?: TrackPublishOptions,
captureContext?: DisplayScreenShareCaptureContext,
): Promise<boolean> {
return voiceEngineV2AppScreenShareExecutionAdapter.replaceActiveDisplayScreenShare(
this.room,
options,
publishOptions,
captureContext,
);
}
@@ -46,6 +46,7 @@ import type {VoiceEngineV2AppSourceLifecycleBridge} from '@app/features/voice/en
import {
captureScreenSharePublicationCleanup,
type DeviceScreenShareCaptureOptions,
type DisplayScreenShareCaptureContext,
logger,
mergeScreenShareCaptureCleanupSnapshots,
releaseScreenShareCaptureCleanup,
@@ -73,6 +74,8 @@ import {
commitNativeAudioBridgeReplacement,
disarmNativeAudio,
} from '@app/features/voice/utils/NativeAudioCaptureBridge';
import {ScreenShareRollbackIncompleteError} from '@app/features/voice/utils/ScreenShareRollbackIncompleteError';
import {handleScreenShareError} from '@app/features/voice/utils/ScreenShareUtils';
import type {NativeAudioStartOptions} from '@app/types/electron.d';
import type {VoiceEngineV2ScreenOptions} from '@fluxer/voice_engine_v2';
import {msg} from '@lingui/core/macro';
@@ -403,6 +406,10 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
getScreenShareSimulcastCleanupSnapshotInternal(screenShareTrack: LocalVideoTrack): ScreenShareCaptureCleanupSnapshot {
const snapshot: ScreenShareCaptureCleanupSnapshot = {mediaTracks: [], senders: []};
const primarySender = screenShareTrack.sender;
if (primarySender) {
snapshot.senders.push({sender: primarySender, expectedTrack: primarySender.track});
}
const simulcastCodecs = (
screenShareTrack as LocalVideoTrack & {
simulcastCodecs?: Map<unknown, SimulcastTrackInfoLike>;
@@ -411,7 +418,17 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
for (const simulcastTrackInfo of simulcastCodecs?.values() ?? []) {
snapshot.mediaTracks.push(simulcastTrackInfo.mediaStreamTrack);
if (simulcastTrackInfo.sender) {
snapshot.senders.push(simulcastTrackInfo.sender);
const senderTarget = {
sender: simulcastTrackInfo.sender,
expectedTrack: simulcastTrackInfo.sender.track,
};
if (
!snapshot.senders.some(
(target) => target.sender === senderTarget.sender && target.expectedTrack === senderTarget.expectedTrack,
)
) {
snapshot.senders.push(senderTarget);
}
}
}
return snapshot;
@@ -558,14 +575,23 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
snapshot,
captureScreenSharePublicationCleanup(...publications),
);
const cleanupErrors: Array<unknown> = [];
const cleanupResult = await unpublishLocalMediaPublications(participant, publications);
for (const failure of cleanupResult.failedPublications) {
cleanupErrors.push(failure.error);
logger.warn('Failed to unpublish lingering screen share track', {
error: failure.error,
source: failure.publication.source,
});
}
await this.releaseScreenShareCapture(participant, cleanupSnapshot);
try {
await this.releaseScreenShareCapture(participant, cleanupSnapshot);
} catch (error) {
cleanupErrors.push(error);
}
if (cleanupErrors.length > 0) {
throw new ScreenShareRollbackIncompleteError(cleanupErrors);
}
}
handleLocalScreenShareTrackUnpublished(room: Room, playSound: boolean, publication?: LocalTrackPublication): void {
@@ -583,6 +609,7 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
this.applyScreenShareStateInternal(false, {reason: 'user', sendUpdate: true});
}
void this.cleanupLingeringScreenShareTracks(participant, cleanupSnapshot).catch((error) => {
if (error instanceof ScreenShareRollbackIncompleteError) handleScreenShareError(error);
logger.warn('Failed to clean up screen-share audio after video unpublish', {error});
});
if (playSound && !this.isScreenSharePending) {
@@ -746,8 +773,9 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store {
room: Room | null,
options?: ScreenShareCaptureOptions,
publishOptions?: TrackPublishOptions,
captureContext?: DisplayScreenShareCaptureContext,
): Promise<boolean> {
return this.liveKitFlows.replaceActiveDisplayShare(room, options, publishOptions);
return this.liveKitFlows.replaceActiveDisplayShare(room, options, publishOptions, captureContext);
}
async replaceActiveDeviceScreenShare(
@@ -6,7 +6,10 @@ import AdaptiveScreenShareEngine from '@app/features/voice/engine/AdaptiveScreen
import {updateLocalParticipantFromRoom} from '@app/features/voice/engine/VoiceMediaEngineBridge';
import {
enforceLocalMediaPublicationCap,
getLocalPublicationMediaStreamTrack,
getLocalScreenShareAudioPublications,
getLocalScreenSharePublications,
getLocalScreenShareVideoPublications,
} from '@app/features/voice/engine/VoiceTrackPublicationUtils';
import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
import type {
@@ -34,26 +37,34 @@ import {
import {
type CapturedScreenShareTracks,
type DeviceScreenShareCaptureOptions,
type DisplayScreenShareCaptureContext,
getReplacementScreenShareSettingsOptions,
logger,
type ScreenShareCaptureCleanupSnapshot,
type SimulcastTrackInfoLike,
stopMediaTrack,
} from '@app/features/voice/engine/voice_screen_share_manager/shared';
import ActiveScreenShareSource, {
type PublishedScreenShareSource,
} from '@app/features/voice/state/ActiveScreenShareSource';
import type LocalVoiceState from '@app/features/voice/state/LocalVoiceState';
import SoftwareEncoderWarning from '@app/features/voice/state/SoftwareEncoderWarning';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {
prepareHighFidelityScreenShareAudioTrack,
SCREEN_SHARE_AUDIO_PUBLISH_OPTIONS,
} from '@app/features/voice/utils/AudioPublishOptions';
import type {ScreenShareContentSource} from '@app/features/voice/utils/CodecCapabilityDetector';
import {commitNativeAudioBridgeReplacement} from '@app/features/voice/utils/NativeAudioCaptureBridge';
import {ScreenShareAudioCaptureError} from '@app/features/voice/utils/ScreenShareAudioCaptureError';
import {ScreenShareRollbackIncompleteError} from '@app/features/voice/utils/ScreenShareRollbackIncompleteError';
import {applyCameraMirrorProcessor} from '@app/features/voice/utils/VideoBackgroundProcessor';
import {
createLocalAudioTrack,
createLocalVideoTrack,
type LocalAudioTrack,
LocalAudioTrack,
type LocalParticipant,
type LocalTrackPublication,
type LocalVideoTrack,
type Room,
type ScreenShareCaptureOptions,
@@ -76,6 +87,62 @@ function isUserCancelledOrPermissionDeniedError(error: unknown): boolean {
return false;
}
const COMMITTED_PUBLICATION_INVARIANT_ATTEMPTS = 2;
interface ScreenShareReplacementSnapshot {
videoTrack: MediaStreamTrack;
videoHadProcessor: boolean;
audioPublication?: LocalTrackPublication;
audioTrack?: LocalAudioTrack;
audioMediaStreamTrack?: MediaStreamTrack;
audioMuted: boolean;
contentSource: ScreenShareContentSource;
publishedSource: PublishedScreenShareSource | null;
sourceId: string | null;
isOwnWindow: boolean;
publishOptions: TrackPublishOptions;
}
type ScreenShareAudioReplacementStage =
| {kind: 'none'}
| {
kind: 'candidate';
publication: LocalTrackPublication;
track: LocalAudioTrack;
mediaStreamTrack: MediaStreamTrack;
};
interface ScreenShareSimulcastReplacementStageEntry {
info: SimulcastTrackInfoLike;
previousTrack: MediaStreamTrack;
nextTrack: MediaStreamTrack;
sender?: RTCRtpSender;
}
type ScreenShareSimulcastReplacementStage = Array<ScreenShareSimulcastReplacementStageEntry>;
function isCurrentScreenShareAudioReplacementStage(
participant: LocalParticipant,
stage: Extract<ScreenShareAudioReplacementStage, {kind: 'candidate'}>,
): boolean {
const publication = Array.from(participant.audioTrackPublications.values()).find(
(candidate) => candidate === stage.publication,
);
const localTrack = publication?.audioTrack ?? publication?.track;
const mediaStreamTrack = localTrack?.mediaStream?.getAudioTracks()[0] ?? localTrack?.mediaStreamTrack;
return (
publication === stage.publication &&
localTrack === stage.track &&
mediaStreamTrack === stage.mediaStreamTrack &&
stage.mediaStreamTrack.readyState === 'live'
);
}
interface ScreenShareReconciliationStep {
name: string;
run: () => void;
}
export class VoiceEngineV2AppScreenShareLiveKitFlows {
private readonly adapter: VoiceEngineV2AppScreenShareExecutionAdapter;
@@ -506,7 +573,7 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
participant: LocalParticipant,
failureContext: {kind: 'display'} | {kind: 'device'; options?: DeviceScreenShareCaptureOptions},
error: unknown,
): void {
): boolean {
assert.ok(participant);
const cancelled = isUserCancelledOrPermissionDeniedError(error);
if (cancelled) {
@@ -526,12 +593,14 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
this.adapter.transitionScreenShareLifecycleInternal(
buildScreenShareFailureTransition({cancelled, active, sourceType}),
);
return cancelled;
}
async replaceActiveDisplayShare(
room: Room | null,
options?: ScreenShareCaptureOptions,
publishOptions?: TrackPublishOptions,
captureContext?: DisplayScreenShareCaptureContext,
): Promise<boolean> {
const platformVerdict = guardScreenShareEntry({
platformUnsupportedWarning: SCREEN_SHARE_SOURCE_SWITCH_UNSUPPORTED_PLATFORM_WARNING,
@@ -560,13 +629,15 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
codecRepublishInFlight: true,
});
try {
const tracks = await createDisplayScreenShareTracks(options);
const tracks = await createDisplayScreenShareTracks(options, captureContext);
didReplace = await this.replaceActiveTracks(room, participant, tracks, options, publishOptions);
this.emitReplaceShareResult(participant, 'display', didReplace);
} catch (error) {
this.handleReplaceShareFailure(participant, {kind: 'display'}, error);
const cancelled = this.handleReplaceShareFailure(participant, {kind: 'display'}, error);
if (!cancelled) throw error;
} finally {
await this.adapter.applyPendingScreenShareRequestsInternal(room, participant);
}
await this.adapter.applyPendingScreenShareRequestsInternal(room, participant);
return didReplace;
}
@@ -612,49 +683,349 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
return didReplace;
}
private async swapScreenShareVideoTrack(
private captureScreenShareReplacementSnapshot(
participant: LocalParticipant,
screenShareTrack: LocalVideoTrack,
tracks: CapturedScreenShareTracks,
nextContentSource: ScreenShareContentSource,
): Promise<boolean> {
assert.ok(screenShareTrack);
assert.ok(tracks);
const previousVideoMediaTrack = screenShareTrack.mediaStreamTrack;
try {
await screenShareTrack.replaceTrack(tracks.videoTrack, false);
if (nextContentSource === 'device') {
await applyCameraMirrorProcessor(screenShareTrack);
}
await this.refreshSimulcastTracks(screenShareTrack);
} catch (error) {
stopMediaTrack(tracks.videoTrack);
stopMediaTrack(tracks.audioTrack);
logger.error('Failed to replace active screen share video track', {error});
return false;
publishOptions: TrackPublishOptions,
): ScreenShareReplacementSnapshot {
const videoHadProcessor = screenShareTrack.getProcessor() != null;
const sourceVideoTrack =
screenShareTrack.mediaStream?.getVideoTracks()[0] ??
(videoHadProcessor ? undefined : screenShareTrack.mediaStreamTrack);
if (!sourceVideoTrack || sourceVideoTrack.readyState !== 'live') {
throw new Error('Active screen share has no live source video track to preserve');
}
if (previousVideoMediaTrack && previousVideoMediaTrack !== screenShareTrack.mediaStreamTrack) {
this.adapter.cleanupActiveScreenShareEndListenerInternal();
stopMediaTrack(previousVideoMediaTrack);
const audioPublications = getLocalScreenShareAudioPublications(participant);
if (audioPublications.length > 1) {
throw new Error('Active screen share has multiple audio publications before source replacement');
}
return true;
const audioPublication = audioPublications[0];
const localAudioTrack = audioPublication?.audioTrack ?? (audioPublication?.track as LocalAudioTrack | undefined);
const sourceAudioTrack = localAudioTrack?.mediaStream?.getAudioTracks()[0] ?? localAudioTrack?.mediaStreamTrack;
if (audioPublication && (!localAudioTrack || !sourceAudioTrack || sourceAudioTrack.readyState !== 'live')) {
throw new Error('Active screen share audio publication has no live source track to preserve');
}
return {
videoTrack: sourceVideoTrack,
videoHadProcessor,
...(audioPublication ? {audioPublication} : {}),
...(localAudioTrack && sourceAudioTrack
? {audioTrack: localAudioTrack, audioMediaStreamTrack: sourceAudioTrack}
: {}),
audioMuted: audioPublication?.isMuted ?? false,
contentSource: this.adapter.getActiveScreenShareContentSourceInternal(),
publishedSource: ActiveScreenShareSource.getPublishedSource(),
sourceId: ActiveScreenShareSource.getSourceId(),
isOwnWindow: ActiveScreenShareSource.isOwnWindow(),
publishOptions,
};
}
private async swapScreenShareAudioTrack(
private restorePublishedScreenShareSource(snapshot: ScreenShareReplacementSnapshot): void {
if (snapshot.publishedSource === null) {
ActiveScreenShareSource.clear();
return;
}
ActiveScreenShareSource.setPublishedSource(snapshot.publishedSource, snapshot.sourceId, {
isOwnWindow: snapshot.isOwnWindow,
});
}
private async restoreScreenShareReplacement(
room: Room,
participant: LocalParticipant,
screenShareTrack: LocalVideoTrack,
snapshot: ScreenShareReplacementSnapshot,
audioStage: ScreenShareAudioReplacementStage,
): Promise<void> {
this.adapter.cleanupActiveScreenShareEndListenerInternal();
if (snapshot.videoTrack.readyState !== 'live') {
throw new Error('Screen share replacement rollback source ended before it could be restored');
}
if (audioStage.kind === 'candidate') {
const stagedPublication = Array.from(participant.audioTrackPublications.values()).find(
(publication) => publication === audioStage.publication,
);
const stagedTrack = stagedPublication?.audioTrack ?? stagedPublication?.track;
if (stagedPublication === audioStage.publication && stagedTrack === audioStage.track) {
await participant.unpublishTrack(audioStage.track, false).catch((error) => {
logger.warn('Failed to unpublish screen share audio candidate during rollback', {error});
});
}
stopMediaTrack(audioStage.mediaStreamTrack);
}
const activeVideoTrack = screenShareTrack.mediaStream?.getVideoTracks()[0] ?? screenShareTrack.mediaStreamTrack;
const sourceNeedsReplacement = activeVideoTrack !== snapshot.videoTrack;
if (sourceNeedsReplacement) {
await screenShareTrack.stageTrackReplacement(snapshot.videoTrack);
}
if (snapshot.videoHadProcessor) {
const restoredProcessor = await applyCameraMirrorProcessor(screenShareTrack, true);
if (!restoredProcessor) {
throw new Error('Screen share rollback could not restore the previous video processor');
}
} else if (screenShareTrack.getProcessor()) {
await screenShareTrack.stopProcessor(false);
}
if (snapshot.audioPublication && snapshot.audioTrack && snapshot.audioMediaStreamTrack) {
const restoredAudioPublication = Array.from(participant.audioTrackPublications.values()).find(
(publication) => publication === snapshot.audioPublication,
);
const restoredAudioTrack = restoredAudioPublication?.audioTrack ?? restoredAudioPublication?.track;
const restoredAudioMediaStreamTrack =
restoredAudioTrack?.mediaStream?.getAudioTracks()[0] ?? restoredAudioTrack?.mediaStreamTrack;
if (
!restoredAudioPublication ||
restoredAudioTrack !== snapshot.audioTrack ||
restoredAudioMediaStreamTrack !== snapshot.audioMediaStreamTrack ||
restoredAudioMediaStreamTrack.readyState !== 'live'
) {
throw new Error('Screen share audio rollback did not restore its publication');
}
if (snapshot.audioMuted) {
await restoredAudioPublication.mute();
} else {
await restoredAudioPublication.unmute();
}
}
this.restorePublishedScreenShareSource(snapshot);
await runScreenShareActivationRitual({
adapter: this.adapter,
room,
participant,
active: true,
steps: {
acquireStreamingPriority: false,
enforcePublicationCap: true,
applyState: () => applyScreenShareState(this.adapter, true, true, true),
applyStatePosition: 'after-pipeline',
publishPipeline: {
contentSource: snapshot.contentSource,
effectivePublishOptions: snapshot.publishOptions,
},
deactivateCleanup: null,
updateLocalParticipant: true,
audioSync: {kind: 'participant-after-watch'},
syncPersistedAudioPreferenceWhenActive: true,
playSound: false,
buildResolveTransition: null,
},
});
if (snapshot.videoTrack.readyState !== 'live') {
throw new Error('Screen share replacement rollback source ended while runtime state was restored');
}
if (sourceNeedsReplacement) {
await screenShareTrack.commitStagedTrackReplacement(snapshot.videoTrack, false);
}
}
private async failClosedScreenShareReplacement(
participant: LocalParticipant,
snapshot: ScreenShareReplacementSnapshot,
cleanupSnapshot?: ScreenShareCaptureCleanupSnapshot,
): Promise<void> {
let cleanupError: unknown;
try {
await this.adapter.cleanupLingeringScreenShareTracks(participant, cleanupSnapshot);
} catch (error) {
cleanupError = error;
logger.error('Failed to clean up screen share after replacement rollback failed', {error});
}
stopMediaTrack(snapshot.videoTrack);
stopMediaTrack(snapshot.audioMediaStreamTrack);
ActiveScreenShareSource.clear();
applyScreenShareState(this.adapter, false, true, true);
this.adapter.syncLocalStreamWatchStateInternal(false);
this.adapter.syncLocalScreenShareAudioStateInternal(participant, false);
AdaptiveScreenShareEngine.stop();
if (cleanupError !== undefined) {
throw new ScreenShareRollbackIncompleteError([cleanupError]);
}
}
private async stageScreenShareAudioReplacement(
participant: LocalParticipant,
tracks: CapturedScreenShareTracks,
): Promise<void> {
): Promise<ScreenShareAudioReplacementStage> {
assert.ok(participant);
assert.ok(tracks);
let audioTrackAdopted = false;
try {
audioTrackAdopted = await this.adapter.replaceActiveScreenShareAudioTrackInternal(participant, tracks.audioTrack);
if (audioTrackAdopted && tracks.audioTrack) {
commitNativeAudioBridgeReplacement();
}
} catch (error) {
if (!audioTrackAdopted) stopMediaTrack(tracks.audioTrack);
logger.warn('Failed to replace active screen share audio track', {error});
if (!tracks.audioTrack) {
return {kind: 'none'};
}
prepareHighFidelityScreenShareAudioTrack(tracks.audioTrack);
let publication: LocalTrackPublication | null = null;
const candidateTrack = new LocalAudioTrack(tracks.audioTrack);
try {
await candidateTrack.mute();
publication = await participant.publishTrack(candidateTrack, SCREEN_SHARE_AUDIO_PUBLISH_OPTIONS);
const track = publication.audioTrack ?? (publication.track as LocalAudioTrack | undefined);
const mediaStreamTrack = track?.mediaStream?.getAudioTracks()[0] ?? track?.mediaStreamTrack;
if (!track || mediaStreamTrack !== tracks.audioTrack || mediaStreamTrack.readyState !== 'live') {
throw new Error('Replacement screen share audio publication has no live local track');
}
if (!publication.isMuted) {
await publication.mute();
}
const stage: Extract<ScreenShareAudioReplacementStage, {kind: 'candidate'}> = {
kind: 'candidate',
publication,
track,
mediaStreamTrack,
};
if (!isCurrentScreenShareAudioReplacementStage(participant, stage)) {
throw new Error('Replacement screen share audio publication changed during staging');
}
return stage;
} catch (error) {
let cleanupError: unknown;
const track = publication?.audioTrack ?? (publication?.track as LocalAudioTrack | undefined);
const currentPublication = Array.from(participant.audioTrackPublications.values()).find(
(candidate) => candidate === publication,
);
if (
publication &&
track &&
currentPublication === publication &&
(publication.audioTrack ?? publication.track) === track
) {
try {
await participant.unpublishTrack(track, false);
} catch (caughtCleanupError) {
cleanupError = caughtCleanupError;
}
}
stopMediaTrack(tracks.audioTrack);
if (cleanupError !== undefined) {
throw new ScreenShareRollbackIncompleteError([error, cleanupError]);
}
throw error;
}
}
private async commitScreenShareAudioReplacement(
participant: LocalParticipant,
videoPublication: LocalTrackPublication,
videoTrack: LocalVideoTrack,
videoMediaStreamTrack: MediaStreamTrack,
snapshot: ScreenShareReplacementSnapshot,
audioStage: ScreenShareAudioReplacementStage,
): Promise<void> {
const cleanupErrors: Array<unknown> = [];
if (audioStage.kind === 'candidate' && !isCurrentScreenShareAudioReplacementStage(participant, audioStage)) {
throw new Error('Replacement screen share audio source changed before replacement commit');
}
const previousAudioTrack = snapshot.audioTrack;
if (previousAudioTrack) {
const previousAudioPublicationPresent = Array.from(participant.audioTrackPublications.values()).some(
(publication) => publication === snapshot.audioPublication,
);
const previousAudioMediaStreamTrack =
previousAudioTrack.mediaStream?.getAudioTracks()[0] ?? previousAudioTrack.mediaStreamTrack;
if (
!previousAudioPublicationPresent ||
previousAudioMediaStreamTrack !== snapshot.audioMediaStreamTrack ||
previousAudioMediaStreamTrack.readyState !== 'live'
) {
throw new Error('Previous screen share audio source changed before replacement commit');
}
try {
await previousAudioTrack.mute();
} catch (error) {
cleanupErrors.push(error);
stopMediaTrack(snapshot.audioMediaStreamTrack);
}
try {
const currentPreviousAudioPublication = Array.from(participant.audioTrackPublications.values()).find(
(publication) => publication === snapshot.audioPublication,
);
const currentPreviousAudioTrack =
currentPreviousAudioPublication?.audioTrack ?? currentPreviousAudioPublication?.track;
const currentPreviousAudioMediaStreamTrack =
currentPreviousAudioTrack?.mediaStream?.getAudioTracks()[0] ?? currentPreviousAudioTrack?.mediaStreamTrack;
if (
currentPreviousAudioPublication !== snapshot.audioPublication ||
currentPreviousAudioTrack !== previousAudioTrack ||
currentPreviousAudioMediaStreamTrack !== snapshot.audioMediaStreamTrack
) {
throw new Error('Previous screen share audio source changed while replacement commit was in progress');
}
if (audioStage.kind === 'candidate') {
await participant.unpublishTrack(previousAudioTrack, false);
stopMediaTrack(snapshot.audioMediaStreamTrack);
} else {
await this.adapter.replaceActiveScreenShareAudioTrackInternal(participant, undefined);
}
} catch (error) {
cleanupErrors.push(error);
stopMediaTrack(snapshot.audioMediaStreamTrack);
}
}
if (audioStage.kind === 'candidate') {
try {
await audioStage.publication.unmute();
} catch (error) {
cleanupErrors.push(error);
stopMediaTrack(audioStage.mediaStreamTrack);
}
}
const hasCommittedPublicationInvariant = (): boolean => {
const videoPublications = getLocalScreenShareVideoPublications(participant);
const currentVideoTrack = videoPublication.videoTrack ?? videoPublication.track;
const currentVideoMediaStreamTrack =
currentVideoTrack?.mediaStream?.getVideoTracks()[0] ?? currentVideoTrack?.mediaStreamTrack;
if (
videoPublications.length !== 1 ||
videoPublications[0] !== videoPublication ||
currentVideoTrack !== videoTrack ||
currentVideoMediaStreamTrack !== videoMediaStreamTrack ||
videoMediaStreamTrack.readyState !== 'live'
) {
return false;
}
const audioPublications = getLocalScreenShareAudioPublications(participant);
if (audioStage.kind === 'none') {
return audioPublications.length === 0;
}
if (audioPublications.length !== 1 || audioPublications[0] !== audioStage.publication) {
return false;
}
const publishedAudioTrack = audioStage.publication.audioTrack ?? audioStage.publication.track;
return (
publishedAudioTrack === audioStage.track && isCurrentScreenShareAudioReplacementStage(participant, audioStage)
);
};
let publicationInvariantSatisfied = false;
for (let attempt = 0; attempt < COMMITTED_PUBLICATION_INVARIANT_ATTEMPTS; attempt++) {
const capResult = await enforceLocalMediaPublicationCap(participant, VoiceTrackSource.ScreenShare, {
preferredPublication: audioStage.kind === 'candidate' ? audioStage.publication : undefined,
stopOnUnpublish: true,
});
for (const failure of capResult.failedPublications) {
cleanupErrors.push(failure.error);
stopMediaTrack(getLocalPublicationMediaStreamTrack(failure.publication) ?? undefined);
}
publicationInvariantSatisfied = hasCommittedPublicationInvariant();
if (publicationInvariantSatisfied) break;
}
if (!publicationInvariantSatisfied) {
throw new AggregateError(
[...cleanupErrors, new Error('Committed screen share publications do not match the replacement')],
'Failed to establish committed screen share publication state',
);
}
if (cleanupErrors.length > 0) {
logger.warn('Recovered from screen share publication cleanup failures after source replacement', {
errors: cleanupErrors,
});
}
if (audioStage.kind === 'candidate') {
commitNativeAudioBridgeReplacement();
}
if (!hasCommittedPublicationInvariant()) {
throw new Error('Committed screen share publication ownership changed during native audio finalization');
}
this.adapter.syncLocalScreenShareAudioStateInternal(participant, true);
this.adapter.syncPersistedScreenShareAudioPreferenceInternal(participant);
}
private async finalizeReplaceActiveTracks(
@@ -663,15 +1034,14 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
tracks: CapturedScreenShareTracks,
nextContentSource: ScreenShareContentSource,
options: ScreenShareCaptureOptions | undefined,
publishOptions: TrackPublishOptions | undefined,
effectivePublishOptions: TrackPublishOptions | undefined,
): Promise<void> {
assert.ok(participant);
assert.ok(tracks);
await enforceLocalMediaPublicationCap(participant, VoiceTrackSource.ScreenShare);
const effectivePublishOptions = await this.adapter.getEffectivePublishOptionsInternal(true, publishOptions);
const replacementSettings = getReplacementScreenShareSettingsOptions(options, tracks.audioTrack != null);
await this.adapter.updateActiveScreenShareSettings(
room,
getReplacementScreenShareSettingsOptions(options, tracks.audioTrack != null),
replacementSettings ? {...replacementSettings, audio: undefined} : undefined,
effectivePublishOptions,
);
await runScreenShareActivationRitual({
@@ -693,7 +1063,48 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
buildResolveTransition: null,
},
});
logger.info('Replaced active screen share source', {audioIncluded: tracks.audioTrack != null});
}
private reconcileCommittedScreenShareReplacement(
room: Room,
participant: LocalParticipant,
nextContentSource: ScreenShareContentSource,
effectivePublishOptions: TrackPublishOptions | undefined,
): void {
const steps: Array<ScreenShareReconciliationStep> = [
{
name: 'content hint',
run: () => this.adapter.applyScreenShareContentHintInternal(participant, nextContentSource),
},
{name: 'keep-alive sink', run: () => this.adapter.ensureScreenShareKeepAliveSinkInternal(participant)},
{name: 'audio content hint', run: () => this.adapter.applyScreenShareAudioContentHintInternal(participant)},
{name: 'end monitor', run: () => this.adapter.monitorActiveScreenShareEndInternal(room, participant)},
{
name: 'encoder verification',
run: () =>
this.adapter.startEncoderVerificationInternal(room, participant, effectivePublishOptions?.videoCodec),
},
{name: 'adaptive engine', run: () => AdaptiveScreenShareEngine.start(room)},
{name: 'local stream state', run: () => applyScreenShareState(this.adapter, true, true, true)},
{name: 'participant snapshot', run: () => updateLocalParticipantFromRoom(room)},
{name: 'watch state', run: () => this.adapter.syncLocalStreamWatchStateInternal(true)},
{name: 'audio state', run: () => this.adapter.syncLocalScreenShareAudioStateInternal(participant, true)},
{
name: 'persisted audio preference',
run: () => this.adapter.syncPersistedScreenShareAudioPreferenceInternal(participant),
},
];
for (const step of steps) {
try {
step.run();
} catch (error) {
logger.error('Failed to reconcile committed screen share runtime state', {
error,
nextContentSource,
step: step.name,
});
}
}
}
private async replaceActiveTracks(
@@ -712,47 +1123,219 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
logger.warn('No active screen share video track to replace');
return false;
}
const nextContentSource = contentSource ?? this.adapter.getActiveScreenShareContentSourceInternal();
if (nextContentSource !== 'device' && screenShareTrack.getProcessor()) {
await screenShareTrack.stopProcessor(false);
const nextContentSource = contentSource ?? 'display';
const previousPublishOptions = {
...(((screenSharePublication as {options?: TrackPublishOptions}).options ?? {}) as TrackPublishOptions),
};
let snapshot: ScreenShareReplacementSnapshot | null = null;
let effectivePublishOptions: TrackPublishOptions | undefined;
try {
snapshot = this.captureScreenShareReplacementSnapshot(participant, screenShareTrack, previousPublishOptions);
effectivePublishOptions = await this.adapter.getEffectivePublishOptionsInternal(true, publishOptions);
await enforceLocalMediaPublicationCap(participant, VoiceTrackSource.ScreenShare);
} catch (error) {
stopMediaTrack(tracks.videoTrack);
stopMediaTrack(tracks.audioTrack);
throw error;
}
const videoSwapped = await this.swapScreenShareVideoTrack(screenShareTrack, tracks, nextContentSource);
if (!videoSwapped) return false;
await this.swapScreenShareAudioTrack(participant, tracks);
await this.finalizeReplaceActiveTracks(room, participant, tracks, nextContentSource, options, publishOptions);
assert.ok(snapshot, 'screen share replacement snapshot must exist after preflight');
if (tracks.videoTrack === snapshot.videoTrack) {
stopMediaTrack(tracks.audioTrack);
throw new Error('Screen share replacement returned the active source track as its candidate');
}
let audioStage: ScreenShareAudioReplacementStage = {kind: 'none'};
let simulcastStage: ScreenShareSimulcastReplacementStage = [];
let requestedAudioStageInProgress = false;
this.adapter.cleanupActiveScreenShareEndListenerInternal();
try {
await screenShareTrack.stageTrackReplacement(tracks.videoTrack);
if (nextContentSource === 'device') {
const mirrorCamera = VoiceSettings.getMirrorCamera();
const processor = await applyCameraMirrorProcessor(screenShareTrack, mirrorCamera);
if (mirrorCamera && !processor) {
throw new Error('Replacement device share could not apply its required mirror processor');
}
if (!mirrorCamera && screenShareTrack.getProcessor()) {
throw new Error('Replacement device share could not clear its previous video processor');
}
} else if (screenShareTrack.getProcessor()) {
await screenShareTrack.stopProcessor(false);
}
const activeVideoTrack = screenShareTrack.mediaStream?.getVideoTracks()[0] ?? screenShareTrack.mediaStreamTrack;
if (activeVideoTrack !== tracks.videoTrack) {
throw new Error('Replacement screen share video track did not take ownership of its publication');
}
if (activeVideoTrack.readyState !== 'live') {
throw new Error('Replacement screen share video track ended before commit');
}
requestedAudioStageInProgress = tracks.displayCapture?.requireAudio === true;
audioStage = await this.stageScreenShareAudioReplacement(participant, tracks);
requestedAudioStageInProgress = false;
if (audioStage.kind === 'candidate' && !isCurrentScreenShareAudioReplacementStage(participant, audioStage)) {
throw new Error('Replacement screen share audio track ended before commit');
}
simulcastStage = await this.stageScreenShareSimulcastReplacement(screenShareTrack, tracks.videoTrack);
await this.finalizeReplaceActiveTracks(
room,
participant,
tracks,
nextContentSource,
options,
effectivePublishOptions,
);
await screenShareTrack.commitStagedTrackReplacement(tracks.videoTrack, false);
} catch (error) {
try {
await this.rollbackScreenShareSimulcastReplacement(simulcastStage);
await this.restoreScreenShareReplacement(room, participant, screenShareTrack, snapshot, audioStage);
} catch (rollbackError) {
try {
await this.failClosedScreenShareReplacement(participant, snapshot);
} catch (cleanupError) {
stopMediaTrack(tracks.videoTrack);
stopMediaTrack(tracks.audioTrack);
throw new ScreenShareRollbackIncompleteError([error, rollbackError, cleanupError]);
}
stopMediaTrack(tracks.videoTrack);
stopMediaTrack(tracks.audioTrack);
throw new AggregateError(
[error, rollbackError],
'Screen share replacement and rollback both failed; the share was stopped',
);
}
stopMediaTrack(tracks.videoTrack);
stopMediaTrack(tracks.audioTrack);
if (
requestedAudioStageInProgress &&
!(error instanceof ScreenShareAudioCaptureError) &&
!(error instanceof ScreenShareRollbackIncompleteError)
) {
throw new ScreenShareAudioCaptureError({
sourceId: tracks.displayCapture?.sourceId,
sourceKind: tracks.displayCapture?.displayShareEnvironment,
reason: 'requested-audio-publication-failed',
detail: error instanceof Error ? `${error.name}: ${error.message}` : 'Unknown audio replacement failure',
});
}
throw error;
}
const committedCleanupSnapshot = this.adapter.getScreenShareCaptureCleanupSnapshotInternal(participant);
let requestedAudioCommitInProgress = false;
try {
requestedAudioCommitInProgress = tracks.displayCapture?.requireAudio === true;
await this.commitScreenShareAudioReplacement(
participant,
screenSharePublication,
screenShareTrack,
tracks.videoTrack,
snapshot,
audioStage,
);
requestedAudioCommitInProgress = false;
this.commitScreenShareSimulcastReplacement(simulcastStage);
} catch (error) {
try {
await this.failClosedScreenShareReplacement(participant, snapshot, committedCleanupSnapshot);
} catch (cleanupError) {
for (const entry of simulcastStage) stopMediaTrack(entry.nextTrack);
stopMediaTrack(tracks.videoTrack);
stopMediaTrack(tracks.audioTrack);
throw new ScreenShareRollbackIncompleteError([error, cleanupError]);
}
for (const entry of simulcastStage) stopMediaTrack(entry.nextTrack);
stopMediaTrack(tracks.videoTrack);
stopMediaTrack(tracks.audioTrack);
if (
requestedAudioCommitInProgress &&
!(error instanceof ScreenShareAudioCaptureError) &&
!(error instanceof ScreenShareRollbackIncompleteError)
) {
throw new ScreenShareAudioCaptureError({
sourceId: tracks.displayCapture?.sourceId,
sourceKind: tracks.displayCapture?.displayShareEnvironment,
reason: 'requested-audio-publication-failed',
detail: error instanceof Error ? `${error.name}: ${error.message}` : 'Unknown audio replacement failure',
});
}
if (error instanceof ScreenShareRollbackIncompleteError) throw error;
throw new AggregateError([error], 'Committed screen share audio replacement failed; the share was stopped');
}
stopMediaTrack(snapshot.videoTrack);
this.reconcileCommittedScreenShareReplacement(room, participant, nextContentSource, effectivePublishOptions);
logger.info('Replaced active screen share source', {audioIncluded: tracks.audioTrack != null});
return true;
}
private async refreshSimulcastTracks(screenShareTrack: LocalVideoTrack): Promise<void> {
private async stageScreenShareSimulcastReplacement(
screenShareTrack: LocalVideoTrack,
candidateTrack: MediaStreamTrack,
): Promise<ScreenShareSimulcastReplacementStage> {
const simulcastCodecs = (
screenShareTrack as LocalVideoTrack & {
simulcastCodecs?: Map<unknown, SimulcastTrackInfoLike>;
}
).simulcastCodecs;
if (!simulcastCodecs?.size) {
return;
return [];
}
const stage: ScreenShareSimulcastReplacementStage = [];
for (const simulcastTrackInfo of simulcastCodecs.values()) {
const previousTrack = simulcastTrackInfo.mediaStreamTrack;
let nextTrack: MediaStreamTrack | undefined;
const sender = simulcastTrackInfo.sender;
const nextTrack = candidateTrack.clone();
const entry = {info: simulcastTrackInfo, previousTrack, nextTrack, sender};
stage.push(entry);
try {
nextTrack = screenShareTrack.mediaStreamTrack.clone();
await simulcastTrackInfo.sender?.replaceTrack(nextTrack);
simulcastTrackInfo.mediaStreamTrack = nextTrack;
} catch (error) {
stopMediaTrack(nextTrack);
try {
await simulcastTrackInfo.sender?.replaceTrack(null);
} catch (replaceError) {
logger.warn('Failed to stop stale screen share simulcast track after replacement failed', {
error: replaceError,
});
if (sender && sender.track !== previousTrack) {
throw new Error('Screen share simulcast sender does not own its recorded source track');
}
logger.warn('Failed to replace active screen share simulcast track', {error});
} finally {
stopMediaTrack(previousTrack);
await sender?.replaceTrack(nextTrack);
} catch (error) {
try {
await this.rollbackScreenShareSimulcastReplacement(stage);
} catch (rollbackError) {
throw new AggregateError(
[error, rollbackError],
'Screen share simulcast replacement and rollback both failed',
);
}
throw error;
}
}
return stage;
}
private async rollbackScreenShareSimulcastReplacement(stage: ScreenShareSimulcastReplacementStage): Promise<void> {
const rollbackErrors: Array<unknown> = [];
for (let index = stage.length - 1; index >= 0; index -= 1) {
const entry = stage[index];
if (!entry) continue;
try {
if (entry.sender?.track === entry.nextTrack) {
await entry.sender.replaceTrack(entry.previousTrack);
}
} catch (error) {
rollbackErrors.push(error);
}
stopMediaTrack(entry.nextTrack);
}
stage.length = 0;
if (rollbackErrors.length > 0) {
throw new AggregateError(rollbackErrors, 'Screen share simulcast rollback was incomplete');
}
}
private commitScreenShareSimulcastReplacement(stage: ScreenShareSimulcastReplacementStage): void {
for (const entry of stage) {
if (entry.sender && entry.sender.track !== entry.nextTrack) {
throw new Error('Screen share simulcast sender changed before replacement commit');
}
}
for (const entry of stage) {
entry.info.mediaStreamTrack = entry.nextTrack;
stopMediaTrack(entry.previousTrack);
}
stage.length = 0;
}
private finalizeReconnectAlreadyEnabled(room: Room | null, participant: LocalParticipant): boolean {
@@ -2,9 +2,11 @@
import {
type CapturedScreenShareTracks,
type DisplayScreenShareCaptureContext,
stopMediaTrack,
stopUnselectedStreamTracks,
} from '@app/features/voice/engine/voice_screen_share_manager/shared';
import {ScreenShareAudioCaptureError} from '@app/features/voice/utils/ScreenShareAudioCaptureError';
import type {ScreenShareCaptureOptions} from 'livekit-client';
type DisplayMediaVideoConstraints = MediaTrackConstraints & {
@@ -92,28 +94,42 @@ export function getDisplayMediaOptions(options?: ScreenShareCaptureOptions): Dis
export async function createDisplayScreenShareTracks(
options?: ScreenShareCaptureOptions,
captureContext?: DisplayScreenShareCaptureContext,
): Promise<CapturedScreenShareTracks> {
if (!navigator.mediaDevices.getDisplayMedia) {
throw new Error('getDisplayMedia not supported');
}
const stream = await navigator.mediaDevices.getDisplayMedia(getDisplayMediaOptions(options));
const videoTrack = stream.getVideoTracks()[0];
if (!videoTrack) {
try {
const videoTrack = stream.getVideoTracks()[0];
if (!videoTrack) {
throw new Error('No video track found in screen share capture');
}
if (options?.contentHint) {
videoTrack.contentHint = options.contentHint;
}
await videoTrack.applyConstraints({colorSpace: 'rec709'} as MediaTrackConstraints).catch(() => undefined);
const cursor = resolveCapturedDisplayMediaCursorCapture(videoTrack, options);
if ((videoTrack.getSettings() as DisplayMediaTrackSettings).cursor !== cursor) {
await videoTrack.applyConstraints({cursor} as MediaTrackConstraints).catch(() => undefined);
}
const capturedAudioTrack = stream.getAudioTracks()[0];
const audioTrack = capturedAudioTrack?.readyState === 'live' ? capturedAudioTrack : undefined;
if (captureContext?.requireAudio && !audioTrack) {
throw new ScreenShareAudioCaptureError({
sourceId: captureContext.sourceId,
reason: 'required-audio-track-missing',
detail: 'display capture completed without the requested native audio track',
});
}
stopUnselectedStreamTracks(stream, [videoTrack, audioTrack]);
return {
videoTrack,
audioTrack,
displayCapture: captureContext,
};
} catch (error) {
stream.getTracks().forEach(stopMediaTrack);
throw new Error('No video track found in screen share capture');
throw error;
}
if (options?.contentHint) {
videoTrack.contentHint = options.contentHint;
}
await videoTrack.applyConstraints({colorSpace: 'rec709'} as MediaTrackConstraints).catch(() => undefined);
const cursor = resolveCapturedDisplayMediaCursorCapture(videoTrack, options);
if ((videoTrack.getSettings() as DisplayMediaTrackSettings).cursor !== cursor) {
await videoTrack.applyConstraints({cursor} as MediaTrackConstraints).catch(() => undefined);
}
const audioTrack = stream.getAudioTracks()[0];
stopUnselectedStreamTracks(stream, [videoTrack, audioTrack]);
return {
videoTrack,
audioTrack,
};
}
@@ -17,6 +17,7 @@ import {
SCREEN_SHARE_DEGRADATION_PREFERENCE,
SCREEN_SHARE_FORCED_VIDEO_BITRATE_BPS,
} from '@app/features/voice/utils/ScreenShareOptions';
import {ScreenShareRollbackIncompleteError} from '@app/features/voice/utils/ScreenShareRollbackIncompleteError';
import {classifyVideoEncoderAcceleration} from '@app/features/voice/utils/VideoAccelerationClassification';
import {
BackupCodecPolicy,
@@ -42,6 +43,13 @@ export interface DeviceScreenShareCaptureOptions {
export interface CapturedScreenShareTracks {
videoTrack: MediaStreamTrack;
audioTrack?: MediaStreamTrack;
displayCapture?: DisplayScreenShareCaptureContext;
}
export interface DisplayScreenShareCaptureContext {
sourceId: string | null;
displayShareEnvironment: string | null;
requireAudio: boolean;
}
export interface SimulcastTrackInfoLike {
@@ -60,12 +68,18 @@ export interface ScreenSharePublishOptionsResolutionOptions {
onCodecReadiness?: (status: ScreenShareCodecReadinessStatus) => void;
}
export interface ScreenShareSenderCleanupTarget {
sender: RTCRtpSender;
expectedTrack: MediaStreamTrack | null;
}
export interface ScreenShareCaptureCleanupSnapshot {
mediaTracks: Array<MediaStreamTrack>;
senders: Array<RTCRtpSender>;
senders: Array<ScreenShareSenderCleanupTarget>;
}
interface ScreenShareTrackLike {
mediaStream?: MediaStream;
mediaStreamTrack?: MediaStreamTrack;
sender?: RTCRtpSender;
simulcastCodecs?: Map<unknown, SimulcastTrackInfoLike>;
@@ -80,15 +94,25 @@ function pushUnique<T>(items: Array<T>, item: T | undefined | null): void {
items.push(item);
}
function pushUniqueSender(snapshot: ScreenShareCaptureCleanupSnapshot, sender: RTCRtpSender | undefined): void {
if (!sender) return;
const expectedTrack = sender.track;
if (snapshot.senders.some((target) => target.sender === sender && target.expectedTrack === expectedTrack)) return;
snapshot.senders.push({sender, expectedTrack});
}
function captureScreenShareTrackCleanup(
snapshot: ScreenShareCaptureCleanupSnapshot,
track: ScreenShareTrackLike | undefined | null,
): void {
for (const mediaStreamTrack of track?.mediaStream?.getTracks() ?? []) {
pushUnique(snapshot.mediaTracks, mediaStreamTrack);
}
pushUnique(snapshot.mediaTracks, track?.mediaStreamTrack);
pushUnique(snapshot.senders, track?.sender);
pushUniqueSender(snapshot, track?.sender);
for (const simulcastTrackInfo of track?.simulcastCodecs?.values() ?? []) {
pushUnique(snapshot.mediaTracks, simulcastTrackInfo.mediaStreamTrack);
pushUnique(snapshot.senders, simulcastTrackInfo.sender);
pushUniqueSender(snapshot, simulcastTrackInfo.sender);
}
}
@@ -112,26 +136,43 @@ export function mergeScreenShareCaptureCleanupSnapshots(
for (const mediaTrack of snapshot?.mediaTracks ?? []) {
pushUnique(merged.mediaTracks, mediaTrack);
}
for (const sender of snapshot?.senders ?? []) {
pushUnique(merged.senders, sender);
for (const senderTarget of snapshot?.senders ?? []) {
if (
!merged.senders.some(
(target) => target.sender === senderTarget.sender && target.expectedTrack === senderTarget.expectedTrack,
)
) {
merged.senders.push(senderTarget);
}
}
}
return merged;
}
async function detachScreenShareSender(sender: RTCRtpSender): Promise<void> {
try {
if (sender.transport?.state === 'closed') return;
await sender.replaceTrack(null);
} catch (error) {
logger.warn('Failed to detach screen share sender during cleanup', {error});
}
async function detachScreenShareSender(target: ScreenShareSenderCleanupTarget): Promise<void> {
if (target.sender.track !== target.expectedTrack) return;
if (target.sender.transport?.state === 'closed') return;
await target.sender.replaceTrack(null);
}
export async function releaseScreenShareCaptureCleanup(snapshot: ScreenShareCaptureCleanupSnapshot): Promise<void> {
await Promise.all(snapshot.senders.map(detachScreenShareSender));
const cleanupErrors: Array<unknown> = [];
const senderResults = await Promise.allSettled(snapshot.senders.map(detachScreenShareSender));
for (const result of senderResults) {
if (result.status === 'rejected') cleanupErrors.push(result.reason);
}
for (const mediaTrack of snapshot.mediaTracks) {
stopMediaTrack(mediaTrack);
try {
mediaTrack.stop();
} catch (error) {
cleanupErrors.push(error);
}
if (mediaTrack.readyState === 'live') {
cleanupErrors.push(new Error('Screen share capture track remained live after cleanup'));
}
}
if (cleanupErrors.length > 0) {
throw new ScreenShareRollbackIncompleteError(cleanupErrors);
}
}
@@ -9,16 +9,19 @@ describe('ActiveScreenShareSource', () => {
});
it('tracks and clears whether the selected source is a Fluxer-owned window', () => {
ActiveScreenShareSource.setSourceId('window:42:0', {isOwnWindow: true});
ActiveScreenShareSource.setPublishedSource('app', 'window:42:0', {isOwnWindow: true});
expect(ActiveScreenShareSource.getSourceId()).toBe('window:42:0');
expect(ActiveScreenShareSource.isOwnWindow()).toBe(true);
expect(ActiveScreenShareSource.getShareContext()).toBe('app');
ActiveScreenShareSource.setSourceId('screen:1:0');
ActiveScreenShareSource.setPublishedSource('display', 'screen:1:0');
expect(ActiveScreenShareSource.getSourceId()).toBe('screen:1:0');
expect(ActiveScreenShareSource.isOwnWindow()).toBe(false);
expect(ActiveScreenShareSource.getShareContext()).toBe('display');
ActiveScreenShareSource.clear();
expect(ActiveScreenShareSource.getSourceId()).toBeNull();
expect(ActiveScreenShareSource.isOwnWindow()).toBe(false);
expect(ActiveScreenShareSource.getShareContext()).toBeNull();
});
});
@@ -1,20 +1,29 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {StreamSettingsShareContext} from '@app/features/voice/utils/StreamSettingsUpdatePolicy';
import {makeAutoObservable} from 'mobx';
export interface ActiveScreenShareSourceOptions {
readonly isOwnWindow?: boolean;
}
export type PublishedScreenShareSource = 'web' | 'wayland' | 'device' | 'app' | 'display';
class ActiveScreenShareSource {
sourceId: string | null = null;
ownWindow = false;
publishedSource: PublishedScreenShareSource | null = null;
constructor() {
makeAutoObservable(this, {}, {autoBind: true});
}
setSourceId(sourceId: string | null, options: ActiveScreenShareSourceOptions = {}): void {
setPublishedSource(
publishedSource: PublishedScreenShareSource,
sourceId: string | null,
options: ActiveScreenShareSourceOptions = {},
): void {
this.publishedSource = publishedSource;
this.sourceId = sourceId;
this.ownWindow = sourceId !== null && options.isOwnWindow === true;
}
@@ -27,9 +36,21 @@ class ActiveScreenShareSource {
return this.ownWindow;
}
getPublishedSource(): PublishedScreenShareSource | null {
return this.publishedSource;
}
getShareContext(): StreamSettingsShareContext | null {
if (this.publishedSource === 'app') return 'app';
if (this.publishedSource === 'device') return 'device';
if (this.publishedSource === null) return null;
return 'display';
}
clear(): void {
this.sourceId = null;
this.ownWindow = false;
this.publishedSource = null;
}
}
@@ -0,0 +1,155 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {beforeEach, describe, expect, test, vi} from 'vitest';
const platform = {current: 'darwin' as NodeJS.Platform | string};
const setScreenShareEnabled = vi.fn(async () => {});
const replaceActiveDisplayScreenShare = vi.fn(async () => true);
const armNativeAudioForNextCapture = vi.fn(async () => true);
const armNativeSystemAudioForNextCapture = vi.fn(async () => true);
const armNativeAudioForLinuxRouting = vi.fn(async () => true);
vi.mock('@app/features/ui/utils/NativeUtils', () => ({
getElectronAPI: () => ({platform: platform.current}),
supportsDesktopScreenShareAudioCapture: () => true,
}));
vi.mock('@app/features/voice/engine/MediaEngineFacade', () => ({
default: {
room: {localParticipant: {isScreenShareEnabled: true}},
setScreenShareEnabled,
replaceActiveDisplayScreenShare,
},
}));
vi.mock('@app/features/voice/utils/ScreenShareEnvironment', () => ({
getDisplayShareEnvironment: async () => 'desktop-custom',
usesNativeDisplayShareAudioSelection: (environment: string) => environment !== 'desktop-custom',
}));
vi.mock('@app/features/voice/utils/NativeAudioCaptureBridge', () => ({
armNativeAudioForLinuxRouting,
armNativeAudioForNextCapture,
armNativeSystemAudioForNextCapture,
disarmNativeAudio: vi.fn(),
disarmPendingNativeAudio: vi.fn(),
getLastNativeAudioArmFailure: () => null,
}));
vi.mock('@app/features/voice/utils/LinuxScreenShareAudio', () => ({
disarmVirtmic: vi.fn(),
}));
vi.mock('@app/features/voice/state/ActiveScreenShareSource', () => ({
default: {setPublishedSource: vi.fn(), clear: vi.fn(), getSourceId: () => null},
}));
vi.mock('@app/features/voice/state/DesktopSourceIntent', () => ({
setDesktopSourceIntent: vi.fn(),
clearDesktopSourceIntent: vi.fn(),
}));
vi.mock('@app/features/voice/state/LocalVoiceState', () => ({
default: {getSelfStream: () => null},
}));
vi.mock('@app/features/voice/utils/ScreenShareUtils', () => ({
executeScreenShareOperation: async (operation: () => Promise<void>) => {
await operation();
},
}));
vi.mock('@app/features/voice/engine/ScreenShareCodecNegotiation', () => ({
default: {selectScreenShareCodec: () => 'vp8'},
}));
vi.mock('@app/features/voice/utils/CodecCapabilityDetector', () => ({
resolveScreenShareContentHintForContext: () => 'motion',
}));
vi.mock('@app/features/app/utils/LimitResolverAdapter', () => ({
LimitResolver: {resolve: () => 0},
}));
vi.mock('@app/features/app/utils/LimitUtils', () => ({
isLimitToggleEnabled: () => false,
}));
vi.mock('@app/features/voice/commands/VoiceSettingsCommands', () => ({
update: vi.fn(),
}));
vi.mock('@app/features/voice/state/VoiceSettings', () => ({
default: {
getShareAppAudio: () => true,
getShareDesktopAudio: () => true,
getShareDeviceAudio: () => true,
getScreenshareResolution: () => 'medium',
getStreamingMode: () => 'screenshare',
getVideoFrameRate: () => 30,
getPreferredScreenShareCodec: () => 'auto',
getScreenShareContentHintOverride: () => 'auto',
getScreenShareMaxBitrateBpsOverride: () => null,
getEffectiveScreenShareAudioDeviceId: () => '',
getScreenShareAudioSourceMode: () => 'system',
getScreenShareAudioIncludeSources: () => [],
getScreenShareAudioExcludeSources: () => [],
},
}));
const {startConfiguredDisplayScreenShare, switchConfiguredDisplayScreenShare} = await import(
'@app/features/voice/utils/ScreenShareStartFlow'
);
const ActiveScreenShareSource = (await import('@app/features/voice/state/ActiveScreenShareSource')).default;
beforeEach(() => {
platform.current = 'darwin';
vi.clearAllMocks();
setScreenShareEnabled.mockResolvedValue(undefined);
armNativeAudioForNextCapture.mockResolvedValue(true);
});
describe('sharing a Fluxer-owned window while app audio is enabled', () => {
test('starts the share video-only instead of failing', async () => {
const started = await startConfiguredDisplayScreenShare('window:99:0', {
isOwnWindow: true,
preferredDisplaySurface: 'window',
});
expect(started).toBe(true);
expect(setScreenShareEnabled).toHaveBeenCalledTimes(1);
const call = setScreenShareEnabled.mock.calls[0] as unknown as [
boolean,
{audio: boolean; systemAudio: string; windowAudio: string},
unknown,
];
expect(call[0]).toBe(true);
expect(call[1].audio).toBe(false);
expect(call[1].systemAudio).toBe('exclude');
expect(call[1].windowAudio).toBe('exclude');
expect(armNativeAudioForNextCapture).not.toHaveBeenCalled();
});
});
describe('switching the display source', () => {
test('preserves the running share state when the switch fails', async () => {
replaceActiveDisplayScreenShare.mockResolvedValue(false);
const switched = await switchConfiguredDisplayScreenShare('window:7:0', {preferredDisplaySurface: 'window'});
expect(switched).toBe(false);
expect(ActiveScreenShareSource.clear).not.toHaveBeenCalled();
expect(ActiveScreenShareSource.setPublishedSource).not.toHaveBeenCalled();
});
test('records the new published source when the switch succeeds', async () => {
replaceActiveDisplayScreenShare.mockResolvedValue(true);
const switched = await switchConfiguredDisplayScreenShare('window:7:0', {preferredDisplaySurface: 'window'});
expect(switched).toBe(true);
expect(ActiveScreenShareSource.setPublishedSource).toHaveBeenCalledWith('app', 'window:7:0', {isOwnWindow: false});
expect(ActiveScreenShareSource.clear).not.toHaveBeenCalled();
});
});
@@ -268,6 +268,7 @@ function getConfiguredScreenShareOptions(
sourceId?: string | null,
preferredDisplaySurface?: 'window' | 'monitor',
videoCodec?: VideoCodec,
includeAudioOverride?: boolean,
) {
const currentResolution = VoiceSettings.getScreenshareResolution();
const currentStreamingMode = VoiceSettings.getStreamingMode();
@@ -288,12 +289,9 @@ function getConfiguredScreenShareOptions(
VoiceSettings.getVideoFrameRate(),
higherQuality,
);
const includeAudio = shouldIncludeAudioForShare(
shareContext,
displayShareEnvironment,
sourceId,
preferredDisplaySurface,
);
const includeAudio =
includeAudioOverride ??
shouldIncludeAudioForShare(shareContext, displayShareEnvironment, sourceId, preferredDisplaySurface);
if (!includeAudio && shareContext !== 'device' && supportsDesktopScreenShareAudioCapture()) {
logger.info('Screen share audio not requested for this surface', {
sourceId,
@@ -328,6 +326,7 @@ export interface ConfiguredDisplayScreenShareOptions {
};
preferredDisplaySurface?: 'window' | 'monitor';
isOwnWindow?: boolean;
includeAudio?: boolean;
}
async function runConfiguredDisplayScreenShare(
@@ -348,6 +347,8 @@ async function runConfiguredDisplayScreenShare(
options?.sourceDimensions,
sourceId,
options?.preferredDisplaySurface,
undefined,
options?.includeAudio,
);
if (electronApi) {
const restartWaylandPortalForSwitch = useWaylandPortal && mode === 'switch';
@@ -373,14 +374,7 @@ async function runConfiguredDisplayScreenShare(
sourceId,
platform: electronApi.platform,
});
degradeAudioToVideoOnly(
captureOptions,
buildAudioCaptureFailureDebug({
sourceId,
platform: electronApi.platform,
reason: 'self-window-audio-route-unavailable',
}),
);
removeAudioFromCaptureOptions(captureOptions);
}
const requestedAppAudioOnLinux =
requestedAudio && !isOwnWindowShare && electronApi.platform === 'linux' && sourceId?.startsWith('window:');
@@ -513,11 +507,14 @@ async function runConfiguredDisplayScreenShare(
}
if (!useWaylandPortal && sourceId) {
setDesktopSourceIntent({sourceId, includeAudio: false});
ActiveScreenShareSource.setSourceId(sourceId, {isOwnWindow: isOwnWindowShare});
}
let operationSucceeded = false;
if (mode === 'switch' && !restartWaylandPortalForSwitch) {
operationSucceeded = await MediaEngine.replaceActiveDisplayScreenShare(captureOptions, publishOptions);
operationSucceeded = await MediaEngine.replaceActiveDisplayScreenShare(captureOptions, publishOptions, {
sourceId: sourceId ?? null,
displayShareEnvironment,
requireAudio: requestedAudio && nativeAudioArmed,
});
} else {
await MediaEngine.setScreenShareEnabled(
true,
@@ -530,7 +527,15 @@ async function runConfiguredDisplayScreenShare(
if (nativeAudioArmed && !captured) {
cleanupNativeAudioAfterCaptureDidNotStart(mode);
}
if (!captured) {
if (captured && !useWaylandPortal && sourceId) {
ActiveScreenShareSource.setPublishedSource(sourceId.startsWith('window:') ? 'app' : 'display', sourceId, {
isOwnWindow: isOwnWindowShare,
});
}
if (captured && useWaylandPortal) {
ActiveScreenShareSource.setPublishedSource('wayland', null);
}
if (!captured && mode === 'start' && !didScreenShareStart()) {
ActiveScreenShareSource.clear();
}
if (!captured && useWaylandPortal && mode !== 'switch') {
@@ -570,7 +575,7 @@ async function runConfiguredDisplayScreenShare(
if (nativeAudioArmed && !capturedAfterError) {
cleanupNativeAudioAfterCaptureDidNotStart(mode);
}
if (!capturedAfterError) {
if (!capturedAfterError && mode === 'start' && !didScreenShareStart()) {
ActiveScreenShareSource.clear();
}
if (useWaylandPortal && !capturedAfterError && mode !== 'switch') {
@@ -589,80 +594,153 @@ async function runConfiguredDisplayScreenShare(
}
let operationSucceeded = false;
if (mode === 'switch') {
operationSucceeded = await MediaEngine.replaceActiveDisplayScreenShare(captureOptions, publishOptions);
operationSucceeded = await MediaEngine.replaceActiveDisplayScreenShare(captureOptions, publishOptions, {
sourceId: null,
displayShareEnvironment,
requireAudio: false,
});
} else {
await MediaEngine.setScreenShareEnabled(true, captureOptions, publishOptions);
operationSucceeded = didScreenShareStart();
}
return mode === 'switch' ? operationSucceeded : didScreenShareStart();
const captured = mode === 'switch' ? operationSucceeded : didScreenShareStart();
if (captured) {
ActiveScreenShareSource.setPublishedSource('web', null);
}
return captured;
}
interface ConfiguredScreenShareMutationRequest {
execute: () => Promise<boolean>;
promise: Promise<boolean>;
resolve: (result: boolean) => void;
reject: (error: unknown) => void;
}
let configuredScreenShareMutationActive = false;
let pendingConfiguredScreenShareMutation: ConfiguredScreenShareMutationRequest | null = null;
function createConfiguredScreenShareMutationRequest(
execute: () => Promise<boolean>,
): ConfiguredScreenShareMutationRequest {
let resolveRequest: ((result: boolean) => void) | undefined;
let rejectRequest: ((error: unknown) => void) | undefined;
const promise = new Promise<boolean>((resolve, reject) => {
resolveRequest = resolve;
rejectRequest = reject;
});
if (!resolveRequest || !rejectRequest) {
throw new Error('Configured screen share mutation deferred was not initialized');
}
return {execute, promise, resolve: resolveRequest, reject: rejectRequest};
}
async function drainConfiguredScreenShareMutations(
initialRequest: ConfiguredScreenShareMutationRequest,
): Promise<void> {
let request: ConfiguredScreenShareMutationRequest | null = initialRequest;
while (request) {
try {
request.resolve(await request.execute());
} catch (error) {
request.reject(error);
}
request = pendingConfiguredScreenShareMutation;
pendingConfiguredScreenShareMutation = null;
}
configuredScreenShareMutationActive = false;
}
function scheduleConfiguredScreenShareMutation(execute: () => Promise<boolean>): Promise<boolean> {
const request = createConfiguredScreenShareMutationRequest(execute);
if (!configuredScreenShareMutationActive) {
configuredScreenShareMutationActive = true;
void drainConfiguredScreenShareMutations(request);
return request.promise;
}
pendingConfiguredScreenShareMutation?.resolve(false);
pendingConfiguredScreenShareMutation = request;
return request.promise;
}
export async function startConfiguredDisplayScreenShare(
sourceId?: string | null,
options?: ConfiguredDisplayScreenShareOptions,
): Promise<boolean> {
let didStart = false;
await executeScreenShareOperation(async () => {
didStart = await runConfiguredDisplayScreenShare(sourceId, options, 'start');
return scheduleConfiguredScreenShareMutation(async () => {
let didStart = false;
await executeScreenShareOperation(async () => {
didStart = await runConfiguredDisplayScreenShare(sourceId, options, 'start');
});
return didStart;
});
return didStart;
}
export async function switchConfiguredDisplayScreenShare(
sourceId?: string | null,
options?: ConfiguredDisplayScreenShareOptions,
): Promise<boolean> {
let didSwitch = false;
await executeScreenShareOperation(async () => {
didSwitch = await runConfiguredDisplayScreenShare(sourceId, options, 'switch');
return scheduleConfiguredScreenShareMutation(async () => {
let didSwitch = false;
await executeScreenShareOperation(async () => {
didSwitch = await runConfiguredDisplayScreenShare(sourceId, options, 'switch');
});
return didSwitch;
});
return didSwitch;
}
export async function startConfiguredDeviceScreenShare(videoDeviceId: string): Promise<boolean> {
normaliseDeviceScreenShareSettings();
const {captureOptions, publishOptions, includeAudio, audioDeviceId} = getConfiguredScreenShareOptions(
'device',
'desktop-custom',
);
try {
await MediaEngine.startDeviceScreenShare(
{
videoDeviceId,
audioDeviceId: includeAudio ? audioDeviceId : undefined,
resolution: captureOptions.resolution,
},
publishOptions,
return scheduleConfiguredScreenShareMutation(async () => {
normaliseDeviceScreenShareSettings();
const {captureOptions, publishOptions, includeAudio, audioDeviceId} = getConfiguredScreenShareOptions(
'device',
'desktop-custom',
);
} catch (error) {
logger.error('Failed to start device screen share', {
error,
videoDeviceId,
});
}
return didScreenShareStart();
try {
await MediaEngine.startDeviceScreenShare(
{
videoDeviceId,
audioDeviceId: includeAudio ? audioDeviceId : undefined,
resolution: captureOptions.resolution,
},
publishOptions,
);
} catch (error) {
logger.error('Failed to start device screen share', {
error,
videoDeviceId,
});
}
const didStart = didScreenShareStart();
if (didStart) ActiveScreenShareSource.setPublishedSource('device', null);
return didStart;
});
}
export async function switchConfiguredDeviceScreenShare(videoDeviceId: string): Promise<boolean> {
normaliseDeviceScreenShareSettings();
const {captureOptions, publishOptions, includeAudio, audioDeviceId} = getConfiguredScreenShareOptions(
'device',
'desktop-custom',
);
try {
return await MediaEngine.replaceActiveDeviceScreenShare(
{
videoDeviceId,
audioDeviceId: includeAudio ? audioDeviceId : undefined,
resolution: captureOptions.resolution,
},
publishOptions,
return scheduleConfiguredScreenShareMutation(async () => {
normaliseDeviceScreenShareSettings();
const {captureOptions, publishOptions, includeAudio, audioDeviceId} = getConfiguredScreenShareOptions(
'device',
'desktop-custom',
);
} catch (error) {
logger.error('Failed to switch device screen share source', {
error,
videoDeviceId,
});
return false;
}
try {
const didSwitch = await MediaEngine.replaceActiveDeviceScreenShare(
{
videoDeviceId,
audioDeviceId: includeAudio ? audioDeviceId : undefined,
resolution: captureOptions.resolution,
},
publishOptions,
);
if (didSwitch) ActiveScreenShareSource.setPublishedSource('device', null);
return didSwitch;
} catch (error) {
logger.error('Failed to switch device screen share source', {
error,
videoDeviceId,
});
return false;
}
});
}