diff --git a/fluxer_app/src/features/channel/components/channel_view/GuildChannelView.tsx b/fluxer_app/src/features/channel/components/channel_view/GuildChannelView.tsx index 8d0b00273..a145b2953 100644 --- a/fluxer_app/src/features/channel/components/channel_view/GuildChannelView.tsx +++ b/fluxer_app/src/features/channel/components/channel_view/GuildChannelView.tsx @@ -44,7 +44,6 @@ import {useVoiceCallFullscreenViewState} from '@app/features/voice/components/us import {VoiceCallView} from '@app/features/voice/components/VoiceCallView'; import {VoiceE2EEIndicator} from '@app/features/voice/components/VoiceE2EEIndicator'; import MediaEngine from '@app/features/voice/engine/MediaEngineFacade'; -import {isNativeVoiceEngineSelected} from '@app/features/voice/engine/native_voice_engine/getVoiceEngine'; import {useCompactCallExpansionState} from '@app/features/voice/hooks/useCompactCallExpansionState'; import {usePendingVoiceConnection} from '@app/features/voice/hooks/usePendingVoiceConnection'; import {getGuildVoiceCallExpansionKey} from '@app/features/voice/state/CompactVoiceCallHeight'; @@ -182,7 +181,7 @@ export const GuildChannelView = observer(({channelId, guildId}: GuildChannelView isVoiceChannel && connectedChannelId === channelId && (connectedGuildId ?? null) === (channel?.guildId ?? null) && - (room || (isNativeVoiceEngineSelected() && MediaEngine.connected)), + room, ); const matureContentGateReason = GuildMatureContentAgree.getGateReason({channelId, guildId}); const matureContentResolved = GuildMatureContentAgree.getResolvedContext({channelId, guildId}); diff --git a/fluxer_app/src/features/channel/components/channel_view/useCallHeaderState.ts b/fluxer_app/src/features/channel/components/channel_view/useCallHeaderState.ts index 281162ad4..8540a4c18 100644 --- a/fluxer_app/src/features/channel/components/channel_view/useCallHeaderState.ts +++ b/fluxer_app/src/features/channel/components/channel_view/useCallHeaderState.ts @@ -3,7 +3,6 @@ import Authentication from '@app/features/auth/state/Authentication'; import type {Channel} from '@app/features/channel/models/Channel'; import MediaEngine from '@app/features/voice/engine/MediaEngineFacade'; -import {isNativeVoiceEngineSelected} from '@app/features/voice/engine/native_voice_engine/getVoiceEngine'; import CallInitiator from '@app/features/voice/state/CallInitiator'; import CallState, {type Call} from '@app/features/voice/state/CallState'; @@ -40,9 +39,7 @@ export function useCallHeaderState(channel?: Channel | null): CallHeaderState { const matchesConnectionContext = Boolean( channelId && MediaEngine.channelId === channelId && (MediaEngine.guildId ?? null) === normalizedGuildId, ); - const isDeviceInRoomForChannelCall = Boolean( - matchesConnectionContext && (MediaEngine.room || (isNativeVoiceEngineSelected() && MediaEngine.connected)), - ); + const isDeviceInRoomForChannelCall = Boolean(matchesConnectionContext && MediaEngine.room); const isDeviceConnectingToChannelCall = matchesConnectionContext && (MediaEngine.connecting || (MediaEngine.connected && !isDeviceInRoomForChannelCall)); const controlsVariant: CallHeaderControlsVariant = !callExistsAndOngoing diff --git a/fluxer_app/src/features/theme/utils/BackgroundImageDB.ts b/fluxer_app/src/features/theme/utils/BackgroundImageDB.ts index ef0d28b82..bc62b46f8 100644 --- a/fluxer_app/src/features/theme/utils/BackgroundImageDB.ts +++ b/fluxer_app/src/features/theme/utils/BackgroundImageDB.ts @@ -53,9 +53,3 @@ export async function getBackgroundImageURL(id: string): Promise if (!media?.dataUrl) return null; return dataUrlToObjectUrl(media.dataUrl); } - -export async function getNativeBackgroundMediaSource(id: string): Promise { - const resolveVoiceBackgroundMedia = getElectronAPI()?.resolveVoiceBackgroundMedia; - if (!resolveVoiceBackgroundMedia) return null; - return resolveVoiceBackgroundMedia(id); -} diff --git a/fluxer_app/src/features/voice/components/PiPOverlay.tsx b/fluxer_app/src/features/voice/components/PiPOverlay.tsx index 8e8909b56..fc15de434 100644 --- a/fluxer_app/src/features/voice/components/PiPOverlay.tsx +++ b/fluxer_app/src/features/voice/components/PiPOverlay.tsx @@ -4,7 +4,6 @@ import PiP from '@app/features/ui/state/PiP'; import {PiPOverlayInner} from '@app/features/voice/components/pip_overlay/PiPOverlayInner'; import {pipOverlayLogger} from '@app/features/voice/components/pip_overlay/shared'; import MediaEngine, {useMediaEngineVersion} from '@app/features/voice/engine/MediaEngineFacade'; -import {isVoiceEngineV2NativeProjectionActiveFromMediaEngine} from '@app/features/voice/engine/VoiceMediaEngineBridge'; import type {Room} from 'livekit-client'; import {observer} from 'mobx-react-lite'; import {useEffect, useState} from 'react'; @@ -19,7 +18,7 @@ export const PiPOverlay = observer(function PiPOverlay() { useEffect(() => { if (room) setActiveRoom(room); }, [room]); - const renderRoom = isVoiceEngineV2NativeProjectionActiveFromMediaEngine() ? room : activeRoom; + const renderRoom = activeRoom; const portalRoot = typeof document === 'undefined' ? null : document.body; const blockReason = !portalRoot ? 'missing-portal-root' diff --git a/fluxer_app/src/features/voice/components/VoiceE2EEIndicator.tsx b/fluxer_app/src/features/voice/components/VoiceE2EEIndicator.tsx index db0499b51..da4d8977f 100644 --- a/fluxer_app/src/features/voice/components/VoiceE2EEIndicator.tsx +++ b/fluxer_app/src/features/voice/components/VoiceE2EEIndicator.tsx @@ -2,9 +2,6 @@ import styles from '@app/features/voice/components/VoiceE2EEIndicator.module.css'; import MediaEngine, {useMediaEngineVersion} from '@app/features/voice/engine/MediaEngineFacade'; -import NativeVoiceE2EEStore from '@app/features/voice/engine/native_voice_engine/NativeVoiceE2EEStore'; -import {useStoreVersion} from '@app/features/voice/engine/Store'; -import {isVoiceEngineV2NativeProjectionActiveFromMediaEngine} from '@app/features/voice/engine/VoiceMediaEngineBridge'; import {computeChannelE2EEStatus} from '@app/features/voice/state/ChannelE2EEStatus'; import { VOICE_CALL_E2EE_BROKEN_DESCRIPTOR, @@ -28,14 +25,9 @@ export const VoiceE2EEIndicator = observer(function VoiceE2EEIndicator({ }: VoiceE2EEIndicatorProps) { const {i18n} = useLingui(); useMediaEngineVersion(); - useStoreVersion(NativeVoiceE2EEStore); void MediaEngine.getAllVoiceStates(); const gatewayStatus = computeChannelE2EEStatus(guildId, channelId, {emptyChannelStatus: 'encrypted'}); - const nativeAggregate = NativeVoiceE2EEStore.aggregateStatus(); - const status = - isVoiceEngineV2NativeProjectionActiveFromMediaEngine() && nativeAggregate !== 'none' - ? nativeAggregate - : gatewayStatus; + const status = gatewayStatus; if (status === 'none') return null; const isEncrypted = status === 'encrypted'; const descriptor = isEncrypted diff --git a/fluxer_app/src/features/voice/components/VoiceParticipantTile.tsx b/fluxer_app/src/features/voice/components/VoiceParticipantTile.tsx index 3c3e39be7..d1d29f2eb 100644 --- a/fluxer_app/src/features/voice/components/VoiceParticipantTile.tsx +++ b/fluxer_app/src/features/voice/components/VoiceParticipantTile.tsx @@ -61,17 +61,12 @@ import { useAutoVideoSubscription, useEffectiveTrackRef, useIntersection, - useNativeCameraSubscriptionQuality, useScreenShareAudioPublication, useScreensharePreviewUploader, useScreenshareWatchSubscription, useTileContextMenuActive, } from '@app/features/voice/components/voice_participant_tile/hooks'; import LastFrameSnapshotCache from '@app/features/voice/components/voice_participant_tile/LastFrameSnapshotCache'; -import { - NativeParticipantVideo, - useNativeParticipantVideoTrack, -} from '@app/features/voice/components/voice_participant_tile/NativeParticipantVideo'; import {ScreenSharePlaceholder} from '@app/features/voice/components/voice_participant_tile/ScreenSharePlaceholder'; import { CAMERA_BUFFERING_DESCRIPTOR, @@ -79,7 +74,6 @@ import { CONNECTION_DESCRIPTOR, DESKTOP_DEVICE_DESCRIPTOR, getSourceDataAttr, - isAudioTrackWithVolume, isCameraSource, logger, MOBILE_DEVICE_DESCRIPTOR, @@ -102,10 +96,8 @@ import { } from '@app/features/voice/components/voice_participant_tile/shared'; import {WatchStreamOverlay} from '@app/features/voice/components/voice_participant_tile/WatchStreamOverlay'; import MediaEngine, {useMediaEngineVersion, useVoiceEngineV2Model} from '@app/features/voice/engine/MediaEngineFacade'; -import NativeVideoTileManager from '@app/features/voice/engine/native_voice_engine/NativeVideoTileManager'; import ScreenSharePublicationMigration from '@app/features/voice/engine/ScreenSharePublicationMigration'; import {useStoreVersion} from '@app/features/voice/engine/Store'; -import {isVoiceEngineV2NativeProjectionActiveFromMediaEngine} from '@app/features/voice/engine/VoiceMediaEngineBridge'; import { selectVoiceMediaGraphDeferredStopKeys, selectVoiceMediaGraphFailure, @@ -118,11 +110,9 @@ import { asVoiceTrackSource, VoiceTrackSource, } from '@app/features/voice/engine/VoiceTrackSource'; -import voiceEngineV2AppDebugLoggingHostAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppDebugLoggingHostAdapter'; import {selectVoiceEngineV2AppEffectiveSelfMuteForVoiceStatePayload} from '@app/features/voice/engine/v2/VoiceEngineV2AppSelectors'; import CallMediaPrefs from '@app/features/voice/state/CallMediaPrefs'; import LocalVoiceState from '@app/features/voice/state/LocalVoiceState'; -import ParticipantVolume from '@app/features/voice/state/ParticipantVolume'; import PopoutWindowManager, {getVoiceTilePopoutKey} from '@app/features/voice/state/PopoutWindowManager'; import { getScreenShareWatchFailureForPublicationOperation, @@ -145,7 +135,7 @@ import { } from '@app/features/voice/utils/VoiceMessageDescriptors'; import {parseVoiceParticipantIdentity} from '@app/features/voice/utils/VoiceParticipantIdentity'; import {isParticipantVoicePermissionMuted} from '@app/features/voice/utils/VoicePermissionUtils'; -import {boostedVoiceVolumePercentToTrackVolume} from '@app/features/voice/utils/VoiceVolumeUtils'; +import {VOICE_VOLUME_MAX_SLIDER_VOLUME} from '@app/features/voice/utils/VoiceVolumeUtils'; import {DEFAULT_ACCENT_COLOR} from '@fluxer/constants/src/AppConstants'; import {Permissions} from '@fluxer/constants/src/ChannelConstants'; import {msg, plural} from '@lingui/core/macro'; @@ -360,26 +350,16 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({ isScreenShare, ); const hasScreenShareAudio = Boolean(screenShareAudioPublication); - const isNativeEngine = isVoiceEngineV2NativeProjectionActiveFromMediaEngine(); - const nativeParticipantSid = isNativeEngine ? (MediaEngine.participants[identity]?.sid ?? participant.sid ?? '') : ''; - const nativeVideoSource = asPinnableVoiceTrackSource(trackRef.source); - const nativeVideoTrack = useNativeParticipantVideoTrack(nativeParticipantSid, nativeVideoSource, identity); - const hasNativeVideo = nativeVideoTrack != null; - const hasNativeVideoFrame = (nativeVideoTrack?.width ?? 0) > 0 && (nativeVideoTrack?.height ?? 0) > 0; const hasVideo = useMemo(() => { - if (isNativeEngine) { - const hasRenderableNativeVideo = hasNativeVideo; - return hasRenderableNativeVideo && !cameraLocallyDisabled; - } if (!isTrackReference(trackRef)) return false; const pub = trackRef.publication; return Boolean(pub?.track) && !pub?.isMuted && !cameraLocallyDisabled; - }, [cameraLocallyDisabled, hasNativeVideo, isNativeEngine, trackRef]); + }, [cameraLocallyDisabled, trackRef]); const videoRef = useRef(null); const liveKitCameraTrackSid = isTrackReference(trackRef) ? trackRef.publication.trackSid : ''; - const cameraVideoFrameTrackKey = nativeVideoTrack?.trackSid ?? publication?.trackSid ?? liveKitCameraTrackSid ?? ''; + const cameraVideoFrameTrackKey = publication?.trackSid ?? liveKitCameraTrackSid ?? ''; const cameraVideoFrameResetKey = isCameraTile - ? `${isNativeEngine ? 'native' : 'livekit'}:${identity}:${cameraVideoFrameTrackKey}:${hasVideo ? 'video' : 'waiting'}` + ? `livekit:${identity}:${cameraVideoFrameTrackKey}:${hasVideo ? 'video' : 'waiting'}` : ''; const hasRenderedCameraVideoFrame = useVideoRenderedFrame({ enabled: isCameraTile && hasVideo, @@ -388,7 +368,6 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({ }); const streamVolume = StreamAudioPrefs.getVolume(streamKey); const isStreamMuted = StreamAudioPrefs.isMuted(streamKey); - const isParticipantLocallyMuted = ParticipantVolume.isLocalMuted(userId); const hasStreamAudioPrefsEntry = StreamAudioPrefs.hasEntry(streamKey); const isSubscribed = Boolean(publication?.isSubscribed); const hasSubscribedScreenShareVideo = isSubscribed && hasVideo; @@ -396,10 +375,6 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({ isScreenShare && !isOwnScreenShare && ScreenSharePublicationMigration.isScreenShareBuffering(participant); const shouldAutoSubscribe = allowAutoSubscribe && !isFocusedPlaceholderTile; const {ref: tileRef, isIntersecting} = useIntersection(shouldAutoSubscribe); - const nativeCameraQuality = useNativeCameraSubscriptionQuality( - tileRef, - isNativeEngine && isCameraTile && shouldAutoSubscribe && isIntersecting, - ); useAutoVideoSubscription({ enabled: shouldAutoSubscribe, trackRef, @@ -407,7 +382,6 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({ videoLocallyDisabled: cameraLocallyDisabled, isLocalParticipant, isScreenShare, - nativeCameraQuality, }); useStoreVersion(voiceMediaGraphStore); const graphSnapshot = voiceMediaGraphStore.getGraphSnapshot(); @@ -453,15 +427,14 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({ ScreenShareWatchFailures.reportFailure({ streamKey, participantIdentity: identity, - participantSid: nativeParticipantSid || undefined, - trackSid: publication?.trackSid ?? nativeVideoTrack?.trackSid, + trackSid: publication?.trackSid, source: VoiceTrackSource.ScreenShare, code: failure.code, reason: failure.reason, error, }); }, - [identity, isWatching, nativeParticipantSid, nativeVideoTrack?.trackSid, publication?.trackSid, streamKey], + [identity, isWatching, publication?.trackSid, streamKey], ); useScreenshareWatchSubscription({ isScreenShare: isInteractiveScreenShareTile, @@ -469,7 +442,7 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({ userWantsToWatch: isWatching, videoLocallyDisabled: false, isOwnScreenShare, - audioEnabled: !isStreamMuted && !isParticipantLocallyMuted, + audioEnabled: !isStreamMuted, audioPublication: screenShareAudioPublication, streamKey, onVideoSubscriptionError: reportVideoSubscriptionError, @@ -480,20 +453,11 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({ if (!isWatching) return; const pub = screenShareAudioPublication; if (!pub) return; - const track = pub.track; - if (isAudioTrackWithVolume(track)) { - try { - track.setVolume(boostedVoiceVolumePercentToTrackVolume(streamVolume)); - } catch (err) { - logger.error('setVolume failed for stream audio', err); - } - } - const shouldEnable = !isStreamMuted && !isParticipantLocallyMuted; + const shouldEnable = !isStreamMuted; logger.debug('Applying runtime screen share audio enabled state', { trackSid: pub.trackSid, isWatching, isStreamMuted, - isParticipantLocallyMuted, shouldEnable, }); syncScreenSharePublication({ @@ -513,7 +477,6 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({ screenShareAudioPublication, streamVolume, isStreamMuted, - isParticipantLocallyMuted, userId, ]); useEffect(() => { @@ -545,7 +508,7 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({ canFetchStreamPreview; const {previewUrl, isPreviewLoading} = useStreamPreview(previewEnabled, streamKey); const isStreamPlaceholder = isScreenShare && !isTrackReference(trackRef); - const screenShareTrackSid = publication?.trackSid ?? nativeVideoTrack?.trackSid ?? null; + const screenShareTrackSid = publication?.trackSid ?? null; const trackInfo = useStreamTrackInfo(isScreenShare && !isFocusPresentationTile ? trackRef : null, { nativeSource: isScreenShare ? VoiceTrackSource.ScreenShare : null, nativeTrackSid: screenShareTrackSid, @@ -562,12 +525,10 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({ !isStreamPlaceholder, streamKey, participantIdentity: identity, - participantSid: nativeParticipantSid || undefined, trackSid: screenShareTrackSid, - hasPublication: publication != null || nativeVideoTrack != null, + hasPublication: publication != null, isPublicationDesired, hasSubscribedVideo: hasSubscribedScreenShareVideo, - hasNativeFrame: hasNativeVideoFrame, operationKey: isScreenShareRepublishBuffering ? `republish:${screenSharePublicationMigrationVersion}` : null, videoRef, }); @@ -580,12 +541,6 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({ if (!lastFrameSnapshotKey) return; if (!isScreenShareRepublishBuffering) return; LastFrameSnapshotCache.captureFromVideoElement(lastFrameSnapshotKey, videoRef.current); - if (screenShareTrackSid && LastFrameSnapshotCache.getSnapshotUrl(lastFrameSnapshotKey) === null) { - LastFrameSnapshotCache.captureFromNativeFrame( - lastFrameSnapshotKey, - NativeVideoTileManager.getRetainedLastFrame(screenShareTrackSid), - ); - } }, [isScreenShareRepublishBuffering, lastFrameSnapshotKey, screenShareTrackSid]); useEffect(() => { if (!lastFrameSnapshotKey) return; @@ -619,12 +574,10 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({ const isLocalCameraRequested = isCameraTile && isOwnContent && LocalVoiceState.getSelfVideo(); const isCameraActive = selectVoiceParticipantTileCameraActive({ isCameraTile, - isNativeEngine, isOwnContent, isCameraPublicationActive, isParticipantCameraActive, isLocalCameraRequested, - hasNativeVideo, }); const isCameraBuffering = shouldShowCameraBuffering({ isScreenShare, @@ -635,50 +588,6 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({ hasVideo, hasRenderedVideoFrame: hasRenderedCameraVideoFrame, }); - useEffect(() => { - if (!isNativeEngine) return; - if (!isCameraTile) return; - voiceEngineV2AppDebugLoggingHostAdapter.recordNativeVideoDiagnostic('tile.camera_state', { - participantIdentity: identity, - participantSid: nativeParticipantSid, - source: nativeVideoSource, - trackSid: nativeVideoTrack?.trackSid ?? null, - hasNativeVideo, - hasNativeVideoFrame, - hasVideo, - hasRenderedVideoFrame: hasRenderedCameraVideoFrame, - isCameraActive, - isCameraPublicationActive, - isParticipantCameraActive, - isLocalCameraRequested, - isCameraBuffering, - cameraLocallyDisabled, - isOwnCameraHidden, - width: nativeVideoTrack?.width ?? null, - height: nativeVideoTrack?.height ?? null, - }); - }, [ - cameraLocallyDisabled, - hasNativeVideo, - hasNativeVideoFrame, - hasRenderedCameraVideoFrame, - hasVideo, - identity, - isCameraActive, - isCameraPublicationActive, - isCameraBuffering, - isCameraTile, - isNativeEngine, - isParticipantCameraActive, - isLocalCameraRequested, - isOwnCameraHidden, - isOwnContent, - nativeParticipantSid, - nativeVideoSource, - nativeVideoTrack?.height, - nativeVideoTrack?.trackSid, - nativeVideoTrack?.width, - ]); const cameraBufferingLabel = i18n._(CAMERA_BUFFERING_DESCRIPTOR); const screenShareBufferingLabel = i18n._(STREAM_BUFFERING_DESCRIPTOR); const watchFailedTitle = i18n._(WATCHING_FAILED_DESCRIPTOR); @@ -854,10 +763,7 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({ videoRef, }); const hasVisibleMediaTile = - !isFocusedPlaceholderTile && - (isNativeEngine ? hasVideo : isTrackReference(trackRef)) && - hasVideo && - !shouldHideOwnScreenShareVideo; + !isFocusedPlaceholderTile && isTrackReference(trackRef) && hasVideo && !shouldHideOwnScreenShareVideo; const isAvatarOnlyTile = !hasVisibleMediaTile && !isScreenShare; const shouldShowTileSpeakingIndicator = !isFocusedPlaceholderTile && isActuallySpeaking && !isScreenShare && !isAvatarOnlyTile; @@ -898,17 +804,6 @@ const VoiceParticipantTileInner = observer(function VoiceParticipantTileInner({ /> ); } - if (isNativeEngine && hasVideo && nativeVideoSource != null && !shouldHideOwnScreenShareVideo) { - return ( - - ); - } if (isTrackReference(trackRef) && hasVideo && !shouldHideOwnScreenShareVideo) { return ( { }); describe('VoiceParticipantTileStateMachine camera active state', () => { - it('keeps stale participant camera flags from holding a stopped own native camera active', () => { - expect( - selectVoiceParticipantTileCameraActive( - cameraActiveSignals({ - isNativeEngine: true, - isOwnContent: true, - isParticipantCameraActive: true, - }), - ), - ).toBe(false); - }); - - it('treats own native camera as active while local capture is requested or native video exists', () => { - expect( - selectVoiceParticipantTileCameraActive( - cameraActiveSignals({isNativeEngine: true, isOwnContent: true, isLocalCameraRequested: true}), - ), - ).toBe(true); - expect( - selectVoiceParticipantTileCameraActive( - cameraActiveSignals({isNativeEngine: true, isOwnContent: true, hasNativeVideo: true}), - ), - ).toBe(true); - }); - it('preserves participant camera flags for remote camera tiles', () => { expect(selectVoiceParticipantTileCameraActive(cameraActiveSignals({isParticipantCameraActive: true}))).toBe(true); }); + + it('treats a local capture request as an active own camera tile', () => { + expect( + selectVoiceParticipantTileCameraActive(cameraActiveSignals({isOwnContent: true, isLocalCameraRequested: true})), + ).toBe(true); + }); + + it('keeps a camera tile inactive without a publication, participant flag, or local request', () => { + expect(selectVoiceParticipantTileCameraActive(cameraActiveSignals())).toBe(false); + expect(selectVoiceParticipantTileCameraActive(cameraActiveSignals({isCameraTile: false}))).toBe(false); + }); }); describe('VoiceParticipantTileStateMachine graph-derived screen share state', () => { diff --git a/fluxer_app/src/features/voice/components/VoiceParticipantTileStateMachine.ts b/fluxer_app/src/features/voice/components/VoiceParticipantTileStateMachine.ts index a2dfad0a4..fe132d931 100644 --- a/fluxer_app/src/features/voice/components/VoiceParticipantTileStateMachine.ts +++ b/fluxer_app/src/features/voice/components/VoiceParticipantTileStateMachine.ts @@ -33,12 +33,10 @@ export interface VoiceParticipantTileCameraBufferingSignals { export interface VoiceParticipantTileCameraActiveSignals { isCameraTile: boolean; - isNativeEngine: boolean; isOwnContent: boolean; isCameraPublicationActive: boolean; isParticipantCameraActive: boolean; isLocalCameraRequested: boolean; - hasNativeVideo: boolean; } type VoiceParticipantTileEvent = { @@ -153,14 +151,10 @@ export function shouldShowCameraBuffering(signals: VoiceParticipantTileCameraBuf export function selectVoiceParticipantTileCameraActive(signals: VoiceParticipantTileCameraActiveSignals): boolean { if (!signals.isCameraTile) return false; - if (signals.isNativeEngine && signals.isOwnContent) { - if (signals.isLocalCameraRequested) return true; - return signals.hasNativeVideo; - } if (signals.isCameraPublicationActive) return true; if (signals.isParticipantCameraActive) return true; if (signals.isLocalCameraRequested) return true; - return signals.hasNativeVideo; + return false; } export const voiceParticipantTileStateMachine = setup({ diff --git a/fluxer_app/src/features/voice/components/modals/CameraPreviewModal.tsx b/fluxer_app/src/features/voice/components/modals/CameraPreviewModal.tsx index aeb9486d5..c36debdc5 100644 --- a/fluxer_app/src/features/voice/components/modals/CameraPreviewModal.tsx +++ b/fluxer_app/src/features/voice/components/modals/CameraPreviewModal.tsx @@ -20,14 +20,8 @@ import {Tooltip} from '@app/features/ui/tooltip/Tooltip'; import {formatRoundedPercentage} from '@app/features/ui/utils/PercentageFormatting'; import * as VoiceSettingsCommands from '@app/features/voice/commands/VoiceSettingsCommands'; import styles from '@app/features/voice/components/modals/CameraPreviewModal.module.css'; -import { - NATIVE_CAMERA_PREVIEW_RETRY_DELAY_MS, - selectNativeCameraPreviewFallback, -} from '@app/features/voice/components/modals/CameraPreviewSessionPolicy'; import MediaEngine, {useMediaEngineVersion} from '@app/features/voice/engine/MediaEngineFacade'; import {VOICE_CAMERA_USER_LIMIT_REACHED_DESCRIPTOR} from '@app/features/voice/engine/media_engine_facade/shared'; -import NativeVideoTileManager from '@app/features/voice/engine/native_voice_engine/NativeVideoTileManager'; -import {useStoreVersion} from '@app/features/voice/engine/Store'; import VoiceDevicePermissionState from '@app/features/voice/engine/VoiceDevicePermissionState'; import {useCameraUserCapBlocked} from '@app/features/voice/hooks/useCameraUserCapBlocked'; import VoiceSettings, { @@ -82,11 +76,6 @@ const YOU_DON_T_HAVE_PERMISSION_TO_TURN_ON_DESCRIPTOR = msg({ comment: 'Tooltip / error shown in the camera preview modal when the user lacks Video permission in the current channel. Tone stays plain.', }); -const EFFECTS_PREVIEW_UNAVAILABLE_DESCRIPTOR = msg({ - message: 'Effects preview unavailable — showing unprocessed camera', - comment: - 'Inline notice in the camera preview modal when the native effects preview cannot start and the raw camera feed is shown instead.', -}); const BLUR_STRENGTH_DESCRIPTOR = msg({ message: 'Blur strength', comment: 'Slider label in the camera preview modal controlling how strong the background blur effect is.', @@ -326,93 +315,6 @@ async function setupPreviewTrackAndProcessor(args: CameraPreviewTrackSetupArgs): return 'ready'; } -function usePublishedNativeCameraPreviewStream( - enabled: boolean, - localParticipant: LocalParticipant | undefined, -): MediaStream | null { - useStoreVersion(NativeVideoTileManager); - if (!enabled) return null; - return MediaEngine.getNativeCameraLocalPreviewStream(localParticipant ?? null); -} - -interface NativeCameraPreviewSession { - stream: MediaStream | null; - failed: boolean; -} - -function useNativeCameraPreviewSession(enabled: boolean): NativeCameraPreviewSession { - useStoreVersion(NativeVideoTileManager); - const [trackSid, setTrackSid] = useState(null); - const [failed, setFailed] = useState(false); - const [retryNonce, setRetryNonce] = useState(0); - const retryAttemptRef = useRef(0); - const voiceSettings = VoiceSettings; - const videoDeviceId = voiceSettings.videoDeviceId; - const backgroundImageId = voiceSettings.backgroundImageId; - const cameraResolution = voiceSettings.cameraResolution; - const videoFrameRate = voiceSettings.videoFrameRate; - const backgroundBlurStrength = voiceSettings.backgroundBlurStrength; - useEffect(() => { - retryAttemptRef.current = 0; - }, [enabled, videoDeviceId, backgroundImageId, cameraResolution, videoFrameRate, backgroundBlurStrength]); - useEffect(() => { - if (!enabled) { - return; - } - let cancelled = false; - let retryTimeoutId: number | null = null; - const scheduleRetry = () => { - const decision = selectNativeCameraPreviewFallback({ - sessionFailed: true, - backgroundEffectConfigured: backgroundImageId !== NONE_BACKGROUND_ID, - retryAttempt: retryAttemptRef.current, - }); - if (!decision.shouldScheduleRetry) return; - retryAttemptRef.current += 1; - retryTimeoutId = window.setTimeout(() => { - retryTimeoutId = null; - setRetryNonce((nonce) => nonce + 1); - }, NATIVE_CAMERA_PREVIEW_RETRY_DELAY_MS); - }; - setFailed(false); - MediaEngine.startNativeCameraPreviewSession() - .then((startedTrackSid) => { - if (cancelled) return; - setTrackSid(startedTrackSid); - if (startedTrackSid) { - retryAttemptRef.current = 0; - return; - } - setFailed(true); - scheduleRetry(); - }) - .catch((error) => { - logger.warn('Failed to start native camera preview session', {error}); - if (cancelled) return; - setTrackSid(null); - setFailed(true); - scheduleRetry(); - }); - return () => { - cancelled = true; - if (retryTimeoutId !== null) { - window.clearTimeout(retryTimeoutId); - } - }; - }, [enabled, retryNonce, videoDeviceId, backgroundImageId, cameraResolution, videoFrameRate, backgroundBlurStrength]); - useEffect(() => { - if (!enabled) { - return; - } - return () => { - setTrackSid(null); - void MediaEngine.stopNativeCameraPreviewSession(); - }; - }, [enabled]); - const stream = enabled && trackSid ? (NativeVideoTileManager.tracks[trackSid]?.stream ?? null) : null; - return {stream, failed}; -} - interface CameraEffectStrengthSliderProps { label: string; value: number; @@ -493,27 +395,13 @@ const CameraPreviewModalContent = observer((props: CameraPreviewModalProps) => { const {i18n} = useLingui(); useMediaEngineVersion(); const {localParticipant, onEnabled, onEnableCamera, isCameraEnabled, showEnableCameraButton = true} = props; - const nativeCameraPublished = MediaEngine.isNativeCameraPublished(); - const nativePreviewStream = usePublishedNativeCameraPreviewStream(nativeCameraPublished, localParticipant); - const hasPublishedNativePreview = nativePreviewStream != null; - const cameraAlreadyOn = isCameraEnabled === true || nativeCameraPublished; - const nativePreviewSessionEnabled = - !nativeCameraPublished && !hasPublishedNativePreview && MediaEngine.isNativeCameraPreviewSessionAvailable(); - const nativePreviewSession = useNativeCameraPreviewSession(nativePreviewSessionEnabled); - const usesNativePreviewSession = nativePreviewSessionEnabled && !nativePreviewSession.failed; - const activeNativeStream = nativePreviewStream ?? (usesNativePreviewSession ? nativePreviewSession.stream : null); + const cameraAlreadyOn = isCameraEnabled === true; const voiceBackgroundsAvailable = areVoiceBackgroundsAvailable(); const channelId = MediaEngine.channelId; const guildId = MediaEngine.guildId; const canStream = !localParticipant || !guildId || !channelId || Permission.can(Permissions.STREAM, {channelId}); const cameraCapBlocked = useCameraUserCapBlocked(cameraAlreadyOn); const selectedBackgroundImageId = voiceBackgroundsAvailable ? VoiceSettings.backgroundImageId : NONE_BACKGROUND_ID; - const backgroundEffectConfigured = selectedBackgroundImageId !== NONE_BACKGROUND_ID; - const showEffectsUnavailableNotice = selectNativeCameraPreviewFallback({ - sessionFailed: nativePreviewSessionEnabled && nativePreviewSession.failed, - backgroundEffectConfigured, - retryAttempt: 0, - }).showEffectsUnavailableNotice; const [videoDevices, setVideoDevices] = useState>([]); const [status, setStatus] = useState< 'idle' | 'initializing' | 'ready' | 'error' | 'fixing' | 'fix-settling' | 'fix-switching-back' @@ -580,12 +468,6 @@ const CameraPreviewModalContent = observer((props: CameraPreviewModalProps) => { } return; } - if (hasPublishedNativePreview || usesNativePreviewSession || nativeCameraPublished) { - if (isCurrentInitialization()) { - setError(null); - } - return; - } if (!isCurrentInitialization()) { return; } @@ -654,16 +536,7 @@ const CameraPreviewModalContent = observer((props: CameraPreviewModalProps) => { }); } } - }, [ - applyResolutionFix, - backgroundOverrideId, - hasPublishedNativePreview, - usesNativePreviewSession, - nativeCameraPublished, - i18n, - videoDevices, - voiceBackgroundsAvailable, - ]); + }, [applyResolutionFix, backgroundOverrideId, i18n, videoDevices, voiceBackgroundsAvailable]); const handleDeviceChange = useCallback((deviceId: string) => { VoiceSettingsCommands.update({videoDeviceId: deviceId}); }, []); @@ -679,7 +552,6 @@ const CameraPreviewModalContent = observer((props: CameraPreviewModalProps) => { }, [voiceBackgroundsAvailable]); const handleEnableCamera = useCallback(async () => { try { - await MediaEngine.stopNativeCameraPreviewSession(); if (!localParticipant) { await onEnableCamera?.(); onEnabled?.(); @@ -740,44 +612,6 @@ const CameraPreviewModalContent = observer((props: CameraPreviewModalProps) => { }; }, [handleDeviceUpdate]); useEffect(() => { - if (!activeNativeStream) return; - initializationGenerationRef.current++; - if (trackRef.current) { - trackRef.current.stop(); - trackRef.current = null; - } - if (processorRef.current) { - processorRef.current.destroy().catch((error) => { - logger.warn('Failed to destroy browser camera preview processor after native preview became available', { - error, - }); - }); - processorRef.current = null; - } - const videoElement = videoRef.current; - if (!videoElement) return; - videoElement.muted = true; - videoElement.autoplay = true; - videoElement.playsInline = true; - videoElement.srcObject = activeNativeStream; - setStatus('ready'); - setError(null); - const playResult = videoElement.play(); - if (playResult && typeof playResult.catch === 'function') { - playResult.catch((error) => { - logger.debug('Native camera preview play() rejected', {error}); - }); - } - return () => { - if (videoRef.current?.srcObject === activeNativeStream) { - videoRef.current.srcObject = null; - } - }; - }, [activeNativeStream]); - useEffect(() => { - if (hasPublishedNativePreview || usesNativePreviewSession || nativeCameraPublished) { - return; - } const voiceSettings = VoiceSettings; const backgroundImageId = backgroundOverrideId ?? (voiceBackgroundsAvailable ? voiceSettings.backgroundImageId : NONE_BACKGROUND_ID); @@ -794,9 +628,6 @@ const CameraPreviewModalContent = observer((props: CameraPreviewModalProps) => { }, [ initializeCamera, backgroundOverrideId, - hasPublishedNativePreview, - usesNativePreviewSession, - nativeCameraPublished, VoiceSettings.videoDeviceId, VoiceSettings.backgroundImageId, VoiceSettings.mirrorCamera, @@ -806,8 +637,7 @@ const CameraPreviewModalContent = observer((props: CameraPreviewModalProps) => { ]); const voiceSettings = VoiceSettings; const effectiveVideoDeviceId = resolveEffectiveDeviceId(voiceSettings.videoDeviceId, videoDevices) ?? 'default'; - const previewVideoClassName = - activeNativeStream && voiceSettings.mirrorCamera ? `${styles.video} ${styles.videoMirrored}` : styles.video; + const previewVideoClassName = styles.video; const videoDeviceOptions = videoDevices.length > 0 ? videoDevices.map((device) => ({ @@ -871,14 +701,6 @@ const CameraPreviewModalContent = observer((props: CameraPreviewModalProps) => { )} )} - {showEffectsUnavailableNotice && ( -
- {i18n._(EFFECTS_PREVIEW_UNAVAILABLE_DESCRIPTOR)} -
- )}
{ - it('does nothing while the native session is healthy', () => { - expect( - selectNativeCameraPreviewFallback({ - sessionFailed: false, - backgroundEffectConfigured: true, - retryAttempt: 0, - }), - ).toEqual({showEffectsUnavailableNotice: false, shouldScheduleRetry: false}); - }); - - it('shows the effects notice when a background effect is configured and the session failed', () => { - expect( - selectNativeCameraPreviewFallback({ - sessionFailed: true, - backgroundEffectConfigured: true, - retryAttempt: 0, - }), - ).toEqual({showEffectsUnavailableNotice: true, shouldScheduleRetry: true}); - }); - - it('hides the notice when no background effect is configured but still retries', () => { - expect( - selectNativeCameraPreviewFallback({ - sessionFailed: true, - backgroundEffectConfigured: false, - retryAttempt: 0, - }), - ).toEqual({showEffectsUnavailableNotice: false, shouldScheduleRetry: true}); - }); - - it('retries up to the named cap and then stops', () => { - for (let attempt = 0; attempt < NATIVE_CAMERA_PREVIEW_RETRY_MAX; attempt++) { - expect( - selectNativeCameraPreviewFallback({ - sessionFailed: true, - backgroundEffectConfigured: true, - retryAttempt: attempt, - }).shouldScheduleRetry, - ).toBe(true); - } - expect( - selectNativeCameraPreviewFallback({ - sessionFailed: true, - backgroundEffectConfigured: true, - retryAttempt: NATIVE_CAMERA_PREVIEW_RETRY_MAX, - }), - ).toEqual({showEffectsUnavailableNotice: true, shouldScheduleRetry: false}); - }); - - it('rejects negative retry attempts', () => { - expect(() => - selectNativeCameraPreviewFallback({ - sessionFailed: true, - backgroundEffectConfigured: true, - retryAttempt: -1, - }), - ).toThrow('native camera preview retry attempt must be non-negative'); - }); -}); diff --git a/fluxer_app/src/features/voice/components/modals/CameraPreviewSessionPolicy.ts b/fluxer_app/src/features/voice/components/modals/CameraPreviewSessionPolicy.ts deleted file mode 100644 index 80320a562..000000000 --- a/fluxer_app/src/features/voice/components/modals/CameraPreviewSessionPolicy.ts +++ /dev/null @@ -1,31 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; - -export const NATIVE_CAMERA_PREVIEW_RETRY_MAX = 3; -export const NATIVE_CAMERA_PREVIEW_RETRY_DELAY_MS = 2000; - -export interface NativeCameraPreviewFallbackInput { - sessionFailed: boolean; - backgroundEffectConfigured: boolean; - retryAttempt: number; -} - -export interface NativeCameraPreviewFallbackDecision { - showEffectsUnavailableNotice: boolean; - shouldScheduleRetry: boolean; -} - -export function selectNativeCameraPreviewFallback( - input: NativeCameraPreviewFallbackInput, -): NativeCameraPreviewFallbackDecision { - assert.ok(Number.isInteger(input.retryAttempt), 'native camera preview retry attempt must be an integer'); - assert.ok(input.retryAttempt >= 0, 'native camera preview retry attempt must be non-negative'); - if (!input.sessionFailed) { - return {showEffectsUnavailableNotice: false, shouldScheduleRetry: false}; - } - return { - showEffectsUnavailableNotice: input.backgroundEffectConfigured, - shouldScheduleRetry: input.retryAttempt < NATIVE_CAMERA_PREVIEW_RETRY_MAX, - }; -} diff --git a/fluxer_app/src/features/voice/components/modals/ScreenSharePickerModal.tsx b/fluxer_app/src/features/voice/components/modals/ScreenSharePickerModal.tsx index 68ef797b8..d043e7641 100644 --- a/fluxer_app/src/features/voice/components/modals/ScreenSharePickerModal.tsx +++ b/fluxer_app/src/features/voice/components/modals/ScreenSharePickerModal.tsx @@ -27,13 +27,11 @@ import {screenRecordingPermissionAllowsPickerSources} from '@app/features/voice/ import { DESKTOP_SOURCE_LIST_POLL_INTERVAL_MS, desktopSourceIdentitiesMatch, - findNativeCaptureSourceForDesktopSource, getDesktopSourceThumbnailStateKey, hasDesktopSourcesMissingThumbnails, isDisplaySource, isUsableImageDataUrl, isWindowSource, - LINUX_GAME_CAPTURE_SELECTION_ID, logger, mergeDesktopSources, NATIVE_DISPLAY_SELECTION_ID, @@ -54,17 +52,13 @@ import { } from '@app/features/voice/components/modals/screen_share_picker_modal/useScreenSharePickerDisplayPermission'; import {StreamSettingsMenuContent} from '@app/features/voice/components/StreamSettingsMenuContent'; import MediaEngine, {useVoiceEngineV2Model} from '@app/features/voice/engine/MediaEngineFacade'; -import ScreenShareCodecNegotiation from '@app/features/voice/engine/ScreenShareCodecNegotiation'; import {selectVoiceEngineV2AppConnection} from '@app/features/voice/engine/v2/VoiceEngineV2AppSelectors'; -import {isNativeScreenCaptureAvailable} from '@app/features/voice/engine/voice_screen_share_manager/DisplayMediaCapture'; import {useMediaDevices} from '@app/features/voice/hooks/useMediaDevices'; import VoiceSettings, { type LastScreenShareSource, type LastScreenShareSourceKind, } from '@app/features/voice/state/VoiceSettings'; -import {shouldUseNativeScreenCaptureForScreenShareCodec} from '@app/features/voice/utils/CodecCapabilityDetector'; import {getNativeAudioAvailabilityCached} from '@app/features/voice/utils/NativeAudioCaptureBridge'; -import {getNativeScreenCaptureApi} from '@app/features/voice/utils/native_screen_capture_bridge/shared'; import { getDisplayShareEnvironment, shouldShowDesktopDownloadCta, @@ -75,17 +69,14 @@ import { normaliseDeviceScreenShareSettings, startConfiguredDeviceScreenShare, startConfiguredDisplayScreenShare, - startConfiguredNativeDisplayScreenShare, switchConfiguredDeviceScreenShare, switchConfiguredDisplayScreenShare, - switchConfiguredNativeDisplayScreenShare, } from '@app/features/voice/utils/ScreenShareStartFlow'; import {formatFallbackCameraLabel} from '@app/features/voice/utils/VoiceMessageDescriptors'; -import type {DesktopSource, NativeAudioAvailability, NativeScreenCaptureSource} from '@app/types/electron.d'; +import type {DesktopSource, NativeAudioAvailability} from '@app/types/electron.d'; import {msg} from '@lingui/core/macro'; import {useLingui} from '@lingui/react/macro'; import {AppWindowIcon, GearIcon, InfoIcon, MonitorIcon, VideoCameraIcon} from '@phosphor-icons/react'; -import type {VideoCodec} from 'livekit-client'; import {observer} from 'mobx-react-lite'; import { type MouseEvent as ReactMouseEvent, @@ -162,10 +153,6 @@ const OPEN_SYSTEM_PICKER_DESCRIPTOR = msg({ message: 'Open system picker', comment: 'Primary button in the screen-share picker on Linux/Wayland. Hands off to the OS xdg-desktop-portal picker.', }); -const GAME_CAPTURE_DESCRIPTOR = msg({ - message: 'Game capture', - comment: 'Button label that starts native Linux game capture through the OBS-compatible hook path.', -}); const CHANGE_STREAM_SOURCE_DESCRIPTOR = msg({ message: 'Change stream source', comment: 'Modal title for the screen-share picker when switching an ongoing stream to a new source.', @@ -306,33 +293,6 @@ function getDesktopSourceDimensions(source: DesktopSource): {width: number; heig : undefined; } -function canUseNativeCaptureForLastSourceCodec(nativeScreenShareCodec: VideoCodec): boolean { - const preferredScreenShareCodecPreference = VoiceSettings.getPreferredScreenShareCodec(); - return ( - preferredScreenShareCodecPreference === 'auto' || - shouldUseNativeScreenCaptureForScreenShareCodec(nativeScreenShareCodec) - ); -} - -async function findNativeCaptureSourceForLastDesktopSource( - source: DesktopSource, - kind: LastScreenShareSourceKind, -): Promise { - const nativeScreenShareCodec = ScreenShareCodecNegotiation.selectNativeScreenShareCodec( - VoiceSettings.getPreferredScreenShareCodec(), - ); - if (!canUseNativeCaptureForLastSourceCodec(nativeScreenShareCodec)) return undefined; - if (kind !== 'app' && kind !== 'display') return undefined; - if (!(await isNativeScreenCaptureAvailable().catch(() => false))) return undefined; - const nativeSourceApi = getNativeScreenCaptureApi(); - if (!nativeSourceApi) return undefined; - const nativeSources = await nativeSourceApi.listSources().catch((error) => { - logger.warn('Failed to list native sources for last screen-share source', {error}); - return []; - }); - return findNativeCaptureSourceForDesktopSource(source, nativeSources); -} - async function tryStartLastDesktopScreenShareSource(lastSource: LastScreenShareSource): Promise { const preload = await preloadScreenSharePickerSources(); if (usesNativeDisplaySharePicker(preload.displayShareEnvironment)) return false; @@ -340,25 +300,11 @@ async function tryStartLastDesktopScreenShareSource(lastSource: LastScreenShareS const source = findLastDesktopSource(lastSource, desktopSources); if (!source) return false; const preferredDisplaySurface = lastSource.kind === 'app' ? 'window' : 'monitor'; - const nativeSource = await findNativeCaptureSourceForLastDesktopSource(source, lastSource.kind); - let didStart: boolean; - if (nativeSource) { - const preferredScreenShareCodecPreference = VoiceSettings.getPreferredScreenShareCodec(); - const nativeScreenShareCodec = ScreenShareCodecNegotiation.selectNativeScreenShareCodec( - preferredScreenShareCodecPreference, - ); - didStart = await startConfiguredNativeDisplayScreenShare(nativeSource, { - desktopSourceId: source.id, - isOwnWindow: source.isOwnWindow, - ...(preferredScreenShareCodecPreference !== 'auto' ? {videoCodec: nativeScreenShareCodec} : {}), - }); - } else { - didStart = await startConfiguredDisplayScreenShare(source.id, { - sourceDimensions: getDesktopSourceDimensions(source), - preferredDisplaySurface, - isOwnWindow: source.isOwnWindow === true, - }); - } + const didStart = await startConfiguredDisplayScreenShare(source.id, { + sourceDimensions: getDesktopSourceDimensions(source), + preferredDisplaySurface, + isOwnWindow: source.isOwnWindow === true, + }); if (didStart) { recordLastScreenShareSource(lastSource.kind, source.id, source.name || lastSource.title); } @@ -374,24 +320,6 @@ async function tryStartLastDeviceScreenShareSource(lastSource: LastScreenShareSo return didStart; } -async function tryStartLastGameScreenShareSource(lastSource: LastScreenShareSource): Promise { - const electronApi = getElectronAPI(); - if (electronApi?.platform !== 'linux') return false; - if (!(await isNativeScreenCaptureAvailable().catch(() => false))) return false; - const source: NativeScreenCaptureSource = { - kind: 'game', - id: lastSource.sourceId ?? 'obs-vkcapture', - name: lastSource.title, - width: 1920, - height: 1080, - }; - const didStart = await startConfiguredNativeDisplayScreenShare(source); - if (didStart) { - recordLastScreenShareSource('game', source.id, source.name); - } - return didStart; -} - export async function tryStartLastScreenShareSource(): Promise { const lastSource = VoiceSettings.getLastScreenShareSource(); if (!lastSource) return false; @@ -400,9 +328,6 @@ export async function tryStartLastScreenShareSource(): Promise { if (lastSource.kind === 'device') { return await tryStartLastDeviceScreenShareSource(lastSource); } - if (lastSource.kind === 'game') { - return await tryStartLastGameScreenShareSource(lastSource); - } return await tryStartLastDesktopScreenShareSource(lastSource); } catch (error) { logger.warn('Failed to start last screen-share source', {error, kind: lastSource.kind}); @@ -772,10 +697,6 @@ const ScreenSharePickerModalLoadedContent = observer( const [pendingSelectionId, setPendingSelectionId] = useState(null); const [invalidThumbnailIds, setInvalidThumbnailIds] = useState>(() => new Set()); const [nativeAudioAvailability, setNativeAudioAvailability] = useState(null); - const [nativeScreenAvailable, setNativeScreenAvailable] = useState(null); - const [nativeSources, setNativeSources] = useState>([]); - const [hasLoadedNativeSources, setHasLoadedNativeSources] = useState(false); - const nativeSourcesRef = useRef>([]); const loadRequestIdRef = useRef(0); const desktopSourcesRef = useRef(desktopSources); const desktopSourceRefreshInFlightRef = useRef(false); @@ -784,9 +705,6 @@ const ScreenSharePickerModalLoadedContent = observer( useEffect(() => { desktopSourcesRef.current = desktopSources; }, [desktopSources]); - useEffect(() => { - nativeSourcesRef.current = nativeSources; - }, [nativeSources]); useEffect(() => { let cancelled = false; void getNativeAudioAvailabilityCached().then((availability) => { @@ -796,53 +714,6 @@ const ScreenSharePickerModalLoadedContent = observer( cancelled = true; }; }, []); - useEffect(() => { - let cancelled = false; - const platform = getElectronAPI()?.platform; - const canMapDesktopSourcesToNativeCapture = platform === 'darwin' || platform === 'win32'; - const canProbeNativeCaptureWithoutSourceList = platform === 'linux'; - if (!canMapDesktopSourcesToNativeCapture && !canProbeNativeCaptureWithoutSourceList) { - setNativeScreenAvailable(false); - setNativeSources([]); - setHasLoadedNativeSources(true); - return () => { - cancelled = true; - }; - } - void isNativeScreenCaptureAvailable().then((available) => { - if (cancelled) return; - setNativeScreenAvailable(available); - if (!available) { - setHasLoadedNativeSources(true); - return; - } - if (usesNativeDisplayPicker || !canMapDesktopSourcesToNativeCapture) { - setNativeSources([]); - setHasLoadedNativeSources(true); - return; - } - const api = getNativeScreenCaptureApi(); - if (!api) { - setHasLoadedNativeSources(true); - return; - } - api - .listSources() - .then((sources) => { - if (cancelled) return; - setNativeSources(sources); - setHasLoadedNativeSources(true); - }) - .catch((error) => { - logger.warn('Failed to load native screen capture source list', {error}); - if (cancelled) return; - setHasLoadedNativeSources(true); - }); - }); - return () => { - cancelled = true; - }; - }, [usesNativeDisplayPicker]); const platform = getElectronAPI()?.platform; const displayPermission = useScreenSharePickerDisplayPermission({ activeTab, @@ -968,15 +839,6 @@ const ScreenSharePickerModalLoadedContent = observer( }, [], ); - const canUseNativeCapture = nativeScreenAvailable === true && hasLoadedNativeSources; - const preferredScreenShareCodecPreference = VoiceSettings.getPreferredScreenShareCodec(); - const nativeScreenShareCodec = useMemo( - () => ScreenShareCodecNegotiation.selectNativeScreenShareCodec(preferredScreenShareCodecPreference), - [preferredScreenShareCodecPreference], - ); - const canUseNativeCaptureForCodec = - preferredScreenShareCodecPreference === 'auto' || - shouldUseNativeScreenCaptureForScreenShareCodec(nativeScreenShareCodec); const activeDesktopSourceThumbnailStateKey = useMemo(() => { if (activeTab === 'devices') { return null; @@ -1100,41 +962,7 @@ const ScreenSharePickerModalLoadedContent = observer( } setPendingSelectionId(cardId); try { - if (cardId === LINUX_GAME_CAPTURE_SELECTION_ID) { - const source: NativeScreenCaptureSource = { - kind: 'game', - id: 'obs-vkcapture', - name: i18n._(GAME_CAPTURE_DESCRIPTOR), - width: 1920, - height: 1080, - }; - const didSelect = - mode === 'switch' - ? await switchConfiguredNativeDisplayScreenShare(source) - : await startConfiguredNativeDisplayScreenShare(source); - if (didSelect) { - recordLastScreenShareSource('game', source.id, source.name); - ModalCommands.pop(); - } - return; - } const selectedSource = desktopSourcesRef.current.find((source) => source.id === cardId); - const canUseNativeSource = - canUseNativeCaptureForCodec && - canUseNativeCapture && - selectedSource != null && - ((activeTab === 'apps' && isWindowSource(selectedSource)) || - (activeTab === 'displays' && isDisplaySource(selectedSource))); - const nativeSource = canUseNativeSource - ? findNativeCaptureSourceForDesktopSource(selectedSource, nativeSourcesRef.current) - : undefined; - const nativeSelectionOptions = selectedSource - ? { - desktopSourceId: selectedSource.id, - isOwnWindow: selectedSource.isOwnWindow, - ...(preferredScreenShareCodecPreference !== 'auto' ? {videoCodec: nativeScreenShareCodec} : {}), - } - : undefined; const sourceDimensions = selectedSource?.nativeWidth && selectedSource.nativeHeight ? {width: selectedSource.nativeWidth, height: selectedSource.nativeHeight} @@ -1145,11 +973,6 @@ const ScreenSharePickerModalLoadedContent = observer( mode === 'switch' ? await switchConfiguredDeviceScreenShare(cardId) : await startConfiguredDeviceScreenShare(cardId); - } else if (nativeSource) { - didSelect = - mode === 'switch' - ? await switchConfiguredNativeDisplayScreenShare(nativeSource, nativeSelectionOptions) - : await startConfiguredNativeDisplayScreenShare(nativeSource, nativeSelectionOptions); } else { const selectedDisplaySourceId = usesNativeDisplayPicker ? null : cardId; const preferredDisplaySurface: 'window' | 'monitor' | undefined = @@ -1191,19 +1014,7 @@ const ScreenSharePickerModalLoadedContent = observer( setPendingSelectionId(null); } }, - [ - activeTab, - canUseNativeCapture, - canUseNativeCaptureForCodec, - i18n, - loadDesktopSources, - mode, - nativeScreenShareCodec, - pendingSelectionId, - platform, - tabCards, - usesNativeDisplayPicker, - ], + [activeTab, loadDesktopSources, mode, pendingSelectionId, platform, tabCards, usesNativeDisplayPicker], ); const handleSettingsClick = useCallback( (event: ReactMouseEvent) => { @@ -1255,9 +1066,6 @@ const ScreenSharePickerModalLoadedContent = observer( ? i18n._(OPEN_BROWSER_PICKER_DESCRIPTOR) : i18n._(OPEN_SYSTEM_PICKER_DESCRIPTOR); const nativeDisplayPending = pendingSelectionId === NATIVE_DISPLAY_SELECTION_ID; - const linuxGameCapturePending = pendingSelectionId === LINUX_GAME_CAPTURE_SELECTION_ID; - const showLinuxGameCaptureAction = - platform === 'linux' && activeTab === 'displays' && nativeScreenAvailable === true && canUseNativeCaptureForCodec; const nativePickerCopy = useNativePickerCopy(activeTab, displayShareEnvironment); const deviceEmptyStateCopy = useDeviceEmptyStateCopy(displayShareEnvironment); return ( @@ -1300,13 +1108,6 @@ const ScreenSharePickerModalLoadedContent = observer( pickerActionLabel={pickerActionLabel} onPickerAction={() => void handleStartSelection(NATIVE_DISPLAY_SELECTION_ID)} pickerActionPending={nativeDisplayPending} - secondaryActionLabel={showLinuxGameCaptureAction ? i18n._(GAME_CAPTURE_DESCRIPTOR) : undefined} - onSecondaryAction={ - showLinuxGameCaptureAction - ? () => void handleStartSelection(LINUX_GAME_CAPTURE_SELECTION_ID) - : undefined - } - secondaryActionPending={linuxGameCapturePending} showDesktopDownloadCta={showDesktopDownloadCta} data-flx="voice.screen-share-picker-modal.screen-share-picker-modal-loaded-content.native-display-picker-state" /> diff --git a/fluxer_app/src/features/voice/components/pip_overlay/PiPOverlayInner.tsx b/fluxer_app/src/features/voice/components/pip_overlay/PiPOverlayInner.tsx index 72d326c9a..b2f17e6aa 100644 --- a/fluxer_app/src/features/voice/components/pip_overlay/PiPOverlayInner.tsx +++ b/fluxer_app/src/features/voice/components/pip_overlay/PiPOverlayInner.tsx @@ -36,13 +36,8 @@ import { import {useFindTrackRef} from '@app/features/voice/components/pip_overlay/useFindTrackRef'; import {getStreamKey} from '@app/features/voice/components/StreamKeys'; import {useStreamSpectators} from '@app/features/voice/components/useStreamSpectators'; -import {buildNativeVoiceCallTrackRefForParticipant} from '@app/features/voice/components/useVoiceCallTracksAndLayout'; import {VoiceParticipantTile} from '@app/features/voice/components/VoiceParticipantTile'; import MediaEngine, {useMediaEngineVersion} from '@app/features/voice/engine/MediaEngineFacade'; -import NativeVideoTileManager from '@app/features/voice/engine/native_voice_engine/NativeVideoTileManager'; -import {useStoreVersion} from '@app/features/voice/engine/Store'; -import {isVoiceEngineV2NativeProjectionActiveFromMediaEngine} from '@app/features/voice/engine/VoiceMediaEngineBridge'; -import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource'; import {VOICE_DISCONNECT_DESCRIPTOR} from '@app/features/voice/utils/VoiceMessageDescriptors'; import {ME} from '@fluxer/constants/src/AppConstants'; import {msg} from '@lingui/core/macro'; @@ -51,7 +46,7 @@ import {isTrackReference, type TrackReferenceOrPlaceholder} from '@livekit/compo import {clsx} from 'clsx'; import type {AnimationPlaybackControls} from 'framer-motion'; import {animate, motion, useMotionValue} from 'framer-motion'; -import type {Room, Track} from 'livekit-client'; +import type {Room} from 'livekit-client'; import {observer} from 'mobx-react-lite'; import type React from 'react'; import {useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState} from 'react'; @@ -123,26 +118,9 @@ interface PiPOverlayInnerBaseProps extends PiPOverlayInnerProps { const PiPOverlayInnerBase = observer(function PiPOverlayInnerBase({content, room, trackRef}: PiPOverlayInnerBaseProps) { const {i18n} = useLingui(); useMediaEngineVersion(); - useStoreVersion(NativeVideoTileManager); const corner = PiP.getEffectiveCorner(); const isScreenShare = content.type === 'stream'; - const isNativeEngine = isVoiceEngineV2NativeProjectionActiveFromMediaEngine(); - const nativeTrackRef = useMemo(() => { - if (!isNativeEngine) return null; - return buildNativeVoiceCallTrackRefForParticipant({ - participantIdentity: content.participantIdentity, - source: (isScreenShare ? VoiceTrackSource.ScreenShare : VoiceTrackSource.Camera) as Track.Source, - participantSnapshots: MediaEngine.participants, - nativeTracks: NativeVideoTileManager.tracks, - }); - }, [ - content.participantIdentity, - isNativeEngine, - isScreenShare, - MediaEngine.participants, - NativeVideoTileManager.tracks, - ]); - const effectiveTrackRef = trackRef ?? nativeTrackRef; + const effectiveTrackRef = trackRef; const trackRefSummary = useMemo(() => { if (!effectiveTrackRef) return null; const summary = { @@ -621,9 +599,8 @@ const PiPOverlayInnerBase = observer(function PiPOverlayInnerBase({content, room roomLocalParticipantIdentity: room?.localParticipant.identity ?? null, remoteParticipantCount: room?.remoteParticipants.size ?? 0, trackRef: trackRefSummary, - usesNativeTrackRef: trackRef == null && nativeTrackRef != null, }); - }, [content, nativeTrackRef, room, trackRef, trackRefSummary]); + }, [content, room, trackRef, trackRefSummary]); return ( { - if (!isNativeEngine) return null; - return buildNativeVoiceCallTrackRefForParticipant({ - participantIdentity: descriptor.participantIdentity, - source: getDescriptorTrackSource(descriptor), - participantSnapshots, - nativeTracks, - }); - }, [descriptor, isNativeEngine, participantSnapshots, nativeTracks]); - const effectiveTrackRef = trackRef ?? nativeTrackRef; const connectionParticipant = MediaEngine.getParticipantByUserIdAndConnectionId( descriptor.userId, descriptor.connectionId, @@ -108,9 +91,9 @@ const VoiceTilePopoutContentBase = observer(function VoiceTilePopoutContentBase( }, [isTrackLive, descriptor.key]); return (
- {effectiveTrackRef && ( + {trackRef && ( ; } @@ -69,7 +68,6 @@ export function useScreenShareWatchFailure({ participantIdentity, participantSid, trackSid, - hasNativeFrame, operationKey, videoRef, }: UseScreenShareWatchFailureOptions): ScreenShareWatchFailureState { @@ -107,7 +105,7 @@ export function useScreenShareWatchFailure({ if (!attemptEnabled || !attemptKey) return; if (isOperationBuffering) return; if (hasRenderedVideoFrame) return; - if (hasNativeFrame || videoElementHasRenderedFrame(videoRef.current)) { + if (videoElementHasRenderedFrame(videoRef.current)) { ScreenShareWatchFailures.markRenderedVideoFrame(target, attemptKey); return; } @@ -117,7 +115,7 @@ export function useScreenShareWatchFailure({ ScreenShareWatchFailures.markRenderedVideoFrame(target, attemptKey); }, }); - }, [attemptEnabled, attemptKey, hasNativeFrame, hasRenderedVideoFrame, isOperationBuffering, target, videoRef]); + }, [attemptEnabled, attemptKey, hasRenderedVideoFrame, isOperationBuffering, target, videoRef]); return { failure, diff --git a/fluxer_app/src/features/voice/components/useStatsForNerds.ts b/fluxer_app/src/features/voice/components/useStatsForNerds.ts index fcea56288..37c68f381 100644 --- a/fluxer_app/src/features/voice/components/useStatsForNerds.ts +++ b/fluxer_app/src/features/voice/components/useStatsForNerds.ts @@ -9,11 +9,9 @@ import type {AppMetricsSnapshot, DesktopInfo, GpuInfo} from '@app/features/platf import {getElectronAPI} from '@app/features/ui/utils/NativeUtils'; import AdaptiveScreenShareEngine from '@app/features/voice/engine/AdaptiveScreenShareEngine'; import MediaEngine, {useMediaEngineVersion} from '@app/features/voice/engine/MediaEngineFacade'; -import NativeVoiceStatsStore from '@app/features/voice/engine/native_voice_engine/NativeVoiceStatsStore'; import ScreenShareCodecNegotiation, { getScreenShareCodecPreferenceOrder, } from '@app/features/voice/engine/ScreenShareCodecNegotiation'; -import {isVoiceEngineV2NativeProjectionActiveFromMediaEngine} from '@app/features/voice/engine/VoiceMediaEngineBridge'; import {getScreenShareAudioPumpDiagnostics} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump'; import VoiceSettings from '@app/features/voice/state/VoiceSettings'; import {getNativeAudioCaptureDiagnosticState} from '@app/features/voice/utils/NativeAudioCaptureBridge'; @@ -187,7 +185,7 @@ export function useStatsForNerds({enabled = true}: UseStatsForNerdsOptions = {}) stats: MediaEngine.voiceStats, perTrackStats: MediaEngine.perTrackStats, statsTimeSeries: MediaEngine.statsTimeSeries, - nativeStats: isVoiceEngineV2NativeProjectionActiveFromMediaEngine() ? NativeVoiceStatsStore.stats : null, + nativeStats: null, publisherTransport: MediaEngine.publisherTransport, subscriberTransport: MediaEngine.subscriberTransport, localParticipant, diff --git a/fluxer_app/src/features/voice/components/useStreamTrackInfo.test.ts b/fluxer_app/src/features/voice/components/useStreamTrackInfo.test.ts index ade7b563c..ed4776489 100644 --- a/fluxer_app/src/features/voice/components/useStreamTrackInfo.test.ts +++ b/fluxer_app/src/features/voice/components/useStreamTrackInfo.test.ts @@ -8,24 +8,45 @@ import { import {createVoiceMediaGraphSnapshot, transitionVoiceMediaGraph} from '@app/features/voice/engine/VoiceMediaGraph'; import { type VoiceMediaGraphStatsView, - voiceMediaGraphStatsObservationsFromNativeStats, voiceMediaGraphStatsObservationsFromPerTrackStats, } from '@app/features/voice/engine/VoiceMediaGraphStats'; +import type {VoiceMediaGraphStatsTrackObservation} from '@app/features/voice/engine/VoiceMediaGraphStatsObservations'; import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource'; -import type {VoiceEngineV2PerTrackStats, VoiceEngineV2Stats} from '@fluxer/voice_engine_v2'; +import type {VoiceEngineV2PerTrackStats} from '@fluxer/voice_engine_v2'; import {describe, expect, it} from 'vitest'; -function nativeStats(overrides: Partial): VoiceEngineV2Stats { - return {rttMs: null, outbound: [], inbound: [], ...overrides}; +function observation(overrides: Partial): VoiceMediaGraphStatsTrackObservation { + return { + trackSid: null, + trackIdentifier: null, + mediaSourceId: null, + mid: null, + rid: null, + ssrc: null, + participantIdentity: null, + participantSid: null, + source: null, + direction: 'recv', + kind: 'video', + fps: null, + width: null, + height: null, + sourceFps: null, + sourceWidth: null, + sourceHeight: null, + ...overrides, + }; } -function graphViewWithNativeStats(stats: VoiceEngineV2Stats): VoiceMediaGraphStatsView { +function graphViewWithObservations( + tracks: ReadonlyArray, +): VoiceMediaGraphStatsView { return transitionVoiceMediaGraph(createVoiceMediaGraphSnapshot(), { type: 'stats.observed', at: 1000, connectionId: 'conn-1', - platform: 'native', - tracks: voiceMediaGraphStatsObservationsFromNativeStats(stats), + platform: 'web', + tracks, }); } @@ -60,24 +81,17 @@ describe('mergeStreamTrackInfo', () => { }); describe('resolveStreamTrackStatsInfo', () => { - it('resolves native screen-share info from the graph by trackSid', () => { - const view = graphViewWithNativeStats( - nativeStats({ - outbound: [ - { - trackSid: 'TR_screen', - source: 'screen_share', - kind: 'video', - bitrateKbps: 2000, - packetsLost: 0, - width: 2560, - height: 1440, - fps: 30, - effectiveFps: 59.6, - }, - ], + it('resolves screen-share info from the graph by trackSid', () => { + const view = graphViewWithObservations([ + observation({ + trackSid: 'TR_screen', + source: 'screen_share', + direction: 'send', + width: 2560, + height: 1440, + fps: 59.6, }), - ); + ]); expect( resolveStreamTrackStatsInfo(view, { @@ -88,25 +102,18 @@ describe('resolveStreamTrackStatsInfo', () => { ).toEqual({width: 2560, height: 1440, fps: 59.6}); }); - it('resolves remote native screen-share info via the participantIdentity fallback', () => { - const view = graphViewWithNativeStats( - nativeStats({ - inbound: [ - { - participantSid: 'PA_1', - participantIdentity: 'user_2_connection_2', - trackSid: 'TR_remote_screen', - source: 'screen_share', - kind: 'video', - bitrateKbps: 4200, - packetsLost: 0, - width: 3840, - height: 2160, - fps: 24.7, - }, - ], + it('resolves remote screen-share info via the participantIdentity fallback', () => { + const view = graphViewWithObservations([ + observation({ + participantSid: 'PA_1', + participantIdentity: 'user_2_connection_2', + trackSid: 'TR_remote_screen', + source: 'screen_share', + width: 3840, + height: 2160, + fps: 24.7, }), - ); + ]); expect( resolveStreamTrackStatsInfo(view, { diff --git a/fluxer_app/src/features/voice/components/useVoiceCallTracksAndLayout.ts b/fluxer_app/src/features/voice/components/useVoiceCallTracksAndLayout.ts index 4e9359604..46dd79903 100644 --- a/fluxer_app/src/features/voice/components/useVoiceCallTracksAndLayout.ts +++ b/fluxer_app/src/features/voice/components/useVoiceCallTracksAndLayout.ts @@ -24,24 +24,12 @@ import { syncVoiceParticipantSortSnapshot, } from '@app/features/voice/components/VoiceParticipantSortUtils'; import MediaEngine, {useMediaEngineVersion} from '@app/features/voice/engine/MediaEngineFacade'; -import NativeVideoTileManager, { - type NativeInboundVideoTrack, -} from '@app/features/voice/engine/native_voice_engine/NativeVideoTileManager'; import ScreenSharePublicationMigration from '@app/features/voice/engine/ScreenSharePublicationMigration'; -import type {VoiceGatewayConnectionVoiceStates} from '@app/features/voice/engine/VoiceGatewayStateMachine'; -import {isVoiceEngineV2NativeProjectionActiveFromMediaEngine} from '@app/features/voice/engine/VoiceMediaEngineBridge'; import {selectVoiceMediaGraphViewerStreamKeys} from '@app/features/voice/engine/VoiceMediaGraph'; import {voiceMediaGraphStore} from '@app/features/voice/engine/VoiceMediaGraphStore'; import {pruneInactiveWatchedStreamsForChannel} from '@app/features/voice/engine/VoiceStreamWatchState'; -import { - asVoiceTrackSource, - isVoiceScreenShareSource, - VoiceTrackSource, -} from '@app/features/voice/engine/VoiceTrackSource'; -import { - isVoiceEngineV2AppParticipantSpeaking, - type VoiceEngineV2AppParticipantSnapshot, -} from '@app/features/voice/engine/v2/VoiceEngineV2AppSelectors'; +import {asVoiceTrackSource, VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource'; +import {isVoiceEngineV2AppParticipantSpeaking} from '@app/features/voice/engine/v2/VoiceEngineV2AppSelectors'; import CallMediaPrefs from '@app/features/voice/state/CallMediaPrefs'; import LocalVoiceState from '@app/features/voice/state/LocalVoiceState'; import VoiceCallLayout from '@app/features/voice/state/VoiceCallLayout'; @@ -51,7 +39,7 @@ import { parseVoiceParticipantIdentity, } from '@app/features/voice/utils/VoiceParticipantIdentity'; import {isTrackReference, type TrackReferenceOrPlaceholder, useMaybeRoomContext} from '@livekit/components-react'; -import {Participant, type Room, RoomEvent, Track, TrackPublication, type VideoQuality} from 'livekit-client'; +import {type Participant, type Room, RoomEvent, type Track} from 'livekit-client'; import {useCallback, useEffect, useMemo, useRef, useState} from 'react'; interface UseVoiceCallTracksAndLayoutArgs { @@ -72,7 +60,6 @@ interface VoiceGridTrackActivityState { const VOICE_GRID_SPEAKER_PRIORITY_HOLD_MS = 4500; const CAMERA_SOURCE = VoiceTrackSource.Camera as Track.Source; const SCREEN_SHARE_SOURCE = VoiceTrackSource.ScreenShare as Track.Source; -const SCREEN_SHARE_AUDIO_SOURCE = VoiceTrackSource.ScreenShareAudio as Track.Source; interface VoiceCallTrackSourceOption { source: Track.Source; @@ -105,72 +92,6 @@ const LIVEKIT_TRACK_UPDATE_EVENTS: ReadonlyArray = [ RoomEvent.ActiveSpeakersChanged, ]; -class NativeVoiceTrackPublication extends TrackPublication { - private subscribed: boolean; - private desired: boolean; - private enabled: boolean; - private readonly local: boolean; - private requestedVideoQuality: VideoQuality | undefined; - - constructor({ - trackSid, - source, - kind, - isMuted, - isLocal, - dimensions, - }: { - trackSid: string; - source: Track.Source; - kind: Track.Kind; - isMuted: boolean; - isLocal: boolean; - dimensions?: Track.Dimensions; - }) { - super(kind, trackSid, trackSid); - this.source = source; - this.metadataMuted = isMuted; - this.subscribed = !isMuted; - this.desired = !isMuted; - this.enabled = !isMuted; - this.local = isLocal; - this.dimensions = dimensions; - } - - override get isLocal(): boolean { - return this.local; - } - - override get isSubscribed(): boolean { - return this.subscribed; - } - - get isDesired(): boolean { - return this.desired; - } - - override get isEnabled(): boolean { - return this.enabled; - } - - setSubscribed(subscribed: boolean): void { - this.desired = subscribed; - this.subscribed = subscribed; - } - - setEnabled(enabled: boolean): void { - this.enabled = enabled; - } - - setVideoQuality(quality: VideoQuality): void { - this.requestedVideoQuality = quality; - } - - emitTrackUpdate(): void { - void this.requestedVideoQuality; - } -} - function difference(setA: Set, setB: Set): Set { const result = new Set(setA); for (const value of setB) { @@ -276,213 +197,6 @@ function useOptionalLiveKitVoiceCallTrackRefs( return useMemo(() => ({tracks, participants: state.participants}), [tracks, state.participants]); } -function getNativeTracksByParticipantSid( - nativeTracks: Readonly>, -): Map> { - const map = new Map>(); - const addTrack = (key: string | undefined, track: NativeInboundVideoTrack): void => { - if (!key) return; - const current = map.get(key) ?? []; - if (!current.includes(track)) { - current.push(track); - } - map.set(key, current); - }; - for (const track of Object.values(nativeTracks)) { - addTrack(track.participantSid, track); - addTrack(track.participantIdentity, track); - } - return map; -} - -function getNativeTracksForParticipantSnapshot( - nativeTracksByParticipantSid: Map>, - snapshot: VoiceEngineV2AppParticipantSnapshot, -): Array { - const tracks: Array = []; - for (const key of [snapshot.sid, snapshot.identity]) { - if (!key) continue; - for (const track of nativeTracksByParticipantSid.get(key) ?? []) { - if (!tracks.includes(track)) { - tracks.push(track); - } - } - } - return tracks; -} - -function selectNativeVideoTrack( - tracks: ReadonlyArray, - source: Track.Source, -): NativeInboundVideoTrack | undefined { - const wantScreenShare = asVoiceTrackSource(source) === VoiceTrackSource.ScreenShare; - return tracks.find((track) => - wantScreenShare ? isVoiceScreenShareSource(track.source) : !isVoiceScreenShareSource(track.source), - ); -} - -function createNativePublication({ - participantSid, - source, - isMuted, - isLocal, - nativeTrack, -}: { - participantSid: string; - source: Track.Source; - isMuted: boolean; - isLocal: boolean; - nativeTrack?: NativeInboundVideoTrack; -}): NativeVoiceTrackPublication { - const isVideo = source === CAMERA_SOURCE || source === SCREEN_SHARE_SOURCE; - return new NativeVoiceTrackPublication({ - trackSid: nativeTrack?.trackSid ?? `${participantSid}:${source}`, - source, - kind: isVideo ? Track.Kind.Video : Track.Kind.Audio, - isMuted, - isLocal, - dimensions: - nativeTrack && nativeTrack.width > 0 && nativeTrack.height > 0 - ? {width: nativeTrack.width, height: nativeTrack.height} - : undefined, - }); -} - -function createNativeParticipant( - snapshot: VoiceEngineV2AppParticipantSnapshot, - nativeTracks: ReadonlyArray, -): Participant { - const participant = new Participant( - snapshot.sid || snapshot.identity, - snapshot.identity, - undefined, - snapshot.metadata, - {...snapshot.attributes}, - ); - Object.defineProperty(participant, 'isLocal', { - configurable: true, - get: () => snapshot.isLocal, - }); - participant.isSpeaking = Boolean(snapshot.isSpeaking || snapshot.isAudioLevelSpeaking); - participant.audioLevel = snapshot.isAudioLevelSpeaking ? 1 : 0; - if (snapshot.lastSpokeAt) { - participant.lastSpokeAt = new Date(snapshot.lastSpokeAt); - } - participant.addTrackPublication( - createNativePublication({ - participantSid: participant.sid, - source: VoiceTrackSource.Microphone as Track.Source, - isMuted: !snapshot.isMicrophoneEnabled, - isLocal: snapshot.isLocal, - }), - ); - const cameraTrack = selectNativeVideoTrack(nativeTracks, CAMERA_SOURCE); - if (snapshot.isCameraEnabled || cameraTrack) { - participant.addTrackPublication( - createNativePublication({ - participantSid: participant.sid, - source: CAMERA_SOURCE, - isMuted: !snapshot.isCameraEnabled && !cameraTrack, - isLocal: snapshot.isLocal, - nativeTrack: cameraTrack, - }), - ); - } - const screenShareTrack = selectNativeVideoTrack(nativeTracks, SCREEN_SHARE_SOURCE); - if (snapshot.isScreenShareEnabled || screenShareTrack) { - participant.addTrackPublication( - createNativePublication({ - participantSid: participant.sid, - source: SCREEN_SHARE_SOURCE, - isMuted: !snapshot.isScreenShareEnabled && !screenShareTrack, - isLocal: snapshot.isLocal, - nativeTrack: screenShareTrack, - }), - ); - } - if (snapshot.isScreenShareAudioEnabled) { - participant.addTrackPublication( - createNativePublication({ - participantSid: participant.sid, - source: SCREEN_SHARE_AUDIO_SOURCE, - isMuted: false, - isLocal: snapshot.isLocal, - }), - ); - } - return participant; -} - -function isNativeSnapshotInChannel( - snapshot: VoiceEngineV2AppParticipantSnapshot, - connectionVoiceStates: VoiceGatewayConnectionVoiceStates, - channel: Channel, -): boolean { - const connectionId = snapshot.connectionId; - if (!connectionId) return true; - const voiceState = connectionVoiceStates[connectionId]; - if (!voiceState) return true; - return voiceState.channel_id === channel.id; -} - -function buildNativeVoiceCallTrackRefs({ - channel, - participantSnapshots, - nativeTracks, - connectionVoiceStates, -}: { - channel: Channel; - participantSnapshots: Readonly>; - nativeTracks: Readonly>; - connectionVoiceStates: VoiceGatewayConnectionVoiceStates; -}): VoiceCallTrackRefsState { - const nativeTracksByParticipantSid = getNativeTracksByParticipantSid(nativeTracks); - const participants = Object.values(participantSnapshots) - .filter((snapshot) => isNativeSnapshotInChannel(snapshot, connectionVoiceStates, channel)) - .map((snapshot) => - createNativeParticipant(snapshot, getNativeTracksForParticipantSnapshot(nativeTracksByParticipantSid, snapshot)), - ); - const tracks: Array = []; - for (const participant of participants) { - const cameraPublication = participant.getTrackPublication(CAMERA_SOURCE); - if (cameraPublication) { - tracks.push({participant, publication: cameraPublication, source: CAMERA_SOURCE}); - } else { - tracks.push({participant, source: CAMERA_SOURCE}); - } - const screenSharePublication = participant.getTrackPublication(SCREEN_SHARE_SOURCE); - if (screenSharePublication && !screenSharePublication.isMuted) { - tracks.push({participant, publication: screenSharePublication, source: SCREEN_SHARE_SOURCE}); - } - } - return {tracks, participants}; -} - -export function buildNativeVoiceCallTrackRefForParticipant({ - participantIdentity, - source, - participantSnapshots, - nativeTracks, -}: { - participantIdentity: string; - source: Track.Source; - participantSnapshots: Readonly>; - nativeTracks: Readonly>; -}): TrackReferenceOrPlaceholder | null { - const snapshot = participantSnapshots[participantIdentity]; - if (!snapshot) return null; - const nativeTracksByParticipantSid = getNativeTracksByParticipantSid(nativeTracks); - const participant = createNativeParticipant( - snapshot, - getNativeTracksForParticipantSnapshot(nativeTracksByParticipantSid, snapshot), - ); - const publication = participant.getTrackPublication(source); - if (publication) { - return {participant, publication, source}; - } - return {participant, source}; -} - function getTrackActivityKey(trackRef: TrackReferenceOrPlaceholder): string { return `${trackRef.participant.identity}:${trackRef.source}`; } @@ -505,22 +219,10 @@ export function useVoiceCallTracksAndLayout({channel, expandedUserIds}: UseVoice [channel.guildId, channel.id], ); const liveKitTrackRefs = useOptionalLiveKitVoiceCallTrackRefs(VOICE_CALL_TRACK_SOURCES, {onlySubscribed: false}); - const nativeInboundVideoTracks = NativeVideoTileManager.tracks; - const isNativeEngine = isVoiceEngineV2NativeProjectionActiveFromMediaEngine(); const connectionVoiceStates = MediaEngine.connectionVoiceStates; const participantSnapshots = MediaEngine.participants; - const nativeTrackRefs = useMemo( - () => - buildNativeVoiceCallTrackRefs({ - channel, - participantSnapshots, - nativeTracks: nativeInboundVideoTracks, - connectionVoiceStates: connectionVoiceStates as VoiceGatewayConnectionVoiceStates, - }), - [channel, participantSnapshots, nativeInboundVideoTracks, connectionVoiceStates], - ); - const tracks = isNativeEngine ? nativeTrackRefs.tracks : liveKitTrackRefs.tracks; - const participants = isNativeEngine ? nativeTrackRefs.participants : liveKitTrackRefs.participants; + const tracks = liveKitTrackRefs.tracks; + const participants = liveKitTrackRefs.participants; const userCacheSize = Users.usersList.length; const participantCount = useMemo(() => countKnownVoiceParticipants(participants), [participants, userCacheSize]); const renderableTracks = useMemo(() => tracks.filter(isKnownVoiceTrackRef), [tracks, userCacheSize]); diff --git a/fluxer_app/src/features/voice/components/voice_participant_tile/LastFrameSnapshotCache.ts b/fluxer_app/src/features/voice/components/voice_participant_tile/LastFrameSnapshotCache.ts index 34e5d5cc8..0d208a151 100644 --- a/fluxer_app/src/features/voice/components/voice_participant_tile/LastFrameSnapshotCache.ts +++ b/fluxer_app/src/features/voice/components/voice_participant_tile/LastFrameSnapshotCache.ts @@ -1,10 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import {videoElementHasRenderedFrame} from '@app/features/voice/components/VideoElementFrameState'; -import {i420VideoFrameLayout} from '@app/features/voice/engine/native_voice_engine/createInboundVideoBridge'; -import type {NativeRetainedVideoFrame} from '@app/features/voice/engine/native_voice_engine/NativeVideoTileManager'; import {Store} from '@app/features/voice/engine/Store'; -import {getVideoFrameCtor} from '@app/features/voice/utils/native_screen_capture_bridge/shared'; export const LAST_FRAME_SNAPSHOTS_MAX = 8; export const LAST_FRAME_SNAPSHOT_WIDTH_MAX = 1280; @@ -73,31 +70,6 @@ class LastFrameSnapshotCache extends Store { this.retainSnapshot(key, dataUrl); } - captureFromNativeFrame(key: string, frame: NativeRetainedVideoFrame | undefined): void { - if (!key || !frame) return; - const VideoFrameImpl = getVideoFrameCtor(); - if (!VideoFrameImpl) return; - try { - const videoFrame = new VideoFrameImpl(new Uint8Array(frame.data), { - format: 'I420', - codedWidth: frame.width, - codedHeight: frame.height, - timestamp: frame.timestampUs, - layout: i420VideoFrameLayout(frame.width, frame.height), - }); - try { - const dataUrl = drawSourceToDataUrl(videoFrame as unknown as CanvasImageSource, frame.width, frame.height); - if (dataUrl) { - this.retainSnapshot(key, dataUrl); - } - } finally { - videoFrame.close(); - } - } catch { - return; - } - } - release(key: string): void { if (!this.snapshots.has(key)) return; this.update(() => { diff --git a/fluxer_app/src/features/voice/components/voice_participant_tile/NativeParticipantVideo.tsx b/fluxer_app/src/features/voice/components/voice_participant_tile/NativeParticipantVideo.tsx deleted file mode 100644 index 143a220ec..000000000 --- a/fluxer_app/src/features/voice/components/voice_participant_tile/NativeParticipantVideo.tsx +++ /dev/null @@ -1,122 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {Logger} from '@app/features/platform/utils/AppLogger'; -import NativeVideoElementPool from '@app/features/voice/components/voice_participant_tile/NativeVideoElementPool'; -import NativeVideoTileManager, { - type NativeInboundVideoTrack, -} from '@app/features/voice/engine/native_voice_engine/NativeVideoTileManager'; -import {useStoreVersion} from '@app/features/voice/engine/Store'; -import { - asPinnableVoiceTrackSource, - isVoiceScreenShareSource, - type PinnableVoiceTrackSource, - VoiceTrackSource, -} from '@app/features/voice/engine/VoiceTrackSource'; -import {observer} from 'mobx-react-lite'; -import type React from 'react'; -import {forwardRef, useEffect, useRef} from 'react'; - -const logger = new Logger('NativeParticipantVideo'); - -interface SelectableNativeVideoTrack { - source: string; - participantSid: string; - trackSid: string; - stream: MediaStream; -} - -export function selectNativeTrackForSource( - tracks: ReadonlyArray, - source: unknown, -): NativeVideoTrack | undefined { - const requestedSource = asPinnableVoiceTrackSource(source); - if (!requestedSource) return undefined; - const wantScreenShare = requestedSource === VoiceTrackSource.ScreenShare; - return tracks.find((track) => - wantScreenShare ? isVoiceScreenShareSource(track.source) : !isVoiceScreenShareSource(track.source), - ); -} - -interface NativeParticipantVideoProps { - participantSid: string; - participantIdentity?: string; - source: PinnableVoiceTrackSource; - className?: string; - 'data-flx'?: string; -} - -export function useHasNativeParticipantVideo(participantSid: string, source: unknown): boolean { - return useNativeParticipantVideoTrack(participantSid, source) != null; -} - -export function useNativeParticipantVideoTrack( - participantSid: string, - source: unknown, - participantIdentity?: string, -): NativeInboundVideoTrack | undefined { - useStoreVersion(NativeVideoTileManager); - if (!participantSid && !participantIdentity) return undefined; - const tracks = NativeVideoTileManager.getTracksForParticipant(participantSid, participantIdentity); - return selectNativeTrackForSource(tracks, source); -} - -const POOL_HOST_STYLE: React.CSSProperties = {display: 'contents'}; - -function assignForwardedVideoRef(ref: React.ForwardedRef, element: HTMLVideoElement | null): void { - if (typeof ref === 'function') { - ref(element); - return; - } - if (!ref) return; - ref.current = element; -} - -export const NativeParticipantVideo = observer( - forwardRef(function NativeParticipantVideo( - {participantSid, participantIdentity, source, className, 'data-flx': dataFlx}, - forwardedRef, - ) { - const hostRef = useRef(null); - const videoRef = useRef(null); - useStoreVersion(NativeVideoTileManager); - const tracks = NativeVideoTileManager.getTracksForParticipant(participantSid, participantIdentity); - const track = selectNativeTrackForSource(tracks, source); - const stream = track?.stream ?? null; - const trackSid = track?.trackSid ?? null; - useEffect(() => { - const host = hostRef.current; - if (!host || !stream || !trackSid) return; - const element = NativeVideoElementPool.acquire(trackSid); - videoRef.current = element; - element.className = className ?? ''; - element.setAttribute('data-flx', dataFlx ?? 'voice.native-participant-video.video'); - if (element.srcObject !== stream) { - element.srcObject = stream; - } - host.appendChild(element); - assignForwardedVideoRef(forwardedRef, element); - const playResult = element.play(); - if (playResult && typeof playResult.catch === 'function') { - playResult.catch((error) => { - logger.debug('Native participant video play() rejected', {participantSid, error}); - }); - } - return () => { - if (element.parentNode === host) { - host.removeChild(element); - } - if (!NativeVideoElementPool.isPooledFor(trackSid, element)) { - element.srcObject = null; - } - if (videoRef.current === element) { - videoRef.current = null; - } - assignForwardedVideoRef(forwardedRef, null); - }; - }, [stream, trackSid, className, dataFlx, participantSid, forwardedRef]); - if (!stream || !trackSid) return null; - return
; - }), -); - -export type NativeParticipantVideoComponent = React.ComponentType; diff --git a/fluxer_app/src/features/voice/components/voice_participant_tile/NativeVideoElementPool.test.ts b/fluxer_app/src/features/voice/components/voice_participant_tile/NativeVideoElementPool.test.ts deleted file mode 100644 index 1cdaa824b..000000000 --- a/fluxer_app/src/features/voice/components/voice_participant_tile/NativeVideoElementPool.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -// @vitest-environment happy-dom -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {beforeEach, describe, expect, it} from 'vitest'; -import {NATIVE_VIDEO_ELEMENT_POOL_MAX, NativeVideoElementPool} from './NativeVideoElementPool'; - -let pool: NativeVideoElementPool; - -beforeEach(() => { - pool = new NativeVideoElementPool(); -}); - -describe('NativeVideoElementPool', () => { - it('creates a configured video element per trackSid', () => { - const element = pool.acquire('TR_1'); - expect(element.tagName.toLowerCase()).toBe('video'); - expect(element.muted).toBe(true); - expect(element.autoplay).toBe(true); - expect(pool.size).toBe(1); - }); - - it('reuses the same element across acquisitions for the same trackSid', () => { - const first = pool.acquire('TR_1'); - const fakeStream = new MediaStream(); - first.srcObject = fakeStream; - const second = pool.acquire('TR_1'); - expect(second).toBe(first); - expect(second.srcObject).toBe(fakeStream); - expect(pool.size).toBe(1); - }); - - it('returns distinct elements for distinct trackSids', () => { - expect(pool.acquire('TR_1')).not.toBe(pool.acquire('TR_2')); - expect(pool.size).toBe(2); - }); - - it('bounds the pool to the named cap by evicting the stalest element', () => { - const first = pool.acquire('TR_0'); - first.srcObject = new MediaStream(); - for (let i = 1; i <= NATIVE_VIDEO_ELEMENT_POOL_MAX; i++) { - pool.acquire(`TR_${i}`); - } - expect(pool.size).toBe(NATIVE_VIDEO_ELEMENT_POOL_MAX); - expect(pool.has('TR_0')).toBe(false); - expect(first.srcObject).toBeNull(); - expect(pool.has(`TR_${NATIVE_VIDEO_ELEMENT_POOL_MAX}`)).toBe(true); - }); - - it('keeps recently re-acquired elements resident under eviction pressure', () => { - pool.acquire('TR_0'); - for (let i = 1; i < NATIVE_VIDEO_ELEMENT_POOL_MAX; i++) { - pool.acquire(`TR_${i}`); - } - pool.acquire('TR_0'); - pool.acquire('TR_extra'); - expect(pool.has('TR_0')).toBe(true); - expect(pool.has('TR_1')).toBe(false); - expect(pool.size).toBe(NATIVE_VIDEO_ELEMENT_POOL_MAX); - }); - - it('hands out a disposable element while the pooled element is attached to a host', () => { - const pooled = pool.acquire('TR_1'); - document.body.appendChild(pooled); - const disposable = pool.acquire('TR_1'); - expect(disposable).not.toBe(pooled); - expect(pool.size).toBe(1); - expect(pool.isPooledFor('TR_1', pooled)).toBe(true); - expect(pool.isPooledFor('TR_1', disposable)).toBe(false); - pooled.remove(); - }); - - it('returns the pooled element again once it is detached', () => { - const pooled = pool.acquire('TR_1'); - document.body.appendChild(pooled); - pool.acquire('TR_1'); - pooled.remove(); - expect(pool.acquire('TR_1')).toBe(pooled); - }); - - it('release detaches the stream and removes the element', () => { - const element = pool.acquire('TR_1'); - element.srcObject = new MediaStream(); - pool.release('TR_1'); - expect(pool.has('TR_1')).toBe(false); - expect(element.srcObject).toBeNull(); - expect(pool.size).toBe(0); - }); - - it('release of an unknown trackSid is a no-op', () => { - expect(() => pool.release('TR_missing')).not.toThrow(); - }); - - it('clear releases every pooled element', () => { - pool.acquire('TR_1'); - pool.acquire('TR_2'); - pool.clear(); - expect(pool.size).toBe(0); - }); -}); diff --git a/fluxer_app/src/features/voice/components/voice_participant_tile/NativeVideoElementPool.ts b/fluxer_app/src/features/voice/components/voice_participant_tile/NativeVideoElementPool.ts deleted file mode 100644 index 24671f79b..000000000 --- a/fluxer_app/src/features/voice/components/voice_participant_tile/NativeVideoElementPool.ts +++ /dev/null @@ -1,64 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -export const NATIVE_VIDEO_ELEMENT_POOL_MAX = 16; - -class NativeVideoElementPool { - private elements = new Map(); - - get size(): number { - return this.elements.size; - } - - has(trackSid: string): boolean { - return this.elements.has(trackSid); - } - - acquire(trackSid: string): HTMLVideoElement { - const existing = this.elements.get(trackSid); - if (existing) { - if (existing.isConnected) { - return this.createElement(); - } - this.elements.delete(trackSid); - this.elements.set(trackSid, existing); - return existing; - } - const element = this.createElement(); - this.elements.set(trackSid, element); - while (this.elements.size > NATIVE_VIDEO_ELEMENT_POOL_MAX) { - const oldestTrackSid = this.elements.keys().next().value; - if (oldestTrackSid === undefined) break; - this.release(oldestTrackSid); - } - return element; - } - - isPooledFor(trackSid: string, element: HTMLVideoElement): boolean { - return this.elements.get(trackSid) === element; - } - - private createElement(): HTMLVideoElement { - const element = document.createElement('video'); - element.autoplay = true; - element.muted = true; - element.playsInline = true; - return element; - } - - release(trackSid: string): void { - const element = this.elements.get(trackSid); - if (!element) return; - this.elements.delete(trackSid); - element.srcObject = null; - element.remove(); - } - - clear(): void { - for (const trackSid of [...this.elements.keys()]) { - this.release(trackSid); - } - } -} - -export default new NativeVideoElementPool(); -export {NativeVideoElementPool}; diff --git a/fluxer_app/src/features/voice/components/voice_participant_tile/hooks.ts b/fluxer_app/src/features/voice/components/voice_participant_tile/hooks.ts index a9fc9ab0c..9966fd066 100644 --- a/fluxer_app/src/features/voice/components/voice_participant_tile/hooks.ts +++ b/fluxer_app/src/features/voice/components/voice_participant_tile/hooks.ts @@ -21,24 +21,19 @@ import { type StreamPreviewUploadUrlCacheEntryLike, } from '@app/features/voice/components/voice_participant_tile/StreamPreviewUploadPolicy'; import {logger} from '@app/features/voice/components/voice_participant_tile/shared'; -import {resolveDevicePixelRatio} from '@app/features/voice/DevicePixelRatio'; import MediaEngine from '@app/features/voice/engine/MediaEngineFacade'; import ScreenSharePublicationMigration from '@app/features/voice/engine/ScreenSharePublicationMigration'; import {useStoreVersion} from '@app/features/voice/engine/Store'; -import {isVoiceEngineV2NativeProjectionActiveFromMediaEngine} from '@app/features/voice/engine/VoiceMediaEngineBridge'; import { - buildVoiceMediaGraphNativeCameraSubscriptionCommand, selectVoiceMediaGraphHasFailureForStreamKey, selectVoiceMediaGraphViewerStreamKeys, type VoiceMediaGraphSnapshot, } from '@app/features/voice/engine/VoiceMediaGraph'; -import type {VoiceMediaGraphVideoQuality} from '@app/features/voice/engine/VoiceMediaGraphSubscriptionTypes'; import { asVoiceTrackSource, isScreenShareAudioPublicationLike, VoiceTrackSource, } from '@app/features/voice/engine/VoiceTrackSource'; -import {pickCameraSubscriptionQuality} from '@app/features/voice/engine/v2/VoiceEngineV2AppCameraResolutionPresets'; import VoiceEngineV2AppSubscriptionAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppSubscriptionAdapter'; import LocalVoiceState from '@app/features/voice/state/LocalVoiceState'; import { @@ -91,57 +86,10 @@ interface AutoVideoSubscriptionRequest { participantIdentity: string | null; trackSid: string | null; desired: boolean | null; - quality: VoiceMediaGraphVideoQuality | null; } function createEmptyAutoVideoSubscriptionRequest(): AutoVideoSubscriptionRequest { - return {participantIdentity: null, trackSid: null, desired: null, quality: null}; -} - -function subscribeNativeCamera(participantIdentity: string, quality: VoiceMediaGraphVideoQuality): void { - VoiceEngineV2AppSubscriptionAdapter.setRemoteTrackSubscription( - buildVoiceMediaGraphNativeCameraSubscriptionCommand({participantIdentity, subscribed: true, quality}), - ).catch((error) => { - logger.error('Native camera subscription update failed', error); - }); -} - -function unsubscribeNativeCamera(participantIdentity: string): void { - VoiceEngineV2AppSubscriptionAdapter.setRemoteTrackSubscription( - buildVoiceMediaGraphNativeCameraSubscriptionCommand({participantIdentity, subscribed: false}), - ).catch((error) => { - logger.error('Native camera subscription update failed', error); - }); -} - -export function useNativeCameraSubscriptionQuality( - ref: React.RefObject, - enabled: boolean, -): VoiceMediaGraphVideoQuality { - const [quality, setQuality] = useState('low'); - useEffect(() => { - if (!enabled) { - setQuality('low'); - return; - } - const element = ref.current; - if (!element) return; - const ownerWindow = element.ownerDocument.defaultView; - const measure = (): void => { - const ratio = resolveDevicePixelRatio(ownerWindow); - setQuality(pickCameraSubscriptionQuality(element.clientWidth * ratio, element.clientHeight * ratio)); - }; - measure(); - if (ownerWindow == null || typeof ownerWindow.ResizeObserver === 'undefined') return; - const observer = new ownerWindow.ResizeObserver(measure); - observer.observe(element); - return () => observer.disconnect(); - }, [enabled, ref]); - return quality; -} - -function isNativeCameraPlaceholderSubscriptionTarget(trackRef: TrackReferenceOrPlaceholder): boolean { - return asVoiceTrackSource(trackRef.source) === VoiceTrackSource.Camera && Boolean(trackRef.participant?.identity); + return {participantIdentity: null, trackSid: null, desired: null}; } interface StreamPreviewUploadUrlCacheEntry extends StreamPreviewUploadUrlCacheEntryLike { @@ -241,17 +189,8 @@ export function useAutoVideoSubscription(opts: { videoLocallyDisabled: boolean; isLocalParticipant: boolean; isScreenShare: boolean; - nativeCameraQuality: VoiceMediaGraphVideoQuality; }) { - const { - enabled, - trackRef, - isIntersecting, - videoLocallyDisabled, - isLocalParticipant, - isScreenShare, - nativeCameraQuality, - } = opts; + const {enabled, trackRef, isIntersecting, videoLocallyDisabled, isLocalParticipant, isScreenShare} = opts; const lastRequestedRef = useRef(createEmptyAutoVideoSubscriptionRequest()); const managedPublicationRef = useRef(null); const graceGateRef = useRef(null); @@ -261,19 +200,11 @@ export function useAutoVideoSubscription(opts: { useEffect(() => { const graceGate = graceGateRef.current; if (!graceGate) return; - const isNativeEngine = isVoiceEngineV2NativeProjectionActiveFromMediaEngine(); - const unsubscribeNativeRequest = (): void => { - const previous = lastRequestedRef.current; - if (!isNativeEngine || !previous.participantIdentity) return; - if (previous.desired !== true && managedPublicationRef.current === null) return; - unsubscribeNativeCamera(previous.participantIdentity); - }; const unsubscribeManagedPublication = (publication: RemoteTrackPublication | null): void => { unsubscribeManagedVideoPublication(managedPublicationRef, publication); }; const reset = (): void => { graceGate.cancel(); - unsubscribeNativeRequest(); unsubscribeManagedPublication(managedPublicationRef.current); lastRequestedRef.current = createEmptyAutoVideoSubscriptionRequest(); }; @@ -281,41 +212,6 @@ export function useAutoVideoSubscription(opts: { reset(); return; } - if (isNativeEngine && !isTrackReference(trackRef) && isNativeCameraPlaceholderSubscriptionTarget(trackRef)) { - const participantIdentity = trackRef.participant.identity; - const shouldSubscribe = isIntersecting && !videoLocallyDisabled; - const desiredQuality = shouldSubscribe ? nativeCameraQuality : null; - const previousRequest = lastRequestedRef.current; - const requestChanged = - previousRequest.trackSid !== null || previousRequest.participantIdentity !== participantIdentity; - if (requestChanged && previousRequest.participantIdentity) { - unsubscribeNativeCamera(previousRequest.participantIdentity); - } - if (requestChanged) { - lastRequestedRef.current = {participantIdentity, trackSid: null, desired: null, quality: null}; - } - if (lastRequestedRef.current.desired === shouldSubscribe && lastRequestedRef.current.quality === desiredQuality) { - return; - } - lastRequestedRef.current.desired = shouldSubscribe; - lastRequestedRef.current.quality = desiredQuality; - if (shouldSubscribe) { - graceGate.cancel(); - subscribeNativeCamera(participantIdentity, nativeCameraQuality); - managedPublicationRef.current = null; - return; - } - if (videoLocallyDisabled) { - graceGate.cancel(); - unsubscribeNativeCamera(participantIdentity); - managedPublicationRef.current = null; - return; - } - graceGate.scheduleDisable(() => { - unsubscribeNativeCamera(participantIdentity); - }); - return; - } if (!isTrackReference(trackRef)) { reset(); return; @@ -335,38 +231,8 @@ export function useAutoVideoSubscription(opts: { const previousRequest = lastRequestedRef.current; const requestChanged = previousRequest.trackSid !== trackSid || previousRequest.participantIdentity !== participantIdentity; - if (isNativeEngine && requestChanged && previousRequest.participantIdentity && managedPublicationRef.current) { - unsubscribeNativeCamera(previousRequest.participantIdentity); - } if (requestChanged) { - lastRequestedRef.current = {participantIdentity, trackSid, desired: null, quality: null}; - } - if (isNativeEngine) { - const desiredQuality = shouldSubscribe ? nativeCameraQuality : null; - if (lastRequestedRef.current.desired === shouldSubscribe && lastRequestedRef.current.quality === desiredQuality) { - return; - } - lastRequestedRef.current.desired = shouldSubscribe; - lastRequestedRef.current.quality = desiredQuality; - if (shouldSubscribe) { - graceGate.cancel(); - subscribeNativeCamera(participantIdentity, nativeCameraQuality); - managedPublicationRef.current = pub; - return; - } - if (videoLocallyDisabled) { - graceGate.cancel(); - unsubscribeNativeCamera(participantIdentity); - managedPublicationRef.current = null; - return; - } - graceGate.scheduleDisable(() => { - unsubscribeNativeCamera(participantIdentity); - if (managedPublicationRef.current === pub) { - managedPublicationRef.current = null; - } - }); - return; + lastRequestedRef.current = {participantIdentity, trackSid, desired: null}; } if (pub.isSubscribed === shouldSubscribe) { lastRequestedRef.current.desired = shouldSubscribe; @@ -402,18 +268,10 @@ export function useAutoVideoSubscription(opts: { logger.error('setSubscribed(false) failed after unsubscribe grace period', err); } }); - }, [enabled, trackRef, isIntersecting, videoLocallyDisabled, isLocalParticipant, isScreenShare, nativeCameraQuality]); + }, [enabled, trackRef, isIntersecting, videoLocallyDisabled, isLocalParticipant, isScreenShare]); useEffect(() => { return () => { graceGateRef.current?.cancel(); - const previous = lastRequestedRef.current; - if ( - isVoiceEngineV2NativeProjectionActiveFromMediaEngine() && - previous.participantIdentity && - (previous.desired === true || managedPublicationRef.current !== null) - ) { - unsubscribeNativeCamera(previous.participantIdentity); - } const publication = managedPublicationRef.current; managedPublicationRef.current = null; lastRequestedRef.current = createEmptyAutoVideoSubscriptionRequest(); diff --git a/fluxer_app/src/features/voice/components/voice_participant_tile/shared.ts b/fluxer_app/src/features/voice/components/voice_participant_tile/shared.ts index 4e5afea71..e1c1b8b4d 100644 --- a/fluxer_app/src/features/voice/components/voice_participant_tile/shared.ts +++ b/fluxer_app/src/features/voice/components/voice_participant_tile/shared.ts @@ -1,7 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import {Logger} from '@app/features/platform/utils/AppLogger'; -import {VoiceTrackKind, VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource'; +import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource'; import {msg} from '@lingui/core/macro'; import type {TrackReferenceOrPlaceholder} from '@livekit/components-react'; import type React from 'react'; @@ -127,16 +127,6 @@ export function isCameraSource(source: unknown) { return source === VoiceTrackSource.Camera; } -export function isAudioTrackWithVolume(track: unknown): track is {kind: string; setVolume: (volume: number) => void} { - return ( - track != null && - typeof track === 'object' && - 'kind' in track && - (track as {kind: string}).kind === VoiceTrackKind.Audio && - 'setVolume' in track - ); -} - export function getSourceDataAttr(source: unknown) { switch (source) { case VoiceTrackSource.ScreenShare: diff --git a/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts b/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts index 3a5b0938f..3ecd4226c 100644 --- a/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts +++ b/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts @@ -18,7 +18,6 @@ import * as NavigationCommands from '@app/features/navigation/commands/Navigatio import {SoundType, setSoundOutputDeviceIdResolver} from '@app/features/notification/utils/SoundUtils'; import MediaPermission from '@app/features/permissions/system/state/MediaPermission'; import {Logger} from '@app/features/platform/utils/AppLogger'; -import * as BackgroundImageDB from '@app/features/theme/utils/BackgroundImageDB'; import * as SoundCommands from '@app/features/ui/commands/SoundCommands'; import * as ToastCommands from '@app/features/ui/commands/ToastCommands'; import Idle from '@app/features/ui/state/Idle'; @@ -30,11 +29,9 @@ import {voiceStatsDB} from '@app/features/voice/diagnostics/VoiceStatsDB'; import AdaptiveScreenShareEngine from '@app/features/voice/engine/AdaptiveScreenShareEngine'; import { createMediaEngineFacadeSnapshot, - hasPlayedNativeVoiceReadySounds, type MediaEngineFacadeEvent, type MediaEngineFacadePendingSessionRestore, type MediaEngineFacadeSnapshot, - rememberNativeVoiceReadySounds, selectMediaEngineConnectPreflightDecision, selectMediaEngineConnectRequestDecision, selectMediaEngineGatewayErrorDecision, @@ -44,7 +41,6 @@ import { shouldRunMediaEngineDeferredDisconnect, transitionMediaEngineFacadeSnapshot, } from '@app/features/voice/engine/MediaEngineFacadeStateMachine'; -import {NativeVoiceStatsSession} from '@app/features/voice/engine/media_engine_facade/NativeVoiceStatsSession'; import { AFK_CHECK_INTERVAL_MS, CLAIM_YOUR_ACCOUNT_TO_JOIN_THIS_VOICE_CHANNEL_DESCRIPTOR, @@ -63,56 +59,12 @@ import { saveCurrentVoiceSessionRestoreSnapshot, type VoiceSessionRestoreSyncHandle, } from '@app/features/voice/engine/media_engine_facade/VoiceSessionRestoreSync'; -import { - isNativeVoiceEngineSelected, - isNativeVoiceEngineSelectionPending, - requireNativeVoiceEngine, - shouldUseNativeVoiceEngine, -} from '@app/features/voice/engine/native_voice_engine/getVoiceEngine'; -import { - type NativeAudioDeviceModuleStatus, - nativeAudioDeviceModuleState, -} from '@app/features/voice/engine/native_voice_engine/NativeAudioDeviceModuleState'; -import {NativeCameraPreviewStartGate} from '@app/features/voice/engine/native_voice_engine/NativeCameraPreviewStartGate'; -import { - type NativeCameraPreviewParticipant, - selectNativeCameraLocalPreviewTrack, -} from '@app/features/voice/engine/native_voice_engine/NativeCameraPreviewTrackSelection'; -import {shouldSuppressNativeLocalTrackStateDuringReconnect} from '@app/features/voice/engine/native_voice_engine/NativeLocalMediaReconnectPolicy'; -import NativeVideoTileManager, { - type NativeInboundVideoTrack, -} from '@app/features/voice/engine/native_voice_engine/NativeVideoTileManager'; -import {shouldRetryNativeVoiceConnectTimeout} from '@app/features/voice/engine/native_voice_engine/NativeVoiceConnectRetryPolicy'; -import {bindNativeVoiceDeviceSync} from '@app/features/voice/engine/native_voice_engine/NativeVoiceDeviceSync'; -import NativeVoiceE2EEStore from '@app/features/voice/engine/native_voice_engine/NativeVoiceE2EEStore'; -import { - awaitNativeVoiceEngineReadiness, - getNativeVoiceEngineCapabilitiesSnapshot, - refreshNativeVoiceEngineCapabilitiesSnapshot, -} from '@app/features/voice/engine/native_voice_engine/NativeVoiceEngineSelection'; -import {NativeVoiceFrameStatsBatcher} from '@app/features/voice/engine/native_voice_engine/NativeVoiceFrameStatsBatcher'; -import NativeVoiceStatsStore from '@app/features/voice/engine/native_voice_engine/NativeVoiceStatsStore'; -import { - applyNativeVoiceEngineConnectedRoster, - collectNativeVoiceEngineConnectedRosterPublishedTracks, - getNativeVoiceEngineConnectionEventAction, - ingestNativeVoiceEngineV2BridgeStats, - isFacadeOwnedConnectionEvent, - mapNativeVoiceEngineV2BridgeEvent, - type NativeVoiceEngineLocalTrackParticipant, - type NativeVoiceEngineLocalTrackPublication, - type NativeVoiceEngineV2BridgeEventManagers, -} from '@app/features/voice/engine/native_voice_engine/nativeVoiceEngineEventMapper'; -import type {VoiceEngine} from '@app/features/voice/engine/native_voice_engine/VoiceEngine'; import {getScreenShareCaptureDiagnosticSnapshot} from '@app/features/voice/engine/ScreenShareCaptureDiagnostics'; import ScreenShareCodecNegotiation, { buildLocalCodecAdvertisements, getScreenShareCodecPreferenceOrder, - SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC, } from '@app/features/voice/engine/ScreenShareCodecNegotiation'; -import ScreenSharePublicationMigration, { - SCREEN_SHARE_PUBLICATION_MIGRATION_TOPIC, -} from '@app/features/voice/engine/ScreenSharePublicationMigration'; +import ScreenSharePublicationMigration from '@app/features/voice/engine/ScreenSharePublicationMigration'; import {Store, useStoreVersion} from '@app/features/voice/engine/Store'; import {shouldMoveToAfkOnTick} from '@app/features/voice/engine/VoiceAfkTracking'; import { @@ -145,7 +97,6 @@ import {startVoiceMediaGraphTimerScheduler} from '@app/features/voice/engine/Voi import {bindOutputDeviceSync} from '@app/features/voice/engine/VoiceOutputDeviceSync'; import type {LivekitParticipantSnapshot} from '@app/features/voice/engine/VoiceParticipantStateMachine'; import {bindRoomEvents, type RoomEventDependencies} from '@app/features/voice/engine/VoiceRoomEventBinder'; -import {playSelfJoinChimeOnce} from '@app/features/voice/engine/VoiceSelfJoinChime'; import type {VoiceStateSyncPartial, VoiceStateSyncPayload} from '@app/features/voice/engine/VoiceStateSyncTypes'; import { deferStopWatchingStreamKey, @@ -153,20 +104,8 @@ import { replaceWatchedStreamKeys, stopWatchingStreamKey, } from '@app/features/voice/engine/VoiceStreamWatchState'; -import { - asVoiceTrackSource, - type VoiceConnectionQuality, - VoiceTrackSource, -} from '@app/features/voice/engine/VoiceTrackSource'; +import {type VoiceConnectionQuality, VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource'; import {bindVoiceEngineV2AppAudioPreferencesSync} from '@app/features/voice/engine/v2/VoiceEngineV2AppAudioPreferencesSyncBinding'; -import { - createVoiceEngineV2AppAudioSettingsSnapshot, - hasVoiceEngineV2InputProcessorSettingsChanged, - hasVoiceEngineV2MicrophoneCaptureSettingsChanged, -} from '@app/features/voice/engine/v2/VoiceEngineV2AppAudioSettingsSync'; -import {buildVoiceEngineV2AppCameraPermissionDeniedError} from '@app/features/voice/engine/v2/VoiceEngineV2AppCameraPermissionDeniedError'; -import {getCameraCaptureDimensions} from '@app/features/voice/engine/v2/VoiceEngineV2AppCameraResolutionPresets'; -import {runCameraTransition} from '@app/features/voice/engine/v2/VoiceEngineV2AppCameraTransition'; import {getLocalDecodableVideoCodecs} from '@app/features/voice/engine/v2/VoiceEngineV2AppCodecCapability'; import { computeVoiceEngineV2WatchedStreamGossip, @@ -185,30 +124,11 @@ import {createVoiceEngineV2AppIngestionPort} from '@app/features/voice/engine/v2 import type {VoiceEngineV2AppLifecycleDisposable} from '@app/features/voice/engine/v2/VoiceEngineV2AppLifecycleAdapter'; import voiceEngineV2AppMediaExecutionAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppMediaExecutionAdapter'; import voiceEngineV2AppMediaStateAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppMediaStateAdapter'; -import { - resolveVoiceEngineV2NativeCameraDeviceId, - type VoiceEngineV2NativeCameraDeviceResolution, -} from '@app/features/voice/engine/v2/VoiceEngineV2AppNativeCameraDeviceMapping'; -import {VoiceEngineV2AppNativeCaptureExecutionAdapter} from '@app/features/voice/engine/v2/VoiceEngineV2AppNativeCaptureExecutionAdapter'; -import {applyVoiceEngineV2NativeScreenShareAudioState} from '@app/features/voice/engine/v2/VoiceEngineV2AppNativeLocalTrackStateSync'; -import { - resolveVoiceEngineV2NativeMicrophoneMaxBitrateBps, - resolveVoiceEngineV2NativeMicrophonePublishOptions, - type VoiceEngineV2NativeMicrophonePublishOptions, -} from '@app/features/voice/engine/v2/VoiceEngineV2AppNativeMicrophoneSettings'; -import { - type VoiceEngineV2AppNativeVoiceConnectAttempt, - VoiceEngineV2AppNativeVoiceConnectionLifecycle, - type VoiceEngineV2AppNativeVoiceConnectReason, -} from '@app/features/voice/engine/v2/VoiceEngineV2AppNativeVoiceConnectionLifecycle'; -import {VoiceEngineV2AppNativeVoiceLiveKitMediaAdapter} from '@app/features/voice/engine/v2/VoiceEngineV2AppNativeVoiceLiveKitMediaAdapter'; import { createVoiceEngineV2AppParticipantAdapter, type VoiceEngineV2AppParticipantAdapter, } from '@app/features/voice/engine/v2/VoiceEngineV2AppParticipantAdapter'; -import VoiceEngineV2AppPermissionAdapter, { - type VoiceEngineV2AppNativePermissionEnforcement, -} from '@app/features/voice/engine/v2/VoiceEngineV2AppPermissionAdapter'; +import VoiceEngineV2AppPermissionAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppPermissionAdapter'; import {createVoiceEngineV2AppProductionHostPorts} from '@app/features/voice/engine/v2/VoiceEngineV2AppProductionHostPorts'; import { createVoiceEngineV2AppProjectionStore, @@ -218,16 +138,13 @@ import VoiceEngineV2AppRemoteSpeakingAdapter from '@app/features/voice/engine/v2 import type {VoiceEngineV2AppScreenShareControllerGateway} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareControllerRouting'; import voiceEngineV2AppScreenShareExecutionAdapter, { type DeviceScreenShareCaptureOptions, - type NativeScreenShareReconnectSnapshot, type ScreenShareReconnectSnapshot, } from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter'; -import {resolveVoiceEngineV2AppSelectedMediaMode} from '@app/features/voice/engine/v2/VoiceEngineV2AppSelectedMediaMode'; import {selectVoiceEngineV2AppIntentSelfMuteForVoiceStatePayload} from '@app/features/voice/engine/v2/VoiceEngineV2AppSelectors'; import {VoiceEngineV2AppSourceLifecycleBridge} from '@app/features/voice/engine/v2/VoiceEngineV2AppSourceLifecycleBridge'; import {VoiceEngineV2AppStatsHostAdapter} from '@app/features/voice/engine/v2/VoiceEngineV2AppStatsHostAdapter'; import VoiceEngineV2AppSubscriptionAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppSubscriptionAdapter'; import voiceEngineV2AppVoiceStateAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppVoiceStateAdapter'; -import type {NativeScreenShareOptions} from '@app/features/voice/engine/voice_screen_share_manager/DisplayMediaCapture'; import type {VoiceStateAckPayload} from '@app/features/voice/events/VoiceStateAck'; import CallMediaPrefs from '@app/features/voice/state/CallMediaPrefs'; import {type ChannelE2EEStatus, computeChannelE2EEStatus} from '@app/features/voice/state/ChannelE2EEStatus'; @@ -235,7 +152,7 @@ import LocalVoiceState from '@app/features/voice/state/LocalVoiceState'; import VoiceCallLayout from '@app/features/voice/state/VoiceCallLayout'; import VoiceRegionTeleport from '@app/features/voice/state/VoiceRegionTeleport'; import VoiceSessionRestore, {type VoiceSessionRestoreSnapshot} from '@app/features/voice/state/VoiceSessionRestore'; -import VoiceSettings, {BLUR_BACKGROUND_ID, NONE_BACKGROUND_ID} from '@app/features/voice/state/VoiceSettings'; +import VoiceSettings from '@app/features/voice/state/VoiceSettings'; import {type CodecPreference, getCodecCapabilityReport} from '@app/features/voice/utils/CodecCapabilityDetector'; import {getGpuEncoderReportSync, loadGpuEncoderReport} from '@app/features/voice/utils/GpuEncoderCapabilities'; import { @@ -244,17 +161,13 @@ import { } from '@app/features/voice/utils/NativeAudioCaptureBridge'; import {getOpenH264StatusSync, loadOpenH264Status} from '@app/features/voice/utils/OpenH264Status'; import {areOrderedStringArraysEqual} from '@app/features/voice/utils/StringArrayUtils'; -import {VideoBackgroundFramePump} from '@app/features/voice/utils/VideoBackgroundFramePump'; import { getVideoDecoderExclusionsSync, loadVideoDecoderExclusions, } from '@app/features/voice/utils/VideoDecoderCapabilities'; -import {areVoiceBackgroundsAvailable} from '@app/features/voice/utils/VoiceBackgroundAvailability'; -import {voiceDeviceManager} from '@app/features/voice/utils/VoiceDeviceManager'; import {buildVoiceParticipantIdentity} from '@app/features/voice/utils/VoiceParticipantIdentity'; import { isVoiceServerMuteActive, - isVoiceSpeakPermissionDenied, resolveVoiceStateSelfMute, shouldPrepareMicrophoneForVoiceConnect, } from '@app/features/voice/utils/VoicePermissionUtils'; @@ -276,7 +189,6 @@ import type { VoiceEngineV2Model, VoiceEngineV2PerTrackStats, VoiceEngineV2Snapshot, - VoiceEngineV2Stats, VoiceEngineV2StatsSample, VoiceEngineV2TransportInfo, VoiceEngineV2VideoCodec, @@ -287,11 +199,6 @@ import { shouldApplyGatewayVoiceStateEcho, VOICE_ENGINE_V2_CODEC_GOSSIP_TOPIC, } from '@fluxer/voice_engine_v2'; -import { - translateVoiceEngineV2BridgeEventToEvents, - type VoiceEngineV2BridgeEvent, - type VoiceEngineV2BridgeVideoFrame, -} from '@fluxer/voice_engine_v2/bridge'; import {type I18n, i18n as linguiI18n, type MessageDescriptor} from '@lingui/core'; import { type ConnectionQuality as LiveKitConnectionQuality, @@ -305,127 +212,9 @@ import { import {makeObservable, observable} from 'mobx'; const logger = new Logger('MediaEngineFacade'); -const voiceEngineV2AppNativeCaptureExecutionAdapter = new VoiceEngineV2AppNativeCaptureExecutionAdapter(); const MAX_DIAGNOSTIC_LINUX_AUDIO_TARGETS = 200; const MAX_DIAGNOSTIC_PIPEWIRE_GRAPH_RECORDS = 500; -class TimeoutError extends Error { - constructor(message: string) { - super(message); - this.name = 'TimeoutError'; - } -} - -const NATIVE_VOICE_ENGINE_CONNECT_TIMEOUT_MS = 2_000; -const NATIVE_VOICE_ENGINE_CONNECT_RETRY_BACKOFF_BASE_MS = 25; -const NATIVE_VOICE_ENGINE_CONNECT_RETRY_BACKOFF_CEILING_MS = 100; -const NATIVE_VOICE_ENGINE_CONNECT_MAX_LOCAL_RETRIES = 1; - -function getNativeVoiceEngineConnectRetryDelayMs(retryAttempt: number): number { - const exponentialDelayMs = NATIVE_VOICE_ENGINE_CONNECT_RETRY_BACKOFF_BASE_MS * 2 ** Math.max(0, retryAttempt - 1); - return Math.min(exponentialDelayMs, NATIVE_VOICE_ENGINE_CONNECT_RETRY_BACKOFF_CEILING_MS); -} - -type LinuxAudioRoutingRule = NonNullable; - -function areVirtmicNodesEqual(left: Record, right: Record): boolean { - const leftKeys = Object.keys(left).sort(); - const rightKeys = Object.keys(right).sort(); - if (!areOrderedStringArraysEqual(leftKeys, rightKeys)) return false; - return leftKeys.every((key) => left[key] === right[key]); -} - -function areVirtmicNodeListsEqual( - left: Array> | undefined, - right: Array> | undefined, -): boolean { - const leftNodes = left ?? []; - const rightNodes = right ?? []; - if (leftNodes.length !== rightNodes.length) return false; - return leftNodes.every((node, index) => areVirtmicNodesEqual(node, rightNodes[index])); -} - -function areLinuxAudioRoutingRulesEqual(left: LinuxAudioRoutingRule, right: LinuxAudioRoutingRule): boolean { - return ( - left.workaround === right.workaround && - left.ignoreDevices === right.ignoreDevices && - left.ignoreInputMedia === right.ignoreInputMedia && - left.ignoreVirtual === right.ignoreVirtual && - left.onlySpeakers === right.onlySpeakers && - left.onlyDefaultSpeakers === right.onlyDefaultSpeakers && - areVirtmicNodeListsEqual(left.include, right.include) && - areVirtmicNodeListsEqual(left.exclude, right.exclude) - ); -} -const NATIVE_VOICE_ENGINE_TRANSPORT_RECONNECT_DELAY_MS = 1250; -function encodeVoiceEngineE2EEKey(key: string | null | undefined): ArrayBuffer | null { - if (!key) return null; - const encoded = new TextEncoder().encode(key); - return encoded.buffer.slice(encoded.byteOffset, encoded.byteOffset + encoded.byteLength); -} - -function nativeCameraDeviceIdFromResolution(resolution: VoiceEngineV2NativeCameraDeviceResolution): string | undefined { - assert.ok(resolution !== null && typeof resolution === 'object', 'camera device resolution must be an object'); - switch (resolution.status) { - case 'default': - return undefined; - case 'direct': - assert.ok(resolution.deviceId, 'direct camera device resolution must carry a native device id'); - logger.debug('Using native camera device id directly', { - requestedDeviceId: resolution.requestedDeviceId, - nativeLabel: resolution.nativeLabel, - }); - return resolution.deviceId; - case 'mapped': - assert.ok(resolution.deviceId, 'mapped camera device resolution must carry a native device id'); - logger.info('Mapped browser camera device to native camera device', { - browserLabel: resolution.browserLabel, - nativeLabel: resolution.nativeLabel, - nativeDeviceId: resolution.deviceId, - }); - return resolution.deviceId; - case 'ambiguous': - logger.warn('Native camera device mapping is ambiguous; falling back to default camera', { - browserLabel: resolution.browserLabel, - matchCount: resolution.matchCount, - }); - return undefined; - case 'unmapped': - logger.warn('Native camera device mapping failed; falling back to default camera', { - browserLabel: resolution.browserLabel, - requestedDeviceId: resolution.requestedDeviceId, - nativeDeviceCount: resolution.nativeDeviceCount, - }); - return undefined; - case 'unavailable': - logger.warn('Native camera enumeration returned no devices; falling back to default camera', { - requestedDeviceId: resolution.requestedDeviceId, - }); - return undefined; - } -} - -async function resolveNativeCameraDeviceId(deviceId?: string): Promise { - if (!deviceId || deviceId === 'default') return undefined; - const trimmedDeviceId = deviceId.trim(); - try { - const nativeEngine = requireNativeVoiceEngine(); - const [browserDeviceState, nativeDevices] = await Promise.all([ - voiceDeviceManager.ensureDevices({requestPermissions: false, forceRefresh: true}), - nativeEngine.listCameraDevices(), - ]); - const resolution = resolveVoiceEngineV2NativeCameraDeviceId({ - requestedDeviceId: trimmedDeviceId, - browserDevices: browserDeviceState.videoDevices, - nativeDevices, - }); - return nativeCameraDeviceIdFromResolution(resolution); - } catch (error) { - logger.warn('Failed to map browser camera device to native camera device', {deviceId, error}); - return undefined; - } -} - function isCameraPermissionDeniedCommandFailure(error: unknown): boolean { if (!(error instanceof Error)) return false; const code = (error as Error & {code?: unknown}).code; @@ -444,42 +233,6 @@ function buildCameraCommandOptions(options?: {deviceId?: string; sendUpdate?: bo return commandOptions; } -function buildScreenEncodingCaptureOptions(options: { - width: number; - height: number; - frameRate?: number; -}): ScreenShareCaptureOptions { - const resolution: NonNullable = { - width: options.width, - height: options.height, - }; - if (options.frameRate !== undefined) { - resolution.frameRate = options.frameRate; - } - return {resolution}; -} - -function buildScreenEncodingPublishOptions(options: { - codec?: '' | 'vp8' | 'vp9' | 'h264' | 'h265' | 'av1'; - frameRate?: number; - maxBitrateBps?: number; -}): TrackPublishOptions | undefined { - const publishOptions: TrackPublishOptions = {}; - if (options.codec) { - publishOptions.videoCodec = options.codec; - } - if (options.maxBitrateBps !== undefined) { - const screenShareEncoding: {maxBitrate: number; maxFramerate?: number} = { - maxBitrate: options.maxBitrateBps, - }; - if (options.frameRate !== undefined) { - screenShareEncoding.maxFramerate = options.frameRate; - } - publishOptions.screenShareEncoding = screenShareEncoding; - } - return Object.keys(publishOptions).length > 0 ? publishOptions : undefined; -} - function buildRoomEventDependencies(facade: { resetStreamTracking: () => void; participants: VoiceEngineV2AppParticipantAdapter; @@ -576,18 +329,6 @@ function voiceDiagnosticError(error: unknown): Record { return {message: String(error)}; } -function withTimeout(task: Promise, timeoutMs: number, message: string): Promise { - let timeoutId: NodeJS.Timeout | null = null; - const timeout = new Promise((_resolve, reject) => { - timeoutId = setTimeout(() => reject(new TimeoutError(message)), timeoutMs); - }); - return Promise.race([task, timeout]).finally(() => { - if (timeoutId !== null) { - clearTimeout(timeoutId); - } - }); -} - function limitLinuxAudioTargets(result: unknown): unknown { if (!result || typeof result !== 'object' || !('targets' in result)) return result; const record = result as {targets?: unknown}; @@ -721,21 +462,8 @@ interface ConnectDirectlyOptions { type TerminalUnloadVoiceDisconnectReason = 'pagehide' | 'beforeunload' | 'unload'; -interface NativeLocalMediaReconnectSnapshot { - connectionId: string | null; - restoreVideo: boolean; - restoreStream: boolean; - screenShare: NativeScreenShareReconnectSnapshot | null; - screenShareRelease: Promise | null; -} - type VoiceMuteReason = 'guild' | 'permission' | 'voice_push_to_talk' | 'self' | null; -interface ConnectViaNativeEngineOptions { - forceReconnect?: boolean; - reason?: VoiceEngineV2AppNativeVoiceConnectReason; -} - export interface VoiceEngineConnectionContext { guildId: string | null; channelId: string | null; @@ -770,35 +498,7 @@ class MediaEngineFacade extends Store { private pendingServerDisconnectTimeout: NodeJS.Timeout | null = null; private facadeSnapshot: MediaEngineFacadeSnapshot = createMediaEngineFacadeSnapshot(); private disconnectPromise: Promise | null = null; - private nativeVoiceEngineV2BridgeEventDisposer: (() => void) | null = null; - private nativeVoiceDeviceSyncDisposer: (() => void) | null = null; - private nativeVoiceDataProtocolDisposer: (() => void) | null = null; - private nativeVoiceConnectRetryTimeoutId: number | null = null; - private nativeVoiceConnectRetryCounts = new Map(); - private readonly nativeVoiceConnectionLifecycle = new VoiceEngineV2AppNativeVoiceConnectionLifecycle(); - private nativeVoiceTransportReconnectTimeoutId: number | null = null; - private readonly nativeVoiceStatsSession = new NativeVoiceStatsSession({ - ingestStats: (stats, timestampMs) => this.ingestNativeVoiceStats(stats, timestampMs), - }); - private readonly nativeVoiceFrameStatsBatcher = new NativeVoiceFrameStatsBatcher({ - dispatch: (event) => { - this.voiceEngineV2Host.dispatch(event); - }, - onDroppedUpdates: (droppedUpdatesCount) => { - logger.warn('Native inbound video frame stats dropped updates at the track cap', {droppedUpdatesCount}); - voiceEngineV2AppDebugLoggingHostAdapter.recordNativeVideoDiagnostic('frame_stats.dropped_updates', { - droppedUpdatesCount, - }); - }, - }); - private lastNativeVoiceServerUpdate: VoiceServerUpdateData | null = null; - private nativeVoiceReadySoundConnectionIds = new Set(); - private pendingNativeLocalMediaReconnect: NativeLocalMediaReconnectSnapshot | null = null; private terminalUnloadVoiceDisconnectSent = false; - private nativeCameraPreviewTrackSid: string | null = null; - private nativeCameraPreviewSessionTrackSid: string | null = null; - private readonly nativeCameraPreviewStartGate = new NativeCameraPreviewStartGate(); - private videoBackgroundFramePump: VideoBackgroundFramePump | null = null; private voiceEngineV2EstablishedConnectionKey: string | null = null; private lastVoiceConnectFailed = false; @@ -806,28 +506,6 @@ class MediaEngineFacade extends Store { super(); this.statsHostAdapter = new VoiceEngineV2AppStatsHostAdapter(); const voiceEngineV2Ingestion = createVoiceEngineV2AppIngestionPort(); - const nativeVoiceMedia = new VoiceEngineV2AppNativeVoiceLiveKitMediaAdapter({ - getEngine: requireNativeVoiceEngine, - camera: { - publishCamera: (options) => this.setCameraEnabledViaEngine(true, options), - updateCameraEncoding: (options) => - this.updateActiveCameraCapture(options.deviceId ? {deviceId: options.deviceId} : undefined), - unpublishCamera: (options) => this.setCameraEnabledViaEngine(false, options), - }, - screen: { - publishScreen: (options) => - voiceEngineV2AppScreenShareExecutionAdapter.publishControllerScreenViaNativeCapture(options), - updateScreenEncoding: async (options) => { - await voiceEngineV2AppScreenShareExecutionAdapter.updateActiveScreenShareSettings( - null, - buildScreenEncodingCaptureOptions(options), - buildScreenEncodingPublishOptions(options), - ); - }, - unpublishScreen: () => voiceEngineV2AppScreenShareExecutionAdapter.unpublishControllerScreenViaNativeCapture(), - }, - logger, - }); this.voiceEngineV2Host = createVoiceEngineV2AppControllerHost({ eventLogSpillSink: createVoiceEngineV2AppEventLogSpillLoggerSink({logger}), ports: createVoiceEngineV2AppProductionHostPorts({ @@ -868,9 +546,6 @@ class MediaEngineFacade extends Store { VoiceSettings.updateSettings({outputDeviceId: deviceId}); }, }, - nativeMedia: voiceEngineV2AppNativeCaptureExecutionAdapter, - nativeVoiceMedia, - getSelectedMediaMode: resolveVoiceEngineV2AppSelectedMediaMode, ingestion: voiceEngineV2Ingestion, subscriptions: VoiceEngineV2AppSubscriptionAdapter, stats: this.statsHostAdapter, @@ -902,7 +577,7 @@ class MediaEngineFacade extends Store { this.initializeTerminalUnloadVoiceDisconnect(); Sound.setSelfDeafenedResolver(() => getEffectiveAudioState().effectiveDeaf); setSoundOutputDeviceIdResolver(() => VoiceSettings.getOutputDeviceId()); - this.prewarmNativeVoiceEngineInBackground('startup'); + this.prewarmVoiceEngineV2InBackground('startup'); ( window as typeof window & { _mediaEngine?: MediaEngineFacade; @@ -938,34 +613,10 @@ class MediaEngineFacade extends Store { VoiceEngineV2AppRemoteSpeakingAdapter.clear(); VoiceEngineV2AppSubscriptionAdapter.cleanup(); VoiceEngineV2AppPermissionAdapter.reset(); - this.nativeVoiceFrameStatsBatcher.teardown(); - NativeVideoTileManager.clear(); this.resetLocalMediaAndScreenShareTracking(); this.voiceEngineV2Participants.clear(); }, }, - { - name: 'native-voice-runtime-state', - dispose: async () => { - this.clearNativeVoiceConnectSession(); - this.clearPendingNativeLocalMediaReconnect(); - this.stopNativeVoiceStatsSession(); - this.lastNativeVoiceServerUpdate = null; - }, - }, - { - name: 'native-voice-timers', - dispose: async () => { - this.clearNativeVoiceTransportReconnect(); - this.clearNativeVoiceConnectRetry(); - }, - }, - { - name: 'native-voice-bindings', - dispose: async () => { - this.disposeNativeVoiceEngineBindings(); - }, - }, ]; } @@ -975,7 +626,7 @@ class MediaEngineFacade extends Store { }); } - private prewarmNativeVoiceEngineInBackground(reason: string): void { + private prewarmVoiceEngineV2InBackground(reason: string): void { void this.voiceEngineV2Host .runAndWait( () => { @@ -1000,9 +651,6 @@ class MediaEngineFacade extends Store { voiceEngineV2AppConnectionHostAdapter.subscribe(forwardChange); voiceEngineV2AppVoiceStateAdapter.subscribe(forwardChange); this.statsHostAdapter.subscribe(forwardChange); - NativeVoiceStatsStore.subscribe(forwardChange); - NativeVideoTileManager.subscribe(forwardChange); - NativeVoiceE2EEStore.subscribe(forwardChange); AdaptiveScreenShareEngine.subscribe(forwardChange); voiceEngineV2AppMediaExecutionAdapter.subscribe(forwardChange); voiceEngineV2AppScreenShareExecutionAdapter.subscribe(forwardChange); @@ -1192,30 +840,11 @@ class MediaEngineFacade extends Store { }; } - private get nativeVoiceStatsActive(): boolean { - if (this.room) return false; - if (!isNativeVoiceEngineSelected()) return false; - return ( - voiceEngineV2AppConnectionHostAdapter.connected || - voiceEngineV2AppConnectionHostAdapter.reconnecting || - NativeVoiceStatsStore.stats != null || - NativeVoiceStatsStore.currentLatency != null - ); - } - - private get nativeVoiceParticipantCount(): number { - const participantCount = Object.keys(this.voiceEngineV2Participants.participants).length; - return Math.max(participantCount, this.connected ? 1 : 0); - } - get participants(): Readonly> { return this.voiceEngineV2Participants.participants; } get localConnectionQuality(): VoiceConnectionQuality | LiveKitConnectionQuality { - if (this.nativeVoiceStatsActive) { - return this.voiceEngineV2Participants.getLocalParticipant()?.connectionQuality ?? 'unknown'; - } return this.room?.localParticipant?.connectionQuality ?? 'unknown'; } @@ -1224,44 +853,34 @@ class MediaEngineFacade extends Store { } get currentLatency(): number | null { - if (this.nativeVoiceStatsActive) return NativeVoiceStatsStore.currentLatency; return this.statsHostAdapter.currentLatency; } get averageLatency(): number | null { - if (this.nativeVoiceStatsActive) return NativeVoiceStatsStore.averageLatency; return this.statsHostAdapter.averageLatency; } get latencyHistory(): Array { - if (this.nativeVoiceStatsActive) return NativeVoiceStatsStore.latencyHistory; return this.statsHostAdapter.latencyHistory; } get voiceStats(): VoiceEngineV2VoiceStats { - if (this.nativeVoiceStatsActive) { - return NativeVoiceStatsStore.getVoiceStats({participantCount: this.nativeVoiceParticipantCount}); - } return this.statsHostAdapter.voiceStats; } get perTrackStats(): Array { - if (this.nativeVoiceStatsActive) return NativeVoiceStatsStore.perTrackStats; return this.statsHostAdapter.perTrackStats; } get statsTimeSeries(): Array { - if (this.nativeVoiceStatsActive) return NativeVoiceStatsStore.statsTimeSeries; return this.statsHostAdapter.statsTimeSeries; } get publisherTransport(): VoiceEngineV2TransportInfo | null { - if (this.nativeVoiceStatsActive) return null; return this.statsHostAdapter.publisherTransport; } get subscriberTransport(): VoiceEngineV2TransportInfo | null { - if (this.nativeVoiceStatsActive) return null; return this.statsHostAdapter.subscriberTransport; } @@ -1285,13 +904,6 @@ class MediaEngineFacade extends Store { } private async refreshMicrophoneFromCurrentEngine(): Promise { - if (await shouldUseNativeVoiceEngine()) { - if (this.voiceEngineV2Snapshot.microphone.published == null) { - return; - } - await this.refreshNativeMicrophoneFromSettings(); - return; - } await voiceEngineV2AppMediaExecutionAdapter.refreshMicrophone(this.room, {forceRepublish: true}); } @@ -1302,20 +914,6 @@ class MediaEngineFacade extends Store { } private async refreshCameraBackgroundFromCurrentEngine(): Promise { - if (await shouldUseNativeVoiceEngine()) { - if (!this.readNativeCameraActualEnabled()) { - this.syncVideoBackgroundFramePumpInBackground('camera-background-refresh'); - return; - } - try { - await requireNativeVoiceEngine().updateCameraCapture(await this.getNativeCameraPublishParams()); - } catch (error) { - logger.warn('Native camera capture hot update failed; falling back to camera republish', {error}); - await this.setCameraEnabledViaEngine(true, {sendUpdate: false}); - } - this.syncVideoBackgroundFramePumpInBackground('camera-background-refresh'); - return; - } await voiceEngineV2AppMediaExecutionAdapter.refreshCameraBackground(); } @@ -1326,33 +924,12 @@ class MediaEngineFacade extends Store { } private async refreshCameraCaptureFromCurrentEngine(): Promise { - if (await shouldUseNativeVoiceEngine()) { - if (!this.readNativeCameraActualEnabled()) { - return; - } - await this.republishNativeCameraForCaptureChange(); - return; - } await voiceEngineV2AppMediaExecutionAdapter.refreshCameraCapture(); } - private async republishNativeCameraForCaptureChange(): Promise { - this.clearNativeCameraLocalPreview(); - try { - await requireNativeVoiceEngine().unpublishCamera(); - await this.publishNativeCameraFromSettings(); - } catch (error) { - logger.warn('Native camera resolution republish failed; attempting to restore camera', {error}); - await this.setCameraEnabledViaEngine(true, {sendUpdate: false}); - } - this.syncVideoBackgroundFramePumpInBackground('camera-resolution-refresh'); - } - async refreshActiveScreenShareCodecNegotiation(): Promise { const room = this.room; - const selection = isNativeVoiceEngineSelected() - ? await ScreenShareCodecNegotiation.publishLocalCapabilitiesNative('manual') - : await ScreenShareCodecNegotiation.publishLocalCapabilities(room, 'manual', {}); + const selection = await ScreenShareCodecNegotiation.publishLocalCapabilities(room, 'manual', {}); const codec = selection?.codec ?? ScreenShareCodecNegotiation.selectScreenShareCodec(VoiceSettings.getPreferredScreenShareCodec()); @@ -1398,151 +975,6 @@ class MediaEngineFacade extends Store { } } - private clearNativeVoiceTransportReconnect(): void { - if (this.nativeVoiceTransportReconnectTimeoutId !== null) { - window.clearTimeout(this.nativeVoiceTransportReconnectTimeoutId); - this.nativeVoiceTransportReconnectTimeoutId = null; - } - } - - private clearNativeVoiceConnectRetryTimeout(): void { - if (this.nativeVoiceConnectRetryTimeoutId !== null) { - window.clearTimeout(this.nativeVoiceConnectRetryTimeoutId); - this.nativeVoiceConnectRetryTimeoutId = null; - } - } - - private clearNativeVoiceConnectRetry(): void { - this.clearNativeVoiceConnectRetryTimeout(); - this.nativeVoiceConnectRetryCounts.clear(); - } - - private clearNativeVoiceConnectSession(): void { - this.nativeVoiceConnectionLifecycle.clearSession(); - } - - private isVoiceConnectionCurrentForNativeAttempt(attempt: VoiceEngineV2AppNativeVoiceConnectAttempt): boolean { - return this.nativeVoiceConnectionLifecycle.isCurrentAttemptForConnection( - attempt, - voiceEngineV2AppConnectionHostAdapter.connectionState, - ); - } - - private isDuplicateNativeVoiceServerUpdate({ - guildId, - channelId, - connectionId, - endpoint, - token, - }: { - guildId: string | null; - channelId: string; - connectionId: string | null; - endpoint: string; - token: string; - }): boolean { - return this.nativeVoiceConnectionLifecycle.isDuplicateServerUpdate( - {guildId, channelId, connectionId, endpoint, token}, - voiceEngineV2AppConnectionHostAdapter.connectionState, - this.lastNativeVoiceServerUpdate?.token ?? null, - ); - } - - private clearPendingNativeLocalMediaReconnect(): void { - this.pendingNativeLocalMediaReconnect = null; - } - - private prepareNativeLocalMediaReconnect(reason: string): void { - const previous = this.pendingNativeLocalMediaReconnect; - const restoreVideo = previous?.restoreVideo === true || LocalVoiceState.getSelfVideo(); - const restoreStream = previous?.restoreStream === true || LocalVoiceState.getSelfStream(); - const screenShare = - previous?.screenShare ?? - (restoreStream ? voiceEngineV2AppScreenShareExecutionAdapter.prepareNativeScreenShareReconnect() : null); - const screenShareRelease = - previous?.screenShareRelease ?? - (screenShare - ? voiceEngineV2AppScreenShareExecutionAdapter - .releaseNativeScreenShareForReconnect(screenShare) - .catch((error) => { - logger.warn('Failed to release native screen share before reconnect restore', {reason, error}); - }) - : null); - if (!restoreVideo && !restoreStream && !screenShare) { - return; - } - this.pendingNativeLocalMediaReconnect = { - connectionId: voiceEngineV2AppConnectionHostAdapter.connectionId, - restoreVideo, - restoreStream, - screenShare, - screenShareRelease, - }; - logger.debug('Prepared native local media reconnect restore', { - reason, - connectionId: voiceEngineV2AppConnectionHostAdapter.connectionId, - restoreVideo, - restoreStream, - hasScreenShareSnapshot: screenShare !== null, - }); - } - - private getPendingNativeLocalTrackRestoreIntent(): {restoreVideo: boolean; restoreStream: boolean} { - const pending = this.pendingNativeLocalMediaReconnect; - return { - restoreVideo: pending?.restoreVideo === true, - restoreStream: pending?.restoreStream === true, - }; - } - - private shouldSuppressNativeLocalTrackState(source: VoiceTrackSource, enabled: boolean): boolean { - const {restoreVideo, restoreStream} = this.getPendingNativeLocalTrackRestoreIntent(); - return shouldSuppressNativeLocalTrackStateDuringReconnect({ - source, - enabled, - reconnecting: - voiceEngineV2AppConnectionHostAdapter.reconnecting || this.pendingNativeLocalMediaReconnect !== null, - restoreVideo, - restoreStream, - }); - } - - private restoreNativeLocalMediaStateInBackground(reason: string): void { - void this.restoreNativeLocalMediaState(reason).catch((error) => { - logger.warn('Native local media reconnect restore failed', {reason, error}); - }); - } - - private async restoreNativeLocalMediaState(reason: string): Promise { - const snapshot = this.pendingNativeLocalMediaReconnect; - if (!snapshot) return; - if (snapshot.connectionId && voiceEngineV2AppConnectionHostAdapter.connectionId !== snapshot.connectionId) { - logger.debug('Dropping stale native local media reconnect restore', { - reason, - snapshotConnectionId: snapshot.connectionId, - currentConnectionId: voiceEngineV2AppConnectionHostAdapter.connectionId, - }); - this.clearPendingNativeLocalMediaReconnect(); - return; - } - this.clearPendingNativeLocalMediaReconnect(); - await snapshot.screenShareRelease; - const screenShareSnapshot = snapshot.screenShare; - await this.restoreLocalMedia({ - reason, - room: this.room, - restoreCamera: snapshot.restoreVideo && LocalVoiceState.getSelfVideo(), - restoreStream: snapshot.restoreStream && LocalVoiceState.getSelfStream(), - restoreStreamFromSnapshot: screenShareSnapshot - ? () => voiceEngineV2AppScreenShareExecutionAdapter.restoreNativeScreenShareReconnect(screenShareSnapshot) - : null, - streamSnapshotFallback: true, - streamPlaySound: false, - settleStreamStateWhenNotRestored: true, - toastWhenStreamNotRestored: true, - }); - } - private async restoreLocalMedia(args: { reason: string; room: Room | null; @@ -1600,34 +1032,6 @@ class MediaEngineFacade extends Store { } } - private disposeNativeVoiceEngineBindings(clearStats = true): void { - this.clearNativeCameraLocalPreview(); - this.nativeVoiceEngineV2BridgeEventDisposer?.(); - this.nativeVoiceEngineV2BridgeEventDisposer = null; - this.nativeVoiceDeviceSyncDisposer?.(); - this.nativeVoiceDeviceSyncDisposer = null; - this.nativeVoiceDataProtocolDisposer?.(); - this.nativeVoiceDataProtocolDisposer = null; - VoiceEngineV2AppSubscriptionAdapter.cleanup(); - this.nativeVoiceFrameStatsBatcher.teardown(); - NativeVideoTileManager.clear(); - NativeVoiceE2EEStore.clear(); - this.stopNativeVoiceStatsSession(clearStats); - this.clearNativeVoiceConnectSession(); - } - - private isGatewayVoiceStateActiveForConnection( - connectionId: string, - guildId: string | null, - channelId: string, - ): boolean { - const voiceState = voiceEngineV2AppVoiceStateAdapter.getVoiceStateByConnectionId(connectionId); - if (!voiceState?.channel_id) return false; - const expectedGuildId = guildId ?? ME; - const actualGuildId = voiceState.guild_id ?? ME; - return voiceState.channel_id === channelId && actualGuildId === expectedGuildId; - } - private initializeLocalAudioStateSync(): void { let previousState = { selfMute: LocalVoiceState.getSelfMute(), @@ -1708,9 +1112,6 @@ class MediaEngineFacade extends Store { private discardVoiceConnection(connectionId: string, options: {clearLocalState?: boolean} = {}): void { voiceEngineV2AppVoiceStateAdapter.removeVoiceStateConnection(connectionId); - for (const trackSid of NativeVideoTileManager.unregisterConnection(connectionId)) { - this.nativeVoiceFrameStatsBatcher.removeTrack(trackSid); - } this.voiceEngineV2Participants.discardConnection(connectionId); if (options.clearLocalState) { LocalVoiceState.clearConnectionState(connectionId); @@ -1771,11 +1172,6 @@ class MediaEngineFacade extends Store { hasRoom: room != null, hasVoiceTransport, }); - this.clearNativeVoiceTransportReconnect(); - this.clearNativeVoiceConnectRetry(); - this.clearNativeVoiceConnectSession(); - this.clearPendingNativeLocalMediaReconnect(); - this.stopNativeVoiceStatsSession(); voiceEngineV2AppScreenShareExecutionAdapter.stopNativeScreenShareForTerminalUnload(); const hasSpecificConnection = connectionId != null; GatewayConnection.sendTerminalVoiceDisconnect( @@ -2016,8 +1412,6 @@ class MediaEngineFacade extends Store { }); await this.disconnectForChannelMove('user'); } - this.clearNativeVoiceConnectRetry(); - this.clearNativeVoiceConnectSession(); if (!voiceEngineV2AppConnectionHostAdapter.startConnection(guildId, channelId)) return; const initialViewerStreamKeys = replaceWatchedStreamKeys(options.initialViewerStreamKeys ?? [], { sync: false, @@ -2083,11 +1477,7 @@ class MediaEngineFacade extends Store { } private async stopActiveScreenShareForTeardown(context: 'disconnect' | 'channel_move'): Promise { - if ( - !LocalVoiceState.getSelfStream() && - !voiceEngineV2AppScreenShareExecutionAdapter.hasActiveScreenShareResources() - ) - return; + if (!LocalVoiceState.getSelfStream()) return; try { await voiceEngineV2AppScreenShareExecutionAdapter.setScreenShareEnabled(this.room, false, { sendUpdate: false, @@ -2115,37 +1505,15 @@ class MediaEngineFacade extends Store { } private async runDisconnectFromVoiceChannel(reason: 'user' | 'error' | 'server'): Promise { - this.clearNativeVoiceTransportReconnect(); - this.clearNativeVoiceConnectRetry(); - this.clearNativeVoiceConnectSession(); - this.clearPendingNativeLocalMediaReconnect(); - this.stopNativeVoiceStatsSession(); - this.lastNativeVoiceServerUpdate = null; const {guildId, connectionId, connected, connecting, channelId} = voiceEngineV2AppConnectionHostAdapter.connectionState; if (!connected && !connecting && !channelId) { - if (this.nativeVoiceEngineV2BridgeEventDisposer) { - this.disposeNativeVoiceEngineBindings(); - try { - await requireNativeVoiceEngine().disconnect(); - } catch (error) { - logger.warn('Native voice engine disconnect failed', {error, reason}); - } - } return; } logger.debug('Voice teardown starting', {guildId, channelId, reason}); this.cancelPendingServerDisconnect(); this.transitionFacadeState({type: 'disconnect.cleanupStarted', reason}); await this.stopActiveScreenShareForTeardown('disconnect'); - if (this.nativeVoiceEngineV2BridgeEventDisposer) { - this.disposeNativeVoiceEngineBindings(); - try { - await requireNativeVoiceEngine().disconnect(); - } catch (error) { - logger.warn('Native voice engine disconnect failed', {error, reason}); - } - } this.clearViewerStreamKeys(); this.stopTracking(); this.statsHostAdapter.reset(); @@ -2176,12 +1544,6 @@ class MediaEngineFacade extends Store { } private async disconnectForChannelMove(reason: 'user' | 'server'): Promise { - this.clearNativeVoiceTransportReconnect(); - this.clearNativeVoiceConnectRetry(); - this.clearNativeVoiceConnectSession(); - this.clearPendingNativeLocalMediaReconnect(); - this.stopNativeVoiceStatsSession(); - this.lastNativeVoiceServerUpdate = null; const {guildId, connectionId, connected, connecting, channelId} = voiceEngineV2AppConnectionHostAdapter.connectionState; if (!connected && !connecting && !channelId) return; @@ -2321,914 +1683,10 @@ class MediaEngineFacade extends Store { } } - private finalizeNativeVoiceConnection( - guildId: string | null, - channelId: string, - connectionId: string | null, - source: 'connect-promise' | 'native-event', - attempt: VoiceEngineV2AppNativeVoiceConnectAttempt, - ): void { - if (!this.nativeVoiceConnectionLifecycle.isActiveAttempt(attempt)) { - logger.debug('Ignoring stale native voice engine connection finalization from replaced attempt', { - source, - connectionId, - attemptId: attempt.id, - activeAttemptId: this.nativeVoiceConnectionLifecycle.activeAttemptId, - }); - return; - } - if (connectionId && voiceEngineV2AppConnectionHostAdapter.connectionId !== connectionId) { - logger.warn('Ignoring stale native voice engine connection finalization', { - source, - connectionId, - currentConnectionId: voiceEngineV2AppConnectionHostAdapter.connectionId, - }); - return; - } - const alreadyReady = - this.nativeVoiceConnectionLifecycle.readyConnectionId === connectionId && - voiceEngineV2AppConnectionHostAdapter.connected && - !voiceEngineV2AppConnectionHostAdapter.connecting; - const suppressSelfJoinSound = this.pendingUserMove !== null; - if ( - !voiceEngineV2AppConnectionHostAdapter.connected || - voiceEngineV2AppConnectionHostAdapter.connecting || - voiceEngineV2AppConnectionHostAdapter.reconnecting - ) { - voiceEngineV2AppConnectionHostAdapter.markConnected(); - } - this.transitionFacadeState({type: 'connection.connected', guildId, channelId}); - if (alreadyReady) return; - this.nativeVoiceConnectionLifecycle.setReadyConnectionId(connectionId); - this.clearNativeVoiceTransportReconnect(); - this.clearNativeVoiceConnectRetry(); - startVoiceMediaGraphTimerScheduler(); - this.startNativeVoiceStatsSession(); - void voiceEngineV2AppDebugLoggingHostAdapter.start({ - guildId, - channelId, - connectionId, - room: null, - collectSnapshot: () => this.createVoiceDiagnosticsSnapshot(), - }); - this.audioPreferencesSyncDisposer?.(); - this.audioPreferencesSyncDisposer = this.bindNativeAudioPreferencesSync(); - this.videoCodecDecodeCapResyncDisposer?.(); - this.videoCodecDecodeCapResyncDisposer = this.bindVideoDecodeCapResync(); - this.videoCodecPublishOverrideSyncDisposer?.(); - this.videoCodecPublishOverrideSyncDisposer = this.bindVideoCodecPublishOverrideSync(); - VoiceEngineV2AppPermissionAdapter.syncWithNativePermissionState( - guildId, - channelId, - this.createNativePermissionEnforcement(), - ); - this.seedNativeParticipantsFromChannelVoiceStates(guildId, channelId); - this.navigateToVoiceChannel(guildId, channelId); - logger.info('Native voice engine connection ready', {guildId, channelId, connectionId, source}); - if (!hasPlayedNativeVoiceReadySounds(this.nativeVoiceReadySoundConnectionIds, connectionId)) { - rememberNativeVoiceReadySounds(this.nativeVoiceReadySoundConnectionIds, connectionId); - if (!suppressSelfJoinSound) { - playSelfJoinChimeOnce(connectionId, 'native-ready'); - } - void voiceEngineV2AppMediaExecutionAdapter.playEntranceSound(); - } - void ScreenShareCodecNegotiation.publishLocalCapabilitiesNative('connected'); - this.restoreNativeLocalMediaStateInBackground('native voice engine connected'); - voiceEngineV2AppMediaExecutionAdapter.resetMicrophoneFailureLatch(); - this.reconcileLocalAudioStateInBackground('native voice engine connected'); - } - - private getNativeVoiceConnectRetryKey(guildId: string | null, channelId: string, connectionId: string): string { - return `${guildId ?? 'dm'}:${channelId}:${connectionId}`; - } - - private async retryNativeVoiceEngineConnectAfterTimeout( - error: unknown, - raw: VoiceServerUpdateData, - guildId: string | null, - channelId: string, - connectionId: string | null, - attempt: VoiceEngineV2AppNativeVoiceConnectAttempt, - ): Promise { - if (!(error instanceof TimeoutError) || !connectionId) return false; - if (!this.isVoiceConnectionCurrentForNativeAttempt(attempt)) { - logger.debug('Skipping native voice engine connect timeout from stale attempt', { - guildId, - channelId, - connectionId, - attemptId: attempt.id, - activeAttemptId: this.nativeVoiceConnectionLifecycle.activeAttemptId, - currentConnectionId: voiceEngineV2AppConnectionHostAdapter.connectionId, - currentChannelId: voiceEngineV2AppConnectionHostAdapter.channelId, - }); - return true; - } - const voiceState = voiceEngineV2AppVoiceStateAdapter.getVoiceStateByConnectionId(connectionId); - if ( - !shouldRetryNativeVoiceConnectTimeout({ - connectionId, - guildId, - channelId, - voiceState, - connectionState: voiceEngineV2AppConnectionHostAdapter.connectionState, - }) - ) { - logger.warn('Native voice engine connect timeout not retried because voice connection is inactive', { - guildId, - channelId, - connectionId, - hasGatewayVoiceState: voiceState != null, - currentConnectionId: voiceEngineV2AppConnectionHostAdapter.connectionId, - currentChannelId: voiceEngineV2AppConnectionHostAdapter.channelId, - connecting: voiceEngineV2AppConnectionHostAdapter.connecting, - connected: voiceEngineV2AppConnectionHostAdapter.connected, - reconnecting: voiceEngineV2AppConnectionHostAdapter.reconnecting, - error, - }); - return false; - } - const retryKey = this.getNativeVoiceConnectRetryKey(guildId, channelId, connectionId); - const retryAttempt = (this.nativeVoiceConnectRetryCounts.get(retryKey) ?? 0) + 1; - if (retryAttempt > NATIVE_VOICE_ENGINE_CONNECT_MAX_LOCAL_RETRIES) { - logger.warn('Native voice engine connect timeout retry budget exhausted', { - guildId, - channelId, - connectionId, - retryAttempt: retryAttempt - 1, - maxRetries: NATIVE_VOICE_ENGINE_CONNECT_MAX_LOCAL_RETRIES, - error, - }); - return false; - } - this.nativeVoiceConnectRetryCounts.set(retryKey, retryAttempt); - const retryDelayMs = getNativeVoiceEngineConnectRetryDelayMs(retryAttempt); - logger.warn('Native voice engine connect timed out; retrying local connect', { - guildId, - channelId, - connectionId, - retryAttempt, - maxRetries: NATIVE_VOICE_ENGINE_CONNECT_MAX_LOCAL_RETRIES, - retryDelayMs, - timeoutMs: NATIVE_VOICE_ENGINE_CONNECT_TIMEOUT_MS, - error, - }); - this.clearNativeVoiceTransportReconnect(); - this.disposeNativeVoiceEngineBindings(false); - try { - await requireNativeVoiceEngine().disconnect(); - } catch (disconnectError) { - logger.warn('Native voice engine disconnect before connect retry failed; retrying anyway', { - guildId, - channelId, - connectionId, - retryAttempt, - disconnectError, - }); - } - if (voiceEngineV2AppConnectionHostAdapter.connectionId !== connectionId) { - logger.debug('Skipping native voice engine connect retry after connection changed', { - guildId, - channelId, - connectionId, - currentConnectionId: voiceEngineV2AppConnectionHostAdapter.connectionId, - }); - return true; - } - this.clearNativeVoiceConnectRetryTimeout(); - this.nativeVoiceConnectRetryTimeoutId = window.setTimeout(() => { - this.nativeVoiceConnectRetryTimeoutId = null; - if (voiceEngineV2AppConnectionHostAdapter.connectionId !== connectionId) { - logger.debug('Skipping stale native voice engine connect retry', { - guildId, - channelId, - connectionId, - currentConnectionId: voiceEngineV2AppConnectionHostAdapter.connectionId, - }); - return; - } - this.connectViaNativeEngine(raw, {forceReconnect: true, reason: 'connect-timeout-retry'}); - }, retryDelayMs); - return true; - } - - private async handleNativeVoiceEngineConnectFailure( - error: unknown, - guildId: string | null, - channelId: string, - connectionId: string | null, - raw: VoiceServerUpdateData, - attempt: VoiceEngineV2AppNativeVoiceConnectAttempt, - ): Promise { - if (!this.nativeVoiceConnectionLifecycle.isActiveAttempt(attempt)) { - logger.warn('Ignoring native voice engine connect failure from replaced attempt', { - guildId, - channelId, - connectionId, - attemptId: attempt.id, - activeAttemptId: this.nativeVoiceConnectionLifecycle.activeAttemptId, - error, - }); - return; - } - if ( - connectionId && - this.nativeVoiceConnectionLifecycle.readyConnectionId === connectionId && - voiceEngineV2AppConnectionHostAdapter.connected && - !voiceEngineV2AppConnectionHostAdapter.connecting - ) { - logger.warn('Ignoring native voice engine connect failure after connection was already finalized', { - guildId, - channelId, - connectionId, - error, - }); - return; - } - if (connectionId && voiceEngineV2AppConnectionHostAdapter.connectionId !== connectionId) { - logger.warn('Ignoring stale native voice engine connect failure', { - guildId, - channelId, - connectionId, - currentConnectionId: voiceEngineV2AppConnectionHostAdapter.connectionId, - error, - }); - return; - } - if (await this.retryNativeVoiceEngineConnectAfterTimeout(error, raw, guildId, channelId, connectionId, attempt)) { - return; - } - logger.error('Native voice engine connect failed', {guildId, channelId, connectionId, error}); - this.clearNativeVoiceTransportReconnect(); - this.clearNativeVoiceConnectRetry(); - this.clearNativeVoiceConnectSession(); - this.clearPendingNativeLocalMediaReconnect(); - this.disposeNativeVoiceEngineBindings(); - this.stopTracking(); - this.statsHostAdapter.reset(); - this.voiceEngineV2Participants.clear(); - if (connectionId) { - this.discardVoiceConnection(connectionId, {clearLocalState: true}); - sendVoiceStateDisconnect(guildId, connectionId); - } - try { - await requireNativeVoiceEngine().disconnect(); - } catch (disconnectError) { - logger.warn('Native voice engine disconnect after connect failure failed', {disconnectError}); - } - voiceEngineV2AppConnectionHostAdapter.markDisconnected('error'); - } - - private scheduleNativeVoiceTransportReconnect( - raw: VoiceServerUpdateData, - connectionId: string, - attempt: VoiceEngineV2AppNativeVoiceConnectAttempt, - ): void { - this.clearNativeVoiceTransportReconnect(); - this.nativeVoiceTransportReconnectTimeoutId = window.setTimeout(() => { - this.nativeVoiceTransportReconnectTimeoutId = null; - if ( - !this.nativeVoiceConnectionLifecycle.isActiveAttempt(attempt) || - voiceEngineV2AppConnectionHostAdapter.connectionId !== connectionId || - !voiceEngineV2AppConnectionHostAdapter.reconnecting || - !this.isGatewayVoiceStateActiveForConnection( - connectionId, - raw.guild_id ?? null, - raw.channel_id ?? voiceEngineV2AppConnectionHostAdapter.channelId ?? '', - ) - ) { - logger.debug('Skipping stale native voice engine transport reconnect', { - connectionId, - attemptId: attempt.id, - activeAttemptId: this.nativeVoiceConnectionLifecycle.activeAttemptId, - currentConnectionId: voiceEngineV2AppConnectionHostAdapter.connectionId, - reconnecting: voiceEngineV2AppConnectionHostAdapter.reconnecting, - }); - return; - } - this.connectViaNativeEngine(raw, {forceReconnect: true, reason: 'transport-reconnect'}); - }, NATIVE_VOICE_ENGINE_TRANSPORT_RECONNECT_DELAY_MS); - } - - private handleNativeVoiceEngineDisconnected( - guildId: string | null, - channelId: string, - attempt: VoiceEngineV2AppNativeVoiceConnectAttempt, - ): void { - if (!this.nativeVoiceConnectionLifecycle.isActiveAttempt(attempt)) { - logger.debug('Ignoring native voice engine disconnect from replaced attempt', { - guildId, - channelId, - attemptId: attempt.id, - activeAttemptId: this.nativeVoiceConnectionLifecycle.activeAttemptId, - }); - return; - } - const connectionId = voiceEngineV2AppConnectionHostAdapter.connectionId; - const lastServerUpdate = this.lastNativeVoiceServerUpdate; - if ( - connectionId && - lastServerUpdate?.connection_id === connectionId && - this.isGatewayVoiceStateActiveForConnection(connectionId, guildId, channelId) - ) { - logger.warn('Native voice engine transport disconnected while gateway voice state is still active', { - guildId, - channelId, - connectionId, - }); - this.prepareNativeLocalMediaReconnect('native voice engine transport disconnected'); - AdaptiveScreenShareEngine.stop(); - voiceEngineV2AppConnectionHostAdapter.markReconnecting(); - this.scheduleNativeVoiceTransportReconnect(lastServerUpdate, connectionId, attempt); - return; - } - void this.disconnectFromVoiceChannel('server'); - } - - private connectViaNativeEngine(raw: VoiceServerUpdateData, options: ConnectViaNativeEngineOptions = {}): void { - const reason = options.reason ?? 'server-update'; - const guildId = raw.guild_id ?? null; - const channelId = raw.channel_id ?? null; - const endpoint = raw.endpoint ?? null; - const token = raw.token ?? null; - const connectionId = raw.connection_id ?? null; - if (!endpoint || !token || !channelId) { - logger.warn('Native voice engine: ignoring VOICE_SERVER_UPDATE missing endpoint/token/channel', { - hasEndpoint: !!endpoint, - hasToken: !!token, - hasChannel: !!channelId, - }); - return; - } - if (this.voiceEngineV2Participants.isConnectionDiscarded(connectionId)) { - logger.warn('Native voice engine: ignoring VOICE_SERVER_UPDATE for discarded connection', { - guildId, - channelId, - connectionId, - }); - return; - } - if ( - !options.forceReconnect && - this.isDuplicateNativeVoiceServerUpdate({guildId, channelId, connectionId, endpoint, token}) - ) { - logger.debug('Native voice engine: ignoring duplicate VOICE_SERVER_UPDATE for active connection', { - guildId, - channelId, - connectionId, - endpoint, - connecting: voiceEngineV2AppConnectionHostAdapter.connecting, - connected: voiceEngineV2AppConnectionHostAdapter.connected, - reconnecting: voiceEngineV2AppConnectionHostAdapter.reconnecting, - activeAttemptId: this.nativeVoiceConnectionLifecycle.activeAttemptId, - }); - return; - } - this.seedLocalVoiceStateFromServerUpdate(raw); - if (!voiceEngineV2AppConnectionHostAdapter.acceptNativeVoiceServerUpdate(raw)) { - return; - } - this.lastNativeVoiceServerUpdate = raw; - this.clearNativeVoiceTransportReconnect(); - const attempt = this.nativeVoiceConnectionLifecycle.createAttempt({ - guildId, - channelId, - connectionId, - endpoint, - reason, - }); - try { - const currentVoiceState = raw.connection_id - ? voiceEngineV2AppVoiceStateAdapter.getVoiceStateByConnectionId(raw.connection_id) - : null; - if (currentVoiceState) { - this.seedNativeParticipantFromVoiceState(guildId, currentVoiceState); - } - const engine = requireNativeVoiceEngine(); - VoiceEngineV2AppSubscriptionAdapter.cleanup(); - VoiceEngineV2AppSubscriptionAdapter.setNativeEngine(engine); - this.nativeVoiceEngineV2BridgeEventDisposer?.(); - this.nativeVoiceEngineV2BridgeEventDisposer = engine.onEvent((event) => { - this.handleNativeVoiceEngineV2BridgeEvent(event, guildId, channelId, connectionId, attempt); - }); - this.bindNativeVoiceDataProtocols(engine); - this.startNativeVideoTileFrameSubscription('native-connect'); - this.nativeVoiceDeviceSyncDisposer?.(); - this.nativeVoiceDeviceSyncDisposer = bindNativeVoiceDeviceSync({ - engine, - getParticipants: () => this.voiceEngineV2Participants.participants, - subscribeParticipants: (listener) => this.voiceEngineV2ProjectionStore.subscribe(listener), - }); - logger.info('Native voice engine connect issuing', { - guildId, - channelId, - connectionId, - attemptId: attempt.id, - reason, - hasE2EE: !!raw.e2ee_key, - timeoutMs: NATIVE_VOICE_ENGINE_CONNECT_TIMEOUT_MS, - }); - void (async (): Promise => { - const readiness = await awaitNativeVoiceEngineReadiness(); - if (!readiness.ready) { - logger.warn('Native voice engine readiness was not confirmed before connect; dialing anyway', { - guildId, - channelId, - connectionId, - attemptId: attempt.id, - readinessReason: readiness.reason ?? null, - }); - } - const audioDeviceModuleStatus = await nativeAudioDeviceModuleState.ensureStatus(); - this.dispatchNativeAudioDeviceModuleStatus(audioDeviceModuleStatus); - await withTimeout( - engine.connect({url: endpoint, token, e2eeKey: encodeVoiceEngineE2EEKey(raw.e2ee_key)}), - NATIVE_VOICE_ENGINE_CONNECT_TIMEOUT_MS, - `Native voice engine connect timed out after ${NATIVE_VOICE_ENGINE_CONNECT_TIMEOUT_MS}ms`, - ); - })() - .then(() => { - this.finalizeNativeVoiceConnection(guildId, channelId, connectionId, 'connect-promise', attempt); - }) - .catch((error) => { - void this.handleNativeVoiceEngineConnectFailure(error, guildId, channelId, connectionId, raw, attempt); - }); - } catch (error) { - void this.handleNativeVoiceEngineConnectFailure(error, guildId, channelId, connectionId, raw, attempt); - } - } - - private applyNativeLocalTrackState( - source: VoiceTrackSource, - enabled: boolean, - eventName: string, - trackSid?: string, - participant?: NativeVoiceEngineLocalTrackParticipant, - publication?: NativeVoiceEngineLocalTrackPublication, - ): void { - const suppressed = this.shouldSuppressNativeLocalTrackState(source, enabled); - if (source === VoiceTrackSource.Camera) { - this.recordNativeVideoDiagnostic('local_track_state.received', { - source, - enabled, - eventName, - trackSid: trackSid ?? null, - participantSid: participant?.participantSid ?? null, - participantIdentity: participant?.participantIdentity ?? null, - publicationTrackSid: publication?.trackSid ?? null, - suppressed, - }); - } - if (suppressed) { - if (source === VoiceTrackSource.Camera) { - this.recordNativeVideoDiagnostic('local_track_state.suppressed', { - source, - enabled, - eventName, - trackSid: trackSid ?? null, - }); - } - logger.debug('Suppressing native local track state during reconnect restore', { - source, - enabled, - eventName, - }); - return; - } - if (source === VoiceTrackSource.ScreenShare) { - voiceEngineV2AppScreenShareExecutionAdapter.syncNativeEngineScreenSharePublishedTrackSidInternal( - enabled, - trackSid, - publication, - ); - voiceEngineV2AppMediaStateAdapter.applyScreenShareState(enabled, {sendUpdate: false}); - } else if (source === VoiceTrackSource.Camera) { - const cameraTrackSid = trackSid ?? publication?.trackSid; - if (enabled) { - this.registerNativeCameraLocalPreviewTrack(cameraTrackSid, participant); - } else { - this.clearNativeCameraLocalPreview(cameraTrackSid); - } - voiceEngineV2AppMediaStateAdapter.applyCameraState(enabled, {sendUpdate: false}); - this.syncVideoBackgroundFramePumpInBackground('native-camera-track-state'); - } else if (source === VoiceTrackSource.ScreenShareAudio) { - applyVoiceEngineV2NativeScreenShareAudioState(enabled); - } else { - logger.debug(`Native engine ${eventName}`, {source}); - } - } - - private readNativeCameraActualEnabled(): boolean { - try { - const enabled = requireNativeVoiceEngine().isPublishingCamera(); - assert.equal(typeof enabled, 'boolean', 'native camera publication state must be a boolean'); - return enabled; - } catch (error) { - logger.warn('Failed to read native camera publication state; falling back to local voice state', {error}); - return LocalVoiceState.getSelfVideo(); - } - } - - private get nativeVoiceEngineV2BridgeEventManagers(): NativeVoiceEngineV2BridgeEventManagers { - return { - participants: { - upsertParticipantFromNative: (fields) => this.voiceEngineV2Participants.upsertParticipantFromNative(fields), - patchParticipantTrackFlags: (identity, flags) => - this.voiceEngineV2Participants.patchParticipantTrackFlags(identity, flags), - setConnectionQualityForNative: (sid, quality) => - this.voiceEngineV2Participants.setConnectionQualityForNative(sid, quality), - updateActiveSpeakersBySid: (sids) => this.voiceEngineV2Participants.updateActiveSpeakersBySid(sids), - applyNativeSpeakingSample: (sample, nowMs) => - this.voiceEngineV2Participants.applyNativeSpeakingSample(sample, nowMs), - sweepNativeSpeakingHeartbeats: (nowMs) => this.voiceEngineV2Participants.sweepNativeSpeakingHeartbeats(nowMs), - removeParticipant: (identity) => this.voiceEngineV2Participants.removeParticipant(identity), - removeParticipantBySid: (sid) => this.voiceEngineV2Participants.removeParticipantBySid(sid), - }, - inboundVideo: { - registerTrack: (participantSid, trackSid, source, participantIdentity) => { - voiceEngineV2AppDebugLoggingHostAdapter.recordNativeVideoDiagnostic('track.register_requested', { - participantSid, - participantIdentity, - trackSid, - source, - reason: 'native-track-subscribed', - }); - NativeVideoTileManager.registerTrack(participantSid, trackSid, source, participantIdentity); - const registered = NativeVideoTileManager.tracks[trackSid] ?? null; - voiceEngineV2AppDebugLoggingHostAdapter.recordNativeVideoDiagnostic( - registered ? 'track.registered' : 'track.register_refused', - { - participantSid, - participantIdentity, - trackSid, - source, - reason: 'native-track-subscribed', - registeredWidth: registered?.width ?? null, - registeredHeight: registered?.height ?? null, - }, - ); - }, - unregisterTrack: (trackSid) => { - voiceEngineV2AppDebugLoggingHostAdapter.recordNativeVideoDiagnostic('track.unregister_requested', { - trackSid, - reason: 'native-track-unsubscribed', - }); - this.nativeVoiceFrameStatsBatcher.removeTrack(trackSid); - NativeVideoTileManager.unregisterTrack(trackSid); - }, - unregisterParticipant: (participantSid) => { - voiceEngineV2AppDebugLoggingHostAdapter.recordNativeVideoDiagnostic('participant.unregister_requested', { - participantSid, - }); - for (const track of NativeVideoTileManager.getTracksForParticipant(participantSid)) { - this.nativeVoiceFrameStatsBatcher.removeTrack(track.trackSid); - } - NativeVideoTileManager.unregisterParticipant(participantSid); - }, - }, - localMedia: { - onLocalTrackPublished: (source, trackSid, participant, publication) => - this.applyNativeLocalTrackState(source, true, 'localTrackPublished', trackSid, participant, publication), - onLocalTrackUnpublished: (source, trackSid, participant, publication) => - this.applyNativeLocalTrackState(source, false, 'localTrackUnpublished', trackSid, participant, publication), - onLocalTrackRepublished: (source, trackSid, participant, publication) => - this.applyNativeLocalTrackState(source, true, 'localTrackRepublished', trackSid, participant, publication), - }, - e2ee: { - setState: (sid, raw) => NativeVoiceE2EEStore.setState(sid, raw), - remove: (sid) => NativeVoiceE2EEStore.remove(sid), - }, - stats: { - setStats: (stats, timestampMs = Date.now()) => - NativeVoiceStatsStore.setStats(stats, timestampMs, {mergeSparseTrackStats: true}), - }, - }; - } - - private getNativeRemoteParticipantIdentities(): Array { - const identities: Array = []; - const participants = this.voiceEngineV2Participants.participants; - for (const participantIdentity in participants) { - const participant = participants[participantIdentity]; - if (!participant || participant.isLocal) continue; - identities.push(participant.identity); - } - return identities; - } - - private bindNativeVoiceDataProtocols(engine: VoiceEngine): void { - this.nativeVoiceDataProtocolDisposer?.(); - const codecDisposer = ScreenShareCodecNegotiation.bindNative( - { - publishData: (params) => engine.publishData(params), - getRemoteParticipantIdentities: () => this.getNativeRemoteParticipantIdentities(), - }, - { - onSelectedCodecChanged: (selection) => { - void voiceEngineV2AppScreenShareExecutionAdapter - .renegotiateActiveScreenShareCodec(null, selection.codec, selection.reason) - .catch((error) => { - logger.warn('Failed to apply negotiated native screen share codec', { - error, - codec: selection.codec, - reason: selection.reason, - }); - }); - }, - }, - ); - this.nativeVoiceDataProtocolDisposer = () => { - codecDisposer(); - ScreenShareCodecNegotiation.dispose(); - ScreenSharePublicationMigration.dispose(); - }; - } - - private stopNativeVoiceStatsSession(clearStats: boolean = true): void { - this.nativeVoiceStatsSession.stop(clearStats); - } - - private startNativeVoiceStatsSession(): void { - this.nativeVoiceStatsSession.start(); - } - - private ingestNativeVoiceStats(stats: VoiceEngineV2Stats, timestampMs: number): void { - ingestNativeVoiceEngineV2BridgeStats(stats, this.nativeVoiceEngineV2BridgeEventManagers, timestampMs); - } - - private recordNativeVideoDiagnostic(type: string, data?: Record): void { - voiceEngineV2AppDebugLoggingHostAdapter.recordNativeVideoDiagnostic(type, data); - } - - private recordNativeVideoFrame(frame: VoiceEngineV2BridgeVideoFrame): void { - voiceEngineV2AppDebugLoggingHostAdapter.recordNativeVideoFrame(frame); - this.nativeVoiceFrameStatsBatcher.recordFrame(frame); - } - - private startNativeVideoTileFrameSubscription(reason: string): boolean { - assert.ok(reason.length > 0, 'native video frame subscription reason must be non-empty'); - if (NativeVideoTileManager.isFrameSubscriptionActive) { - voiceEngineV2AppDebugLoggingHostAdapter.recordNativeVideoDiagnostic('frame_subscription.already_active', { - reason, - }); - return true; - } - const bridge = window.electron?.voiceEngine; - if (!bridge) { - voiceEngineV2AppDebugLoggingHostAdapter.recordNativeVideoDiagnostic('frame_subscription.bridge_missing', { - reason, - }); - logger.warn('Cannot subscribe to native video frames; bridge unavailable', {reason}); - return false; - } - NativeVideoTileManager.start(bridge, { - onFrame: (frame) => this.recordNativeVideoFrame(frame), - }); - voiceEngineV2AppDebugLoggingHostAdapter.recordNativeVideoDiagnostic('frame_subscription.started', {reason}); - return true; - } - - private seedNativeParticipantsFromChannelVoiceStates(guildId: string | null, channelId: string): void { - if (!isNativeVoiceEngineSelected()) return; - const channelVoiceStates = voiceEngineV2AppVoiceStateAdapter.getAllVoiceStatesInChannel(guildId ?? ME, channelId); - for (const connectionId in channelVoiceStates) { - const voiceState = channelVoiceStates[connectionId]; - if (!voiceState) continue; - this.seedNativeParticipantFromVoiceState(guildId, { - ...voiceState, - viewer_stream_keys: [...voiceState.viewer_stream_keys], - }); - } - } - - private seedNativeParticipantFromVoiceState(guildId: string | null, voiceState: VoiceState): void { - if (!isNativeVoiceEngineSelected()) return; - if (!voiceState.connection_id) return; - if (this.voiceEngineV2Participants.isConnectionDiscarded(voiceState.connection_id)) return; - const identity = buildVoiceParticipantIdentity(voiceState.user_id, voiceState.connection_id); - const rawIncomingGuildId = guildId ?? voiceState.guild_id ?? null; - const incomingGuildId = rawIncomingGuildId === ME ? null : rawIncomingGuildId; - const isCurrentVoiceContext = - voiceState.channel_id === voiceEngineV2AppConnectionHostAdapter.channelId && - incomingGuildId === (voiceEngineV2AppConnectionHostAdapter.guildId ?? null); - if (!voiceState.channel_id || !isCurrentVoiceContext) { - if (this.voiceEngineV2Participants.getParticipant(identity)) { - this.voiceEngineV2Participants.removeParticipant(identity); - } - return; - } - const isLocal = - voiceState.user_id === Users.getCurrentUser()?.id && - voiceState.connection_id === voiceEngineV2AppConnectionHostAdapter.connectionId; - this.voiceEngineV2Participants.upsertParticipantFromNative({ - identity, - sid: this.voiceEngineV2Participants.getParticipant(identity)?.sid ?? '', - isLocal, - isMicrophoneEnabled: !voiceState.self_mute && !voiceState.mute, - isCameraEnabled: voiceState.self_video, - isScreenShareEnabled: voiceState.self_stream, - }); - } - - private bytesFromNativeDataPayload(payload: Record): Uint8Array | null { - const bytes = payload.payloadBytes; - if (!Array.isArray(bytes)) return null; - const output = new Uint8Array(bytes.length); - for (let i = 0; i < bytes.length; i++) { - const value = bytes[i]; - if (typeof value !== 'number' || !Number.isInteger(value) || value < 0 || value > 255) return null; - output[i] = value; - } - return output; - } - - private getNativeVoiceEventConnectionId(event: VoiceEngineV2BridgeEvent): string | null { - const payload = event.payload as Record; - const identity = - typeof payload.identity === 'string' - ? payload.identity - : typeof payload.participantIdentity === 'string' - ? payload.participantIdentity - : null; - return identity ? this.voiceEngineV2Participants.extractConnectionId(identity) : null; - } - - private shouldIgnoreNativeVoiceEventForDiscardedConnection(event: VoiceEngineV2BridgeEvent): boolean { - const connectionId = this.getNativeVoiceEventConnectionId(event); - if (!this.voiceEngineV2Participants.isConnectionDiscarded(connectionId)) return false; - logger.debug('Ignoring native voice engine event for discarded connection', {type: event.type, connectionId}); - return true; - } - - private handleNativeDataReceivedEvent(event: VoiceEngineV2BridgeEvent): boolean { - if (event.type !== 'dataReceived') return false; - const payload = event.payload as Record; - const participantIdentity = typeof payload.identity === 'string' ? payload.identity : null; - const topic = typeof payload.topic === 'string' ? payload.topic : null; - const bytes = this.bytesFromNativeDataPayload(payload); - if (!participantIdentity || !topic || !bytes) return true; - if (topic === SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC) { - ScreenShareCodecNegotiation.handleNativeDataMessage(participantIdentity, bytes); - } else if (topic === SCREEN_SHARE_PUBLICATION_MIGRATION_TOPIC) { - ScreenSharePublicationMigration.handleNativeDataMessage(participantIdentity, bytes); - } else if (topic === VOICE_ENGINE_V2_CODEC_GOSSIP_TOPIC) { - ingestVoiceEngineV2CodecGossip(this.voiceEngineV2Controller, participantIdentity, bytes); - } - return true; - } - - private handleNativeParticipantProtocolEvent(event: VoiceEngineV2BridgeEvent): void { - const payload = event.payload as Record; - const identity = typeof payload.identity === 'string' ? payload.identity : null; - if (event.type === 'participantJoined') { - ScreenShareCodecNegotiation.handleNativeParticipantConnected(); - } else if (event.type === 'participantLeft' && identity) { - ScreenShareCodecNegotiation.handleNativeParticipantDisconnected(identity); - } - } - - private handleNativeRemoteTrackPublicationEvent(event: VoiceEngineV2BridgeEvent): void { - if (event.type !== 'trackPublished') return; - const payload = event.payload as Record; - const identity = typeof payload.identity === 'string' ? payload.identity : null; - if (!identity) return; - const source = asVoiceTrackSource(payload.source); - if (source === VoiceTrackSource.Camera) { - VoiceEngineV2AppSubscriptionAdapter.reattachVideoAfterPublish(identity); - } else if (source === VoiceTrackSource.ScreenShare || source === VoiceTrackSource.ScreenShareAudio) { - VoiceEngineV2AppSubscriptionAdapter.reattachScreenShareAfterPublish(identity); - } - } - - private reattachNativeRemoteTrackSubscriptionsFromConnectedRoster(payload: Record): void { - for (const track of collectNativeVoiceEngineConnectedRosterPublishedTracks(payload)) { - if (track.source === VoiceTrackSource.Camera) { - VoiceEngineV2AppSubscriptionAdapter.reattachVideoAfterPublish(track.identity); - } else if (track.source === VoiceTrackSource.ScreenShare || track.source === VoiceTrackSource.ScreenShareAudio) { - VoiceEngineV2AppSubscriptionAdapter.reattachScreenShareAfterPublish(track.identity); - } - } - } - - private handleNativeVoiceEngineV2BridgeEvent( - event: VoiceEngineV2BridgeEvent, - guildId: string | null, - channelId: string, - connectionId: string | null, - attempt: VoiceEngineV2AppNativeVoiceConnectAttempt, - ): void { - if (!this.nativeVoiceConnectionLifecycle.isActiveAttempt(attempt)) { - logger.debug('Ignoring native voice engine event from replaced attempt', { - type: event.type, - connectionId, - attemptId: attempt.id, - activeAttemptId: this.nativeVoiceConnectionLifecycle.activeAttemptId, - }); - return; - } - voiceEngineV2AppDebugLoggingHostAdapter.recordNativeEngineEvent(event); - if (isFacadeOwnedConnectionEvent(event.type)) { - if (connectionId && voiceEngineV2AppConnectionHostAdapter.connectionId !== connectionId) { - logger.debug('Ignoring stale native voice engine connection event', { - type: event.type, - connectionId, - currentConnectionId: voiceEngineV2AppConnectionHostAdapter.connectionId, - }); - return; - } - const action = getNativeVoiceEngineConnectionEventAction(event); - switch (action) { - case 'connected': - applyNativeVoiceEngineConnectedRoster( - event.payload as Record, - this.nativeVoiceEngineV2BridgeEventManagers, - ); - this.reattachNativeRemoteTrackSubscriptionsFromConnectedRoster(event.payload as Record); - this.finalizeNativeVoiceConnection(guildId, channelId, connectionId, 'native-event', attempt); - break; - case 'disconnected': - this.handleNativeVoiceEngineDisconnected(guildId, channelId, attempt); - break; - case 'reconnecting': - this.prepareNativeLocalMediaReconnect('native voice engine reconnecting'); - AdaptiveScreenShareEngine.stop(); - this.statsHostAdapter.stopLatencyTracking(); - this.statsHostAdapter.stopStatsTracking(); - voiceEngineV2AppConnectionHostAdapter.markReconnecting(); - break; - case 'reconnected': - this.clearNativeVoiceTransportReconnect(); - this.clearNativeVoiceConnectRetry(); - this.nativeVoiceConnectionLifecycle.setReadyConnectionId(connectionId); - voiceEngineV2AppConnectionHostAdapter.markReconnected(); - this.statsHostAdapter.incrementReconnectionCount(); - this.statsHostAdapter.startLatencyTracking(); - this.statsHostAdapter.startStatsTracking(); - void ScreenShareCodecNegotiation.publishLocalCapabilitiesNative('reconnected'); - this.restoreNativeLocalMediaStateInBackground('native voice engine reconnected'); - voiceEngineV2AppMediaExecutionAdapter.resetMicrophoneFailureLatch(); - this.reconcileLocalAudioStateInBackground('native voice engine reconnected'); - break; - default: - logger.debug('Native voice engine connection event ignored', { - type: event.type, - state: (event.payload as Record).state, - }); - break; - } - return; - } - if (this.shouldIgnoreNativeVoiceEventForDiscardedConnection(event)) return; - if (this.handleNativeDataReceivedEvent(event)) return; - this.dispatchNativeVoiceEngineV2BridgeEvent(event); - mapNativeVoiceEngineV2BridgeEvent(event, this.nativeVoiceEngineV2BridgeEventManagers); - this.handleNativeParticipantProtocolEvent(event); - this.handleNativeRemoteTrackPublicationEvent(event); - } - - private dispatchNativeVoiceEngineV2BridgeEvent(event: VoiceEngineV2BridgeEvent): void { - const translatedEvents = translateVoiceEngineV2BridgeEventToEvents(event); - const eventCount = translatedEvents.length; - for (let eventIndex = 0; eventIndex < eventCount; eventIndex += 1) { - const translatedEvent = translatedEvents[eventIndex]; - if (!translatedEvent) continue; - this.voiceEngineV2Host.dispatch(translatedEvent); - } - } - - private dispatchNativeAudioDeviceModuleStatus(status: NativeAudioDeviceModuleStatus): void { - assert.equal(typeof status, 'string', 'native audio device module status must be a string'); - this.voiceEngineV2Host.dispatch({ - type: 'nativeAudioDeviceModule.statusChanged', - status, - }); - } - handleVoiceServerUpdate(raw: VoiceServerUpdateData): void { if (raw.connection_id && raw.connection_id === this.pendingServerDisconnectConnectionId) { this.cancelPendingServerDisconnect(); } - if (isNativeVoiceEngineSelected()) { - this.connectViaNativeEngine(raw); - return; - } - if (isNativeVoiceEngineSelectionPending()) { - void this.handleVoiceServerUpdateAfterNativeProbe(raw); - return; - } - this.handleVoiceServerUpdateViaJs(raw); - } - - private async handleVoiceServerUpdateAfterNativeProbe(raw: VoiceServerUpdateData): Promise { - try { - if (await shouldUseNativeVoiceEngine()) { - this.connectViaNativeEngine(raw); - return; - } - } catch (error) { - logger.warn('Native voice engine probe failed while handling VOICE_SERVER_UPDATE', {error}); - voiceEngineV2AppConnectionHostAdapter.resetConnectionState(); - return; - } this.handleVoiceServerUpdateViaJs(raw); } @@ -3409,7 +1867,6 @@ class MediaEngineFacade extends Store { guild_id: voiceState.guild_id ?? guildId, }; voiceEngineV2AppVoiceStateAdapter.handleGatewayVoiceStateUpdate(guildId, stateWithGuild); - this.seedNativeParticipantFromVoiceState(guildId, stateWithGuild); } } @@ -3485,7 +1942,6 @@ class MediaEngineFacade extends Store { } : voiceState; voiceEngineV2AppVoiceStateAdapter.handleGatewayVoiceStateUpdate(guildId, projectedVoiceState); - this.seedNativeParticipantFromVoiceState(guildId, projectedVoiceState); if (!isLocalConnection && voiceEngineV2AppConnectionHostAdapter.connected && voiceState.channel_id) { if (!areOrderedStringArraysEqual(previousViewerStreamKeys, incomingViewerStreamKeys)) { this.playSpectatorSounds(previousViewerStreamKeys, incomingViewerStreamKeys); @@ -3787,11 +2243,6 @@ class MediaEngineFacade extends Store { channelId, ); const serverDeaf = serverVoiceState?.deaf ?? false; - if (await shouldUseNativeVoiceEngine()) { - await this.reconcileNativeEngineMicState({channelId, serverMute, serverDeaf}); - this.syncLocalVoiceStateWithServer(); - return; - } await voiceEngineV2AppMediaExecutionAdapter.reconcileEffectiveAudioState( voiceEngineV2AppConnectionHostAdapter.room, { @@ -3803,38 +2254,6 @@ class MediaEngineFacade extends Store { this.syncLocalVoiceStateWithServer(); } - private createNativePermissionEnforcement(): VoiceEngineV2AppNativePermissionEnforcement { - return { - revokeMicrophone: () => this.reconcileLocalAudioState('speak permission revoked'), - revokeCamera: () => this.setCameraEnabled(false, {sendUpdate: true}), - revokeScreenShare: () => this.setScreenShareEnabled(false, {sendUpdate: true, playSound: false}), - }; - } - - private async reconcileNativeEngineMicState(params: { - channelId: string; - serverMute: boolean; - serverDeaf: boolean; - }): Promise { - const permissionMuted = isVoiceSpeakPermissionDenied( - voiceEngineV2AppConnectionHostAdapter.guildId, - params.channelId, - ); - const audioState = getEffectiveAudioState({ - serverMute: params.serverMute || permissionMuted, - serverDeaf: params.serverDeaf, - }); - const pttActive = this.isPushToTalkActive(); - const ptmActive = this.isPushToMuteActive(); - const effectiveMute = - audioState.effectiveDeaf || - audioState.serverMute || - audioState.selfMute || - (pttActive && !Keybind.pushToTalkHeld) || - (ptmActive && Keybind.pushToMuteHeld); - await this.setMicEnabledViaEngine(!effectiveMute); - } - getParticipantByUserIdAndConnectionId( userId: string, connectionId: string | null, @@ -3891,571 +2310,14 @@ class MediaEngineFacade extends Store { async refreshMicrophonePublishSettingsForChannel(channelId: string): Promise { assert.ok(channelId.length > 0, 'microphone publish settings refresh requires a channelId'); if (channelId !== voiceEngineV2AppConnectionHostAdapter.channelId) return; - if (await shouldUseNativeVoiceEngine()) { - await this.refreshNativeMicrophonePublishSettingsForChannel(channelId); - return; - } await voiceEngineV2AppMediaExecutionAdapter.refreshMicrophonePublishSettings( voiceEngineV2AppConnectionHostAdapter.room, channelId, ); } - private async refreshNativeMicrophonePublishSettingsForChannel(channelId: string): Promise { - assert.ok(channelId.length > 0, 'native microphone publish settings refresh requires a channelId'); - assert.equal( - channelId, - voiceEngineV2AppConnectionHostAdapter.channelId, - 'native microphone publish settings refresh requires the active voice channel', - ); - const microphone = this.voiceEngineV2Snapshot.microphone; - if (microphone.status !== 'published') { - logger.debug('Skipping native microphone bitrate refresh because the microphone is not published', { - channelId, - status: microphone.status, - }); - return; - } - const publishedOptions = microphone.published as VoiceEngineV2NativeMicrophonePublishOptions | null; - const publishedMaxBitrateBps = publishedOptions?.maxBitrateBps; - const nextMaxBitrateBps = resolveVoiceEngineV2NativeMicrophoneMaxBitrateBps( - Channels.getChannel(channelId)?.bitrate ?? null, - ); - if (nextMaxBitrateBps === publishedMaxBitrateBps) { - logger.debug('Skipping native microphone bitrate refresh because the effective bitrate is unchanged', { - channelId, - maxBitrateBps: publishedMaxBitrateBps ?? null, - }); - return; - } - logger.info('Refreshing native microphone publish settings for channel bitrate change', { - channelId, - previousMaxBitrateBps: publishedMaxBitrateBps ?? null, - nextMaxBitrateBps: nextMaxBitrateBps ?? null, - }); - await this.refreshNativeMicrophoneFromSettings(); - } - - private async getNativeCameraPublishParams(options?: {deviceId?: string}): Promise<{ - deviceId?: string; - width: number; - height: number; - frameRate: number; - mirror?: boolean; - backgroundMode?: 'none' | 'blur' | 'custom'; - backgroundCustomMediaPath?: string; - backgroundCustomMediaKind?: 'static' | 'animated' | 'video'; - backgroundBlurStrength: number; - }> { - const preset = getCameraCaptureDimensions(VoiceSettings.getCameraResolution()); - const deviceId = options?.deviceId ?? VoiceSettings.getVideoDeviceId(); - const nativeDeviceId = await resolveNativeCameraDeviceId(deviceId); - const backgroundImageId = areVoiceBackgroundsAvailable() - ? VoiceSettings.getBackgroundImageId() - : NONE_BACKGROUND_ID; - const backgroundMode = - backgroundImageId === BLUR_BACKGROUND_ID ? 'blur' : backgroundImageId === NONE_BACKGROUND_ID ? 'none' : 'custom'; - const customMedia = - backgroundMode === 'custom' ? await BackgroundImageDB.getNativeBackgroundMediaSource(backgroundImageId) : null; - if (backgroundMode === 'custom' && !customMedia) { - throw new Error(`Selected native camera background media is missing: ${backgroundImageId}`); - } - return { - ...preset, - mirror: VoiceSettings.getMirrorCamera(), - backgroundMode: customMedia || backgroundMode !== 'custom' ? backgroundMode : 'none', - ...(customMedia - ? { - backgroundCustomMediaPath: customMedia.path, - backgroundCustomMediaKind: customMedia.mediaKind, - } - : {}), - ...(nativeDeviceId ? {deviceId: nativeDeviceId} : {}), - backgroundBlurStrength: VoiceSettings.getBackgroundBlurStrength(), - }; - } - - private async getNativeDeviceScreenShareCaptureOptions( - options?: DeviceScreenShareCaptureOptions, - ): Promise { - const nativeVideoDeviceId = await resolveNativeCameraDeviceId(options?.videoDeviceId); - const {videoDeviceId: _videoDeviceId, ...rest} = options ?? {}; - return { - ...rest, - ...(options?.videoDeviceId ? {previewVideoDeviceId: options.videoDeviceId} : {}), - ...(nativeVideoDeviceId ? {videoDeviceId: nativeVideoDeviceId} : {}), - }; - } - - private clearNativeCameraLocalPreviewTrack(trackSid: string): void { - assert.ok(trackSid.length > 0, 'native camera preview trackSid must be non-empty before clearing'); - this.recordNativeVideoDiagnostic('camera_preview.local_cleared', { - trackSid, - }); - this.nativeVoiceFrameStatsBatcher.removeTrack(trackSid); - NativeVideoTileManager.unregisterTrack(trackSid); - } - - private clearNativeCameraLocalPreview(trackSidHint?: string): void { - const trackSid = this.nativeCameraPreviewTrackSid; - assert.ok(trackSid === null || trackSid.length > 0, 'native camera preview trackSid must be null or non-empty'); - this.nativeCameraPreviewTrackSid = null; - if (trackSid) { - this.clearNativeCameraLocalPreviewTrack(trackSid); - } - if (trackSidHint && trackSidHint !== trackSid && trackSidHint !== this.nativeCameraPreviewSessionTrackSid) { - this.clearNativeCameraLocalPreviewTrack(trackSidHint); - } - assert.equal(this.nativeCameraPreviewTrackSid, null, 'native camera preview must be cleared'); - } - - private shouldAcceptNativeCameraPreviewParticipant(identity: string | undefined): boolean { - if (!identity) return true; - const connectionId = this.voiceEngineV2Participants.extractConnectionId(identity); - if (this.voiceEngineV2Participants.isConnectionDiscarded(connectionId)) return false; - const currentConnectionId = voiceEngineV2AppConnectionHostAdapter.connectionId; - if (!connectionId || !currentConnectionId) return true; - return connectionId === currentConnectionId; - } - - private resolveNativeCameraPreviewParticipant( - participant?: NativeVoiceEngineLocalTrackParticipant, - ): NativeCameraPreviewParticipant | null { - if (!this.shouldAcceptNativeCameraPreviewParticipant(participant?.participantIdentity)) { - logger.debug('Ignoring native camera preview participant from stale connection', { - participantIdentity: participant?.participantIdentity, - currentConnectionId: voiceEngineV2AppConnectionHostAdapter.connectionId, - }); - return null; - } - if (participant?.participantSid) { - return { - ...(participant.participantIdentity ? {identity: participant.participantIdentity} : {}), - sid: participant.participantSid, - }; - } - if (participant?.participantIdentity) { - return { - identity: participant.participantIdentity, - sid: participant.participantIdentity, - }; - } - const localParticipant = this.voiceEngineV2Participants.getLocalParticipant(); - if (localParticipant?.identity) { - return { - identity: localParticipant.identity, - sid: localParticipant.sid || localParticipant.identity, - }; - } - const currentUserId = Users.getCurrentUser()?.id; - const connectionId = voiceEngineV2AppConnectionHostAdapter.connectionId; - if (!currentUserId || !connectionId) return null; - const identity = buildVoiceParticipantIdentity(currentUserId, connectionId); - return { - identity, - sid: identity, - }; - } - - private recordNativeCameraPreviewLocalRegisterRequest( - trackSid: string | undefined, - participant?: NativeVoiceEngineLocalTrackParticipant, - ): void { - this.recordNativeVideoDiagnostic('camera_preview.local_register_requested', { - trackSid: trackSid ?? null, - participantSid: participant?.participantSid ?? null, - participantIdentity: participant?.participantIdentity ?? null, - }); - } - - private recordNativeCameraPreviewLocalRegisterRefused( - reason: string, - trackSid: string | null, - details: Record = {}, - ): void { - this.recordNativeVideoDiagnostic('camera_preview.local_register_refused', { - trackSid, - reason, - ...details, - }); - } - - private isNativeCameraPreviewLocalTrackCurrent( - trackSid: string, - localParticipant: NativeCameraPreviewParticipant, - registered: NativeInboundVideoTrack | undefined, - ): registered is NativeInboundVideoTrack { - return ( - this.nativeCameraPreviewTrackSid === trackSid && - registered != null && - registered.participantSid === localParticipant.sid && - registered.participantIdentity === localParticipant.identity - ); - } - - private isNativeCameraPreviewTrackOwner( - registered: NativeInboundVideoTrack, - localParticipant: NativeCameraPreviewParticipant, - ): boolean { - return ( - registered.participantSid === localParticipant.sid && registered.participantIdentity === localParticipant.identity - ); - } - - private registerNativeCameraPreviewTrackWithTileManager( - trackSid: string, - localParticipant: NativeCameraPreviewParticipant, - ): NativeInboundVideoTrack | null { - this.clearNativeCameraLocalPreview(); - NativeVideoTileManager.registerTrack( - localParticipant.sid, - trackSid, - VoiceTrackSource.Camera, - localParticipant.identity, - ); - return NativeVideoTileManager.tracks[trackSid] ?? null; - } - - private recordNativeCameraPreviewLocalRegistered( - trackSid: string, - localParticipant: NativeCameraPreviewParticipant, - registered: NativeInboundVideoTrack, - ): void { - this.recordNativeVideoDiagnostic('camera_preview.local_registered', { - trackSid, - participantSid: localParticipant.sid, - participantIdentity: localParticipant.identity, - width: registered.width, - height: registered.height, - }); - } - - private registerNativeCameraLocalPreviewTrack( - trackSid: string | undefined, - participant?: NativeVoiceEngineLocalTrackParticipant, - ): void { - this.recordNativeCameraPreviewLocalRegisterRequest(trackSid, participant); - if (!trackSid) { - this.recordNativeCameraPreviewLocalRegisterRefused('missing-track-sid', null); - logger.warn('Cannot attach native camera preview without native track SID'); - return; - } - assert.ok(trackSid.length > 0, 'native camera preview trackSid must be non-empty'); - const localParticipant = this.resolveNativeCameraPreviewParticipant(participant); - if (!localParticipant) { - this.recordNativeCameraPreviewLocalRegisterRefused('missing-local-participant', trackSid); - logger.warn('Cannot attach native camera preview without local participant'); - return; - } - assert.ok(localParticipant.sid.length > 0, 'native camera preview participant sid must be non-empty'); - const registered = NativeVideoTileManager.tracks[trackSid]; - if (this.isNativeCameraPreviewLocalTrackCurrent(trackSid, localParticipant, registered)) { - this.recordNativeVideoDiagnostic('camera_preview.local_already_registered', { - trackSid, - participantSid: localParticipant.sid, - participantIdentity: localParticipant.identity, - width: registered.width, - height: registered.height, - }); - return; - } - const confirmed = this.registerNativeCameraPreviewTrackWithTileManager(trackSid, localParticipant); - if (confirmed == null) { - this.recordNativeCameraPreviewLocalRegisterRefused('tile-manager-refused', trackSid, { - participantSid: localParticipant.sid, - participantIdentity: localParticipant.identity, - }); - logger.warn('Native camera preview registration was refused', {trackSid}); - return; - } - if (!this.isNativeCameraPreviewTrackOwner(confirmed, localParticipant)) { - this.recordNativeCameraPreviewLocalRegisterRefused('owner-conflict', trackSid, { - participantSid: localParticipant.sid, - participantIdentity: localParticipant.identity, - registeredParticipantSid: confirmed.participantSid, - registeredParticipantIdentity: confirmed.participantIdentity, - }); - logger.warn('Native camera preview registration conflicted with an existing track owner', { - trackSid, - registeredParticipantSid: confirmed.participantSid, - registeredParticipantIdentity: confirmed.participantIdentity, - expectedParticipantSid: localParticipant.sid, - expectedParticipantIdentity: localParticipant.identity, - }); - return; - } - this.nativeCameraPreviewTrackSid = trackSid; - this.recordNativeCameraPreviewLocalRegistered(trackSid, localParticipant, confirmed); - assert.equal(this.nativeCameraPreviewTrackSid, trackSid, 'native camera preview must record the registered track'); - } - - private async publishNativeCameraFromSettings(options?: {deviceId?: string}): Promise { - await requireNativeVoiceEngine().publishCamera(await this.getNativeCameraPublishParams(options)); - } - async updateActiveCameraCapture(options?: {deviceId?: string}): Promise { - if (!(await shouldUseNativeVoiceEngine()) || !this.readNativeCameraActualEnabled()) { - await this.setCameraEnabled(true, options); - return; - } - try { - await requireNativeVoiceEngine().updateCameraCapture(await this.getNativeCameraPublishParams(options)); - } catch (error) { - logger.warn('Native camera device hot update failed; falling back to camera republish', {error}); - await this.setCameraEnabled(true, options); - } - this.syncVideoBackgroundFramePumpInBackground('camera-capture-update'); - } - - private getVideoBackgroundFramePump(): VideoBackgroundFramePump | null { - if (this.videoBackgroundFramePump) return this.videoBackgroundFramePump; - const bridge = window.electron?.voiceEngine; - if (!bridge) return null; - this.videoBackgroundFramePump = new VideoBackgroundFramePump({ - bridge, - getCaptureDimensions: () => getCameraCaptureDimensions(VoiceSettings.getCameraResolution()), - }); - return this.videoBackgroundFramePump; - } - - private getSelectedVideoBackgroundId(): string | null { - if (!areVoiceBackgroundsAvailable()) return null; - const backgroundImageId = VoiceSettings.getBackgroundImageId(); - if (backgroundImageId === NONE_BACKGROUND_ID || backgroundImageId === BLUR_BACKGROUND_ID) return null; - const image = VoiceSettings.getBackgroundImages().find((entry) => entry.id === backgroundImageId); - return image?.mediaKind === 'video' ? backgroundImageId : null; - } - - private shouldRunVideoBackgroundFramePump(): boolean { - if (!isNativeVoiceEngineSelected()) return false; - const cameraActive = this.readNativeCameraActualEnabled() || this.nativeCameraPreviewSessionTrackSid !== null; - if (!cameraActive) return false; - return this.getSelectedVideoBackgroundId() !== null; - } - - private async syncVideoBackgroundFramePump(reason: string): Promise { - const pump = this.getVideoBackgroundFramePump(); - if (!pump) return; - const backgroundId = this.shouldRunVideoBackgroundFramePump() ? this.getSelectedVideoBackgroundId() : null; - if (backgroundId) { - const started = await pump.start(backgroundId); - if (!started) { - logger.warn('Video background frame pump could not start', {reason, backgroundId}); - } - return; - } - if (pump.isRunning()) { - await pump.stop(); - } - } - - private syncVideoBackgroundFramePumpInBackground(reason: string): void { - void this.syncVideoBackgroundFramePump(reason).catch((error) => { - logger.warn('Failed to sync video background frame pump', {reason, error}); - }); - } - - isNativeCameraPreviewSessionAvailable(): boolean { - if (!isNativeVoiceEngineSelected()) return false; - const capabilities = getNativeVoiceEngineCapabilitiesSnapshot(); - if (capabilities == null) return true; - return capabilities.cameraCapture === true; - } - - private async resolveNativeCameraPreviewCapability(): Promise { - if (!isNativeVoiceEngineSelected()) return false; - const cached = getNativeVoiceEngineCapabilitiesSnapshot(); - if (cached != null) return cached.cameraCapture === true; - try { - const live = await refreshNativeVoiceEngineCapabilitiesSnapshot(); - return live?.cameraCapture === true; - } catch (error) { - logger.warn('Failed to query native voice engine capabilities for camera preview', {error}); - return false; - } - } - - private ensureNativeVideoTileFrameSubscription(): void { - this.startNativeVideoTileFrameSubscription('camera-preview'); - } - - private registerNativeCameraPreviewSessionTrack(trackSid: string): boolean { - assert.ok(trackSid.length > 0, 'native camera preview session trackSid must be non-empty'); - this.recordNativeVideoDiagnostic('camera_preview.session_register_requested', {trackSid}); - const previousTrackSid = this.nativeCameraPreviewSessionTrackSid; - if (previousTrackSid && previousTrackSid !== trackSid) { - this.recordNativeVideoDiagnostic('camera_preview.session_previous_cleared', { - trackSid: previousTrackSid, - nextTrackSid: trackSid, - }); - this.nativeVoiceFrameStatsBatcher.removeTrack(previousTrackSid); - NativeVideoTileManager.unregisterTrack(previousTrackSid); - } - NativeVideoTileManager.registerTrack(trackSid, trackSid, VoiceTrackSource.Camera); - const registered = NativeVideoTileManager.tracks[trackSid]; - if (!registered) { - this.recordNativeVideoDiagnostic('camera_preview.session_register_refused', { - trackSid, - reason: 'tile-manager-refused', - }); - logger.warn('Native camera preview session registration was refused', {trackSid}); - return false; - } - if (registered.participantSid !== trackSid || registered.source !== VoiceTrackSource.Camera) { - this.recordNativeVideoDiagnostic('camera_preview.session_register_refused', { - trackSid, - reason: 'owner-conflict', - registeredParticipantSid: registered.participantSid, - registeredSource: registered.source, - }); - logger.warn('Native camera preview session registration resolved to a conflicting track', { - trackSid, - registeredParticipantSid: registered.participantSid, - registeredSource: registered.source, - }); - return false; - } - this.nativeCameraPreviewSessionTrackSid = trackSid; - assert.equal( - this.nativeCameraPreviewSessionTrackSid, - trackSid, - 'native camera preview session must record the registered track', - ); - this.recordNativeVideoDiagnostic('camera_preview.session_registered', { - trackSid, - width: registered.width, - height: registered.height, - }); - return true; - } - - private async startNativeCameraPreviewSessionNow( - options: {deviceId?: string} | undefined, - generation: number, - ): Promise { - assert.ok( - this.nativeCameraPreviewStartGate.isCurrent(generation), - 'native camera preview start must begin on the current generation', - ); - if (!(await this.resolveNativeCameraPreviewCapability())) { - return null; - } - const engine = requireNativeVoiceEngine(); - const params = await this.getNativeCameraPublishParams(options); - if (!this.nativeCameraPreviewStartGate.isCurrent(generation)) { - return null; - } - const info = await engine.startCameraPreview(params); - assert.ok(info.trackSid.length > 0, 'native camera preview session trackSid must be non-empty'); - if (!this.nativeCameraPreviewStartGate.isCurrent(generation)) { - await engine.stopCameraPreview().catch((error) => { - logger.warn('Failed to stop superseded native camera preview session', {error}); - }); - return null; - } - this.ensureNativeVideoTileFrameSubscription(); - if (!this.registerNativeCameraPreviewSessionTrack(info.trackSid)) { - await engine.stopCameraPreview().catch((error) => { - logger.warn('Failed to stop unregistered native camera preview session', {error}); - }); - return null; - } - this.syncVideoBackgroundFramePumpInBackground('camera-preview-session-start'); - return info.trackSid; - } - - async startNativeCameraPreviewSession(options?: {deviceId?: string}): Promise { - const generation = this.nativeCameraPreviewStartGate.nextGeneration(); - return this.nativeCameraPreviewStartGate.runLatest(generation, () => - this.startNativeCameraPreviewSessionNow(options, generation), - ); - } - - async stopNativeCameraPreviewSession(): Promise { - this.nativeCameraPreviewStartGate.invalidate(); - const trackSid = this.nativeCameraPreviewSessionTrackSid; - this.nativeCameraPreviewSessionTrackSid = null; - this.syncVideoBackgroundFramePumpInBackground('camera-preview-session-stop'); - if (trackSid) { - this.recordNativeVideoDiagnostic('camera_preview.session_cleared', {trackSid}); - } - if (isNativeVoiceEngineSelected()) { - try { - await requireNativeVoiceEngine().stopCameraPreview(); - } catch (error) { - logger.warn('Failed to stop native camera preview session', {error}); - } - } - if (trackSid) { - this.nativeVoiceFrameStatsBatcher.removeTrack(trackSid); - NativeVideoTileManager.unregisterTrack(trackSid); - } - } - - getNativeCameraPreviewSessionStream(): MediaStream | null { - const trackSid = this.nativeCameraPreviewSessionTrackSid; - if (!trackSid) return null; - return NativeVideoTileManager.tracks[trackSid]?.stream ?? null; - } - - private getNativeCameraLocalPreviewTrack( - localParticipant?: NativeCameraPreviewParticipant | null, - ): NativeInboundVideoTrack | null { - return selectNativeCameraLocalPreviewTrack({ - currentTrackSid: this.nativeCameraPreviewTrackSid, - localParticipant: localParticipant ?? this.resolveNativeCameraPreviewParticipant(), - sessionTrackSid: this.nativeCameraPreviewSessionTrackSid, - tracks: NativeVideoTileManager.tracks, - }); - } - - getNativeCameraLocalPreviewStream(localParticipant?: NativeCameraPreviewParticipant | null): MediaStream | null { - return this.getNativeCameraLocalPreviewTrack(localParticipant)?.stream ?? null; - } - - isNativeCameraPublished(): boolean { - if (!isNativeVoiceEngineSelected()) return false; - return this.readNativeCameraActualEnabled(); - } - - private async setCameraEnabledViaEngine( - enabled: boolean, - options?: { - deviceId?: string; - sendUpdate?: boolean; - }, - ): Promise { - const sendUpdate = options?.sendUpdate ?? true; - const outcome = await runCameraTransition({ - enabled, - sendUpdate, - publish: async () => { - if (enabled) { - await this.publishNativeCameraFromSettings(options); - } else { - this.clearNativeCameraLocalPreview(); - await requireNativeVoiceEngine().unpublishCamera(); - } - }, - readActualEnabled: () => this.readNativeCameraActualEnabled(), - onPermissionDenied: () => { - voiceEngineV2AppMediaStateAdapter.applyCameraState(false, {sendUpdate}); - }, - onSuccessSettled: null, - onFailure: (_actual, error) => { - logger.warn('Native voice engine camera state update failed', {enabled, error}); - this.clearNativeCameraLocalPreview(); - }, - rethrowOnFailure: true, - }); - this.syncVideoBackgroundFramePumpInBackground('camera-state-transition'); - if (outcome === 'denied') { - logger.warn('Native voice engine camera state update denied by permission', {enabled}); - throw buildVoiceEngineV2AppCameraPermissionDeniedError(); - } - if (outcome === 'applied') { - logger.info('Native voice engine camera state updated', {enabled}); - } + await this.setCameraEnabled(true, options); } async setScreenShareEnabled( @@ -4479,18 +2341,7 @@ class MediaEngineFacade extends Store { options?: DeviceScreenShareCaptureOptions, publishOptions?: TrackPublishOptions, ): Promise { - if (!isNativeVoiceEngineSelected()) { - await voiceEngineV2AppScreenShareExecutionAdapter.startDeviceScreenShare(this.room, options, publishOptions); - return; - } - await voiceEngineV2AppScreenShareExecutionAdapter.startNativeDeviceScreenShare( - await this.getNativeDeviceScreenShareCaptureOptions(options), - { - sendUpdate: options?.sendUpdate, - playSound: options?.playSound, - }, - publishOptions, - ); + await voiceEngineV2AppScreenShareExecutionAdapter.startDeviceScreenShare(this.room, options, publishOptions); } private createScreenShareControllerGateway(): VoiceEngineV2AppScreenShareControllerGateway { @@ -4542,53 +2393,12 @@ class MediaEngineFacade extends Store { }; } - private getNativeMicrophonePublishOptions( - options: VoiceEngineV2NativeMicrophonePublishOptions = {}, - ): VoiceEngineV2NativeMicrophonePublishOptions { - const channelId = voiceEngineV2AppConnectionHostAdapter.channelId; - const channelBitrateBps = channelId ? (Channels.getChannel(channelId)?.bitrate ?? null) : null; - return resolveVoiceEngineV2NativeMicrophonePublishOptions(VoiceSettings, options, channelBitrateBps); - } - - private async refreshNativeMicrophoneFromSettings(): Promise { - await this.voiceEngineV2Host.runAndWait( - () => { - this.voiceEngineV2Controller.publishMicrophone(this.getNativeMicrophonePublishOptions()); - }, - {description: 'refresh microphone settings'}, - ); - } - - async setMicEnabledViaEngine(enabled: boolean): Promise { - assert.equal(typeof enabled, 'boolean', 'setMicEnabledViaEngine enabled must be a boolean'); - if (enabled) { - await this.voiceEngineV2Host.runAndWait( - () => { - this.voiceEngineV2Controller.publishMicrophone(this.getNativeMicrophonePublishOptions()); - }, - {description: 'publish microphone', staleCompletion: 'resolve'}, - ); - } - await this.voiceEngineV2Host.runAndWait( - () => { - this.voiceEngineV2Controller.setMicrophoneEnabled(enabled); - }, - {description: 'set microphone enabled', staleCompletion: 'resolve'}, - ); - } - async replaceActiveDisplayScreenShare( options?: ScreenShareCaptureOptions, publishOptions?: TrackPublishOptions, ): Promise { - if (!isNativeVoiceEngineSelected()) { - return voiceEngineV2AppScreenShareExecutionAdapter.replaceActiveDisplayScreenShare( - this.room, - options, - publishOptions, - ); - } - return voiceEngineV2AppScreenShareExecutionAdapter.replaceActiveNativeDisplayScreenShareFromActiveSource( + return voiceEngineV2AppScreenShareExecutionAdapter.replaceActiveDisplayScreenShare( + this.room, options, publishOptions, ); @@ -4598,40 +2408,8 @@ class MediaEngineFacade extends Store { options?: DeviceScreenShareCaptureOptions, publishOptions?: TrackPublishOptions, ): Promise { - if (!isNativeVoiceEngineSelected()) { - return voiceEngineV2AppScreenShareExecutionAdapter.replaceActiveDeviceScreenShare( - this.room, - options, - publishOptions, - ); - } - return voiceEngineV2AppScreenShareExecutionAdapter.replaceActiveNativeDeviceScreenShare( - await this.getNativeDeviceScreenShareCaptureOptions(options), - publishOptions, - ); - } - - async startNativeDisplayScreenShare( - nativeOptions: NativeScreenShareOptions, - options?: {sendUpdate?: boolean; playSound?: boolean}, - publishOptions?: TrackPublishOptions, - ): Promise { - await voiceEngineV2AppScreenShareExecutionAdapter.startNativeDisplayScreenShare( + return voiceEngineV2AppScreenShareExecutionAdapter.replaceActiveDeviceScreenShare( this.room, - nativeOptions, - options, - publishOptions, - ); - } - - async replaceActiveNativeDisplayScreenShare( - nativeOptions: NativeScreenShareOptions, - options?: ScreenShareCaptureOptions, - publishOptions?: TrackPublishOptions, - ): Promise { - return voiceEngineV2AppScreenShareExecutionAdapter.replaceActiveNativeDisplayScreenShare( - this.room, - nativeOptions, options, publishOptions, ); @@ -4641,9 +2419,6 @@ class MediaEngineFacade extends Store { linuxRule?: NonNullable, options?: {includeSelfWindowAudio?: boolean; replaceExisting?: boolean}, ): Promise { - if (isNativeVoiceEngineSelected()) { - return this.ensureNativeLinuxScreenShareAudioPublication(linuxRule); - } return voiceEngineV2AppScreenShareExecutionAdapter.ensureLinuxScreenShareAudioPublication( this.room, linuxRule, @@ -4651,41 +2426,6 @@ class MediaEngineFacade extends Store { ); } - private async ensureNativeLinuxScreenShareAudioPublication( - linuxRule?: NonNullable, - ): Promise { - assert.ok(isNativeVoiceEngineSelected(), 'native Linux screen-share audio link requires the native engine'); - if (!linuxRule) return false; - const adapter = voiceEngineV2AppScreenShareExecutionAdapter; - const currentOptions = adapter.captureCoordinator.activeCaptureOptions; - if (!adapter.captureCoordinator.activeCaptureId || !currentOptions) { - logger.debug('Skipping native Linux screen-share audio link without an active native screen share'); - return false; - } - if (adapter.isScreenSharePending) { - logger.warn('Skipping native Linux screen-share audio link while a screen-share operation is pending'); - return false; - } - const activeRule = currentOptions.nativeAudioLinuxRule ?? null; - const ruleUnchanged = activeRule != null && areLinuxAudioRoutingRulesEqual(activeRule, linuxRule); - if (adapter.nativeEngineScreenShareAudioPump && ruleUnchanged) { - logger.debug('Native Linux screen-share audio link already matches the requested routing rule'); - return true; - } - const {audioTrack: _audioTrack, ...restOptions} = currentOptions; - const nextOptions: NativeScreenShareOptions = {...restOptions, nativeAudioLinuxRule: linuxRule}; - const published = await adapter.audioPump.startAudio(nextOptions); - if (!published) { - logger.warn('Native Linux screen-share audio link failed to adopt the requested routing rule'); - return false; - } - adapter.adoptNativeEngineScreenShareOptionsInternal(nextOptions); - LocalVoiceState.updateSelfStreamAudio(true); - assert.ok(adapter.nativeEngineScreenShareAudioPump != null, 'native screen-share audio pump must be active'); - logger.info('Native Linux screen-share audio link adopted new routing rule'); - return true; - } - setScreenShareAudioMuted(muted: boolean): void { voiceEngineV2AppScreenShareExecutionAdapter.setScreenShareAudioMuted(this.room, muted); } @@ -4828,7 +2568,6 @@ class MediaEngineFacade extends Store { voiceServerEndpoint: voiceEngineV2AppConnectionHostAdapter.voiceServerEndpoint, regionHotSwapInProgress: voiceEngineV2AppConnectionHostAdapter.regionHotSwapInProgress, reconnectAttempts: voiceEngineV2AppConnectionHostAdapter.reconnectAttempts, - nativeVoiceEngineSelected: isNativeVoiceEngineSelected(), }, localVoiceState: { selfMute: LocalVoiceState.getSelfMute(), @@ -4994,39 +2733,12 @@ class MediaEngineFacade extends Store { this.voiceEngineV2Controller.registerLocalStreamCodec(source, streamIdentity, codec); } - private bindNativeAudioPreferencesSync(): () => void { - assert.ok(this.voiceEngineV2Host != null, 'native audio preferences sync requires the v2 host'); - let previous = createVoiceEngineV2AppAudioSettingsSnapshot(); - const sync = (): void => { - const current = createVoiceEngineV2AppAudioSettingsSnapshot(); - const captureChanged = hasVoiceEngineV2MicrophoneCaptureSettingsChanged(previous, current); - const processorChanged = hasVoiceEngineV2InputProcessorSettingsChanged(previous, current); - previous = current; - if (!captureChanged && !processorChanged) return; - if (this.voiceEngineV2Snapshot.microphone.published == null) return; - void this.refreshNativeMicrophoneFromSettings().catch((error) => { - logger.warn('Failed to refresh native microphone after audio settings change', {error}); - }); - }; - const disposers = [VoiceSettings.subscribe(sync), VoiceDevicePermissionState.subscribe(sync)]; - assert.equal(disposers.length, 2, 'native audio preferences sync must subscribe to both settings stores'); - return () => { - for (const dispose of disposers) { - dispose(); - } - }; - } - private bindAudioPreferencesSync(room: Room): () => void { return bindVoiceEngineV2AppAudioPreferencesSync( room, { refreshMicrophone: async () => this.refreshMicrophoneFromCurrentEngine(), refreshLocalVoiceInputProcessor: async () => { - if (await shouldUseNativeVoiceEngine()) { - await this.refreshNativeMicrophoneFromSettings(); - return; - } await voiceEngineV2AppMediaExecutionAdapter.refreshLocalVoiceInputProcessor(room); }, applyLocalInputVolume: () => voiceEngineV2AppMediaExecutionAdapter.applyLocalInputVolume(room), @@ -5245,12 +2957,6 @@ class MediaEngineFacade extends Store { async handleLogout(): Promise { this.terminalUnloadVoiceDisconnectSent = false; this.cancelPendingServerDisconnect(); - this.clearNativeVoiceTransportReconnect(); - this.clearNativeVoiceConnectRetry(); - this.clearNativeVoiceConnectSession(); - this.clearPendingNativeLocalMediaReconnect(); - this.stopNativeVoiceStatsSession(); - this.lastNativeVoiceServerUpdate = null; this.transitionFacadeState({type: 'cleanup.logoutStarted'}); this.clearViewerStreamKeys(); this.stopTracking(); @@ -5317,13 +3023,6 @@ class MediaEngineFacade extends Store { cleanup(): void { this.terminalUnloadVoiceDisconnectSent = false; this.cancelPendingServerDisconnect(); - this.syncVideoBackgroundFramePumpInBackground('facade-cleanup'); - this.clearNativeVoiceTransportReconnect(); - this.clearNativeVoiceConnectRetry(); - this.clearNativeVoiceConnectSession(); - this.clearPendingNativeLocalMediaReconnect(); - this.stopNativeVoiceStatsSession(); - this.lastNativeVoiceServerUpdate = null; this.transitionFacadeState({type: 'cleanup.cleanupStarted'}); this.clearViewerStreamKeys(); this.stopTracking(); @@ -5341,12 +3040,6 @@ class MediaEngineFacade extends Store { reset(): void { this.terminalUnloadVoiceDisconnectSent = false; this.cancelPendingServerDisconnect(); - this.clearNativeVoiceTransportReconnect(); - this.clearNativeVoiceConnectRetry(); - this.clearNativeVoiceConnectSession(); - this.clearPendingNativeLocalMediaReconnect(); - this.stopNativeVoiceStatsSession(); - this.lastNativeVoiceServerUpdate = null; this.clearViewerStreamKeys(); this.statsHostAdapter.reset(); this.transitionFacadeState({type: 'cleanup.reset'}); diff --git a/fluxer_app/src/features/voice/engine/MediaEngineFacadeStateMachine.test.ts b/fluxer_app/src/features/voice/engine/MediaEngineFacadeStateMachine.test.ts index 60477a20c..0f33c7828 100644 --- a/fluxer_app/src/features/voice/engine/MediaEngineFacadeStateMachine.test.ts +++ b/fluxer_app/src/features/voice/engine/MediaEngineFacadeStateMachine.test.ts @@ -5,9 +5,6 @@ import {describe, expect, it} from 'vitest'; import { createMediaEngineFacadeSnapshot, getMediaEngineFacadeStateValue, - hasPlayedNativeVoiceReadySounds, - NATIVE_VOICE_READY_SOUND_CONNECTION_MEMORY_LIMIT, - rememberNativeVoiceReadySounds, selectMediaEngineConnectPreflightDecision, selectMediaEngineConnectRequestDecision, selectMediaEngineGatewayErrorDecision, @@ -372,36 +369,6 @@ describe('MediaEngineFacadeStateMachine', () => { ).toEqual({type: 'navigate-channel-gate'}); }); - it('suppresses ready sounds for connection ids that already played them', () => { - const playedConnectionIds = new Set(); - expect(hasPlayedNativeVoiceReadySounds(playedConnectionIds, 'conn-1')).toBe(false); - rememberNativeVoiceReadySounds(playedConnectionIds, 'conn-1'); - expect(hasPlayedNativeVoiceReadySounds(playedConnectionIds, 'conn-1')).toBe(true); - expect(hasPlayedNativeVoiceReadySounds(playedConnectionIds, 'conn-2')).toBe(false); - }); - - it('never suppresses ready sounds for a null connection id', () => { - const playedConnectionIds = new Set(); - rememberNativeVoiceReadySounds(playedConnectionIds, null); - expect(playedConnectionIds.size).toBe(0); - expect(hasPlayedNativeVoiceReadySounds(playedConnectionIds, null)).toBe(false); - }); - - it('bounds ready sound memory by evicting the oldest connection ids', () => { - const playedConnectionIds = new Set(); - for (let i = 0; i < NATIVE_VOICE_READY_SOUND_CONNECTION_MEMORY_LIMIT + 4; i++) { - rememberNativeVoiceReadySounds(playedConnectionIds, `conn-${i}`); - } - expect(playedConnectionIds.size).toBe(NATIVE_VOICE_READY_SOUND_CONNECTION_MEMORY_LIMIT); - expect(hasPlayedNativeVoiceReadySounds(playedConnectionIds, 'conn-0')).toBe(false); - expect( - hasPlayedNativeVoiceReadySounds( - playedConnectionIds, - `conn-${NATIVE_VOICE_READY_SOUND_CONNECTION_MEMORY_LIMIT + 3}`, - ), - ).toBe(true); - }); - it('notifies only for rejected voice state acks carrying the camera user limit error code', () => { expect( shouldNotifyCameraUserLimitRejection({ diff --git a/fluxer_app/src/features/voice/engine/MediaEngineFacadeStateMachine.ts b/fluxer_app/src/features/voice/engine/MediaEngineFacadeStateMachine.ts index e412a217e..d188bd087 100644 --- a/fluxer_app/src/features/voice/engine/MediaEngineFacadeStateMachine.ts +++ b/fluxer_app/src/features/voice/engine/MediaEngineFacadeStateMachine.ts @@ -125,26 +125,6 @@ export interface MediaEngineFacadeServerVoiceStateRemovalInput { connecting: boolean; } -export const NATIVE_VOICE_READY_SOUND_CONNECTION_MEMORY_LIMIT = 16; - -export function hasPlayedNativeVoiceReadySounds( - playedConnectionIds: ReadonlySet, - connectionId: string | null, -): boolean { - if (!connectionId) return false; - return playedConnectionIds.has(connectionId); -} - -export function rememberNativeVoiceReadySounds(playedConnectionIds: Set, connectionId: string | null): void { - if (!connectionId) return; - playedConnectionIds.add(connectionId); - while (playedConnectionIds.size > NATIVE_VOICE_READY_SOUND_CONNECTION_MEMORY_LIMIT) { - const oldest = playedConnectionIds.values().next().value; - if (oldest === undefined) break; - playedConnectionIds.delete(oldest); - } -} - export function createInitialMediaEngineFacadeContext(): MediaEngineFacadeMachineContext { return { activeConnection: null, diff --git a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts index b583f9e1f..23fb3a6dc 100644 --- a/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts +++ b/fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts @@ -135,16 +135,6 @@ interface BindOptions { onSelectedCodecChanged?: (selection: CodecNegotiationSelection) => void | Promise; } -interface NativeCodecNegotiationAdapter { - publishData: (params: { - payload: Uint8Array; - reliable?: boolean; - topic?: string; - destinationIdentities?: Array; - }) => Promise; - getRemoteParticipantIdentities: () => ReadonlyArray; -} - function createId(prefix: string): string { const cryptoObject = globalThis.crypto as Crypto | undefined; if (typeof cryptoObject?.randomUUID === 'function') return `${prefix}_${cryptoObject.randomUUID()}`; @@ -478,7 +468,6 @@ function parseMessage(payload: Uint8Array): FluxerCodecNegotiationMessage | null class ScreenShareCodecNegotiation { private room: Room | null = null; - private nativeAdapter: NativeCodecNegotiationAdapter | null = null; private bindDisposer: (() => void) | null = null; private selectedCodec: VideoCodec | null = null; private localCodecs: Array = []; @@ -559,25 +548,6 @@ class ScreenShareCodecNegotiation { return {knownRemoteCodecs, unknownParticipants}; } - private getNativeRemoteCodecInputs(): { - knownRemoteCodecs: Array>; - unknownParticipants: number; - remoteParticipants: number; - } { - const identities = this.nativeAdapter?.getRemoteParticipantIdentities() ?? []; - const knownRemoteCodecs: Array> = []; - let unknownParticipants = 0; - for (const identity of identities) { - const codecs = this.remoteCodecsByIdentity.get(identity); - if (codecs) { - knownRemoteCodecs.push(codecs); - } else { - unknownParticipants++; - } - } - return {knownRemoteCodecs, unknownParticipants, remoteParticipants: identities.length}; - } - bind(room: Room, options: BindOptions = {}): () => void { this.dispose(); this.room = room; @@ -615,21 +585,10 @@ class ScreenShareCodecNegotiation { return this.bindDisposer; } - bindNative(adapter: NativeCodecNegotiationAdapter, options: BindOptions = {}): () => void { - this.dispose(); - this.nativeAdapter = adapter; - this.bindOptions = options; - this.bindDisposer = () => { - this.nativeAdapter = null; - }; - return this.bindDisposer; - } - dispose(): void { this.bindDisposer?.(); this.bindDisposer = null; this.room = null; - this.nativeAdapter = null; this.selectedCodec = null; this.localCodecs = []; this.bindOptions = {}; @@ -669,35 +628,6 @@ class ScreenShareCodecNegotiation { return await this.updateSelection(room, reason, options); } - async publishLocalCapabilitiesNative( - reason: NegotiationReason = 'manual', - options: BindOptions = this.bindOptions, - ): Promise { - if (typeof window === 'undefined') return null; - if (!this.nativeAdapter) return null; - await Promise.allSettled([ - loadGpuEncoderReport(), - loadNativeHardwareEncoderCapabilities(), - loadVideoDecoderExclusions(), - loadOpenH264Status(), - ]); - this.localCodecs = buildLocalCodecAdvertisements(); - const message: FluxerSelectProtocolMessage = { - op: SELECT_PROTOCOL_OP, - d: { - protocol: 'livekit', - data: { - mode: 'livekit-sfu', - }, - codecs: this.localCodecs, - rtc_connection_id: this.rtcConnectionId, - experiments: ['fixed_keyframe_interval', 'maintain_framerate', 'opus_red', 'transport_cc', 'loss_based_bwe_v2'], - }, - }; - await this.publishNativeMessage(message); - return await this.updateNativeSelection(reason, options); - } - private handleDataMessage(room: Room, participant: Participant, payload: Uint8Array, options: BindOptions): void { const message = parseMessage(payload); if (!message) return; @@ -716,37 +646,6 @@ class ScreenShareCodecNegotiation { } } - handleNativeDataMessage( - participantIdentity: string, - payload: Uint8Array, - options: BindOptions = this.bindOptions, - ): void { - const message = parseMessage(payload); - if (!message) return; - if (message.op === SELECT_PROTOCOL_OP) { - this.remoteCodecsByIdentity.set(participantIdentity, message.d.codecs); - void this.updateNativeSelection('data', options); - } else if (message.op === SESSION_UPDATE_OP) { - this.remoteCodecsByIdentity.set(participantIdentity, message.d.codecs); - logger.debug('Received native remote codec session update', { - participantIdentity, - videoCodec: message.d.video_codec, - mediaSessionId: message.d.media_session_id, - reason: message.d.reason, - }); - void this.updateNativeSelection('data', options); - } - } - - handleNativeParticipantConnected(): void { - void this.publishLocalCapabilitiesNative('participant-connected', this.bindOptions); - } - - handleNativeParticipantDisconnected(participantIdentity: string): void { - this.remoteCodecsByIdentity.delete(participantIdentity); - void this.updateNativeSelection('participant-disconnected', this.bindOptions); - } - private async updateSelection( room: Room, reason: NegotiationReason, @@ -781,39 +680,6 @@ class ScreenShareCodecNegotiation { return selection; } - private async updateNativeSelection( - reason: NegotiationReason, - options: BindOptions, - ): Promise { - if (!this.nativeAdapter) return null; - if (this.localCodecs.length === 0) this.localCodecs = buildLocalCodecAdvertisements(); - const {knownRemoteCodecs, unknownParticipants, remoteParticipants} = this.getNativeRemoteCodecInputs(); - if (reason === 'participant-disconnected' && remoteParticipants === 0 && this.selectedCodec) { - logger.debug('Keeping active native screen share codec after last viewer disconnected', { - codec: this.selectedCodec, - }); - return null; - } - const previousCodec = this.selectedCodec; - this.negotiationSnapshot = transitionScreenShareCodecNegotiationSnapshot(this.negotiationSnapshot, { - type: 'negotiation.evaluate', - localCodecs: this.localCodecs, - remoteCodecs: knownRemoteCodecs, - unknownParticipants, - reason, - codecPreference: getScreenShareCodecPreferenceOrder(), - }); - const selection = this.negotiationSnapshot.context.selection; - if (!selection) return null; - this.selectedCodec = selection.codec; - if (selection.codec === previousCodec) return selection; - this.mediaSessionId = createId('media'); - logger.info('Selected native screen share codec from XState capability intersection', selection); - await this.publishNativeSessionUpdate(selection); - await options.onSelectedCodecChanged?.(selection); - return selection; - } - private async publishSessionUpdate(room: Room, selection: CodecNegotiationSelection): Promise { const message: FluxerSessionUpdateMessage = { op: SESSION_UPDATE_OP, @@ -837,31 +703,6 @@ class ScreenShareCodecNegotiation { logger.debug('Failed to publish codec negotiation message', {error, op: message.op}); } } - - private async publishNativeSessionUpdate(selection: CodecNegotiationSelection): Promise { - const message: FluxerSessionUpdateMessage = { - op: SESSION_UPDATE_OP, - d: { - video_codec: VIDEO_CODEC_NAMES[selection.codec], - media_session_id: this.mediaSessionId, - reason: selection.reason, - codecs: this.localCodecs, - }, - }; - await this.publishNativeMessage(message); - } - - private async publishNativeMessage(message: FluxerCodecNegotiationMessage): Promise { - try { - await this.nativeAdapter?.publishData({ - payload: TEXT_ENCODER.encode(JSON.stringify(message)), - reliable: true, - topic: PROTOCOL_TOPIC, - }); - } catch (error) { - logger.debug('Failed to publish native codec negotiation message', {error, op: message.op}); - } - } } export {CODEC_PREFERENCE, PROTOCOL_TOPIC as SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC}; diff --git a/fluxer_app/src/features/voice/engine/ScreenSharePublicationMigration.ts b/fluxer_app/src/features/voice/engine/ScreenSharePublicationMigration.ts index e3663bddc..0d787f7a8 100644 --- a/fluxer_app/src/features/voice/engine/ScreenSharePublicationMigration.ts +++ b/fluxer_app/src/features/voice/engine/ScreenSharePublicationMigration.ts @@ -643,28 +643,6 @@ class ScreenSharePublicationMigration extends Store { } } - handleNativeDataMessage(participantIdentity: string, payload: Uint8Array): void { - const message = parseScreenShareMigrationMessage(payload); - if (!message) return; - switch (message.op) { - case BREAK_OP: - this.applyBreak(participantIdentity, message.d); - break; - case CANDIDATE_OP: - this.applyCandidate(participantIdentity, message.d); - break; - case READY_OP: - this.applyReady(participantIdentity, message.d); - break; - case COMMIT_OP: - this.applyCommit(participantIdentity, message.d); - break; - case ABORT_OP: - this.applyAbort(participantIdentity, message.d); - break; - } - } - private applyCandidate(participantIdentity: string, message: ScreenShareMigrationCandidateMessage['d']): void { this.commitRemoteMigrationState(participantIdentity, { type: 'migration.candidate', diff --git a/fluxer_app/src/features/voice/engine/ScreenShareSubscriptionManager.ts b/fluxer_app/src/features/voice/engine/ScreenShareSubscriptionManager.ts index c7ae06c69..14060a31d 100644 --- a/fluxer_app/src/features/voice/engine/ScreenShareSubscriptionManager.ts +++ b/fluxer_app/src/features/voice/engine/ScreenShareSubscriptionManager.ts @@ -4,12 +4,7 @@ import {Logger} from '@app/features/platform/utils/AppLogger'; import ScreenSharePublicationMigration from '@app/features/voice/engine/ScreenSharePublicationMigration'; import {Store} from '@app/features/voice/engine/Store'; import { - buildVoiceMediaGraphNativeScreenShareEnabledCommand, - buildVoiceMediaGraphNativeScreenShareQualityCommand, - buildVoiceMediaGraphNativeScreenShareSubscriptionCommands, selectVoiceMediaGraphSubscriptionEntry, - type VoiceMediaGraphRemoteSubscriptionCommand, - type VoiceMediaGraphRemoteTrackSubscriptionController, type VoiceMediaGraphSubscriptionCommand, type VoiceMediaGraphSubscriptionContext, type VoiceMediaGraphSubscriptionEntry, @@ -37,7 +32,6 @@ const qualityMap: Record = { export class ScreenShareSubscriptionManager extends Store { private room: Room | null = null; - private nativeController: VoiceMediaGraphRemoteTrackSubscriptionController | null = null; private observers = new Map(); private reattachPublicationTargets = new Map(); private resubscribePulseTokens = new Map(); @@ -55,12 +49,6 @@ export class ScreenShareSubscriptionManager extends Store { }); } - setNativeController(controller: VoiceMediaGraphRemoteTrackSubscriptionController | null): void { - this.update(() => { - this.nativeController = controller; - }); - } - cleanup(): void { this.transition({type: 'subscription.cleanup', source: VoiceTrackSource.ScreenShare}); } @@ -70,27 +58,25 @@ export class ScreenShareSubscriptionManager extends Store { element: HTMLElement | null, context: VoiceMediaGraphSubscriptionContext = 'carousel', ): void { - if (!this.room && !this.nativeController) { + if (!this.room) { logger.warn('No room available'); return; } - const participant = this.room?.remoteParticipants.get(participantIdentity) ?? null; + const participant = this.room.remoteParticipants.get(participantIdentity) ?? null; if (!participant) { - if (!this.nativeController) { - logger.warn('Participant not found', {participantIdentity}); - return; - } + logger.warn('Participant not found', {participantIdentity}); + return; } logger.info('Subscribing to screen share', {participantIdentity, context}); const screenSharePublication = this.findScreenSharePublication(participant); - if (!screenSharePublication && !this.nativeController) { + if (!screenSharePublication) { logger.debug('No screen share publication found', {participantIdentity}); } this.transition({ type: 'subscription.subscribe', participantIdentity, source: VoiceTrackSource.ScreenShare, - hasPublication: this.nativeController != null || screenSharePublication != null, + hasPublication: screenSharePublication != null, observedElement: element, context, }); @@ -136,7 +122,7 @@ export class ScreenShareSubscriptionManager extends Store { reattachAfterPublish(participantIdentity: string, publication?: RemoteTrackPublication): void { const state = this.getSubscriptionEntry(participantIdentity); - if (!state?.subscribed || (!this.room && !this.nativeController)) return; + if (!state?.subscribed || !this.room) return; logger.info('Reattaching screen share subscription after republish', { participantIdentity, context: state.context, @@ -151,10 +137,7 @@ export class ScreenShareSubscriptionManager extends Store { type: 'subscription.reattachAfterPublish', participantIdentity, source: VoiceTrackSource.ScreenShare, - hasPublication: - this.nativeController != null || - publication != null || - this.hasScreenSharePublicationForIdentity(participantIdentity), + hasPublication: publication != null || this.hasScreenSharePublicationForIdentity(participantIdentity), forceResubscribe: true, }); } finally { @@ -181,7 +164,7 @@ export class ScreenShareSubscriptionManager extends Store { } private hasScreenSharePublicationForIdentity(participantIdentity: string): boolean { - return this.nativeController != null || this.findScreenSharePublicationForIdentity(participantIdentity) != null; + return this.findScreenSharePublicationForIdentity(participantIdentity) != null; } private getParticipant(participantIdentity: string): RemoteParticipant | null { @@ -412,13 +395,6 @@ export class ScreenShareSubscriptionManager extends Store { } } - private applyNativeSubscriptionCommands(commands: ReadonlyArray): void { - if (!this.nativeController) return; - for (const command of commands) { - this.nativeController.setRemoteTrackSubscription(command); - } - } - private applySubscribeOperations( participantIdentity: string, publication: RemoteTrackPublication, @@ -444,17 +420,6 @@ export class ScreenShareSubscriptionManager extends Store { logger.debug('Skipping subscribe while resubscribe pulse is pending', {participantIdentity}); return; } - if (this.nativeController) { - this.applyNativeSubscriptionCommands( - buildVoiceMediaGraphNativeScreenShareSubscriptionCommands({ - participantIdentity, - subscribed: true, - enabled, - quality, - }), - ); - return; - } this.withActiveScreenSharePublications(participantIdentity, (publications, participant) => { let applied = true; for (const publication of publications) { @@ -480,20 +445,6 @@ export class ScreenShareSubscriptionManager extends Store { enabled: boolean, quality: VoiceMediaGraphVideoQuality, ): void { - if (this.nativeController) { - this.applyNativeSubscriptionCommands( - buildVoiceMediaGraphNativeScreenShareSubscriptionCommands({participantIdentity, subscribed: false}), - ); - this.applyNativeSubscriptionCommands( - buildVoiceMediaGraphNativeScreenShareSubscriptionCommands({ - participantIdentity, - subscribed: true, - enabled, - quality, - }), - ); - return; - } const publications = this.getTargetScreenSharePublications( participantIdentity, this.getParticipant(participantIdentity), @@ -574,12 +525,6 @@ export class ScreenShareSubscriptionManager extends Store { private unsubscribePublication(participantIdentity: string): void { this.resubscribePulseTokens.delete(participantIdentity); this.pendingResubscribePulses.delete(participantIdentity); - if (this.nativeController) { - this.applyNativeSubscriptionCommands( - buildVoiceMediaGraphNativeScreenShareSubscriptionCommands({participantIdentity, subscribed: false}), - ); - return; - } const publications = this.getAllScreenSharePublications(this.getParticipant(participantIdentity)); let applied = publications.length > 0; for (const publication of publications) { @@ -594,12 +539,6 @@ export class ScreenShareSubscriptionManager extends Store { } private setPublicationEnabled(participantIdentity: string, enabled: boolean): void { - if (this.nativeController) { - this.applyNativeSubscriptionCommands([ - buildVoiceMediaGraphNativeScreenShareEnabledCommand({participantIdentity, enabled}), - ]); - return; - } this.withActiveScreenSharePublications(participantIdentity, (publications) => { let applied = true; for (const publication of publications) { @@ -615,16 +554,6 @@ export class ScreenShareSubscriptionManager extends Store { } private setPublicationQuality(participantIdentity: string, quality: VoiceMediaGraphVideoQuality): void { - if (this.nativeController) { - const state = this.getSubscriptionEntry(participantIdentity); - const command = buildVoiceMediaGraphNativeScreenShareQualityCommand({ - participantIdentity, - enabled: state?.enabled ?? false, - quality, - }); - if (command) this.applyNativeSubscriptionCommands([command]); - return; - } this.withActiveScreenSharePublications(participantIdentity, (publications) => { let applied = true; for (const publication of publications) { diff --git a/fluxer_app/src/features/voice/engine/VideoSubscriptionManager.ts b/fluxer_app/src/features/voice/engine/VideoSubscriptionManager.ts index c252ddd4c..6d292e346 100644 --- a/fluxer_app/src/features/voice/engine/VideoSubscriptionManager.ts +++ b/fluxer_app/src/features/voice/engine/VideoSubscriptionManager.ts @@ -3,10 +3,7 @@ import {Logger} from '@app/features/platform/utils/AppLogger'; import {Store} from '@app/features/voice/engine/Store'; import { - buildVoiceMediaGraphNativeCameraQualityCommand, - buildVoiceMediaGraphNativeCameraSubscriptionCommand, selectVoiceMediaGraphSubscriptionEntry, - type VoiceMediaGraphRemoteTrackSubscriptionController, type VoiceMediaGraphSubscriptionCommand, type VoiceMediaGraphSubscriptionEntry, type VoiceMediaGraphSubscriptionEvent, @@ -32,7 +29,6 @@ const qualityMap: Record = { export class VideoSubscriptionManager extends Store { private room: Room | null = null; - private nativeController: VoiceMediaGraphRemoteTrackSubscriptionController | null = null; private observers = new Map(); private readonly intersectionOptions: IntersectionObserverInit = { root: null, @@ -46,12 +42,6 @@ export class VideoSubscriptionManager extends Store { }); } - setNativeController(controller: VoiceMediaGraphRemoteTrackSubscriptionController | null): void { - this.update(() => { - this.nativeController = controller; - }); - } - cleanup(): void { this.transition({type: 'subscription.cleanup', source: VoiceTrackSource.Camera}); } @@ -61,20 +51,18 @@ export class VideoSubscriptionManager extends Store { element: HTMLElement | null, initialQuality: VoiceMediaGraphVideoQuality = 'low', ): void { - if (!this.room && !this.nativeController) { + if (!this.room) { logger.warn('No room available'); return; } - const participant = this.room?.remoteParticipants.get(participantIdentity) ?? null; + const participant = this.room.remoteParticipants.get(participantIdentity) ?? null; if (!participant) { - if (!this.nativeController) { - logger.warn('Participant not found', {participantIdentity}); - return; - } + logger.warn('Participant not found', {participantIdentity}); + return; } logger.info('Subscribing to video', {participantIdentity, quality: initialQuality}); const cameraPublication = this.findCameraPublication(participant); - if (!cameraPublication && !this.nativeController) { + if (!cameraPublication) { logger.debug('No camera publication found', {participantIdentity}); } const existingState = this.getSubscriptionEntry(participantIdentity); @@ -82,7 +70,7 @@ export class VideoSubscriptionManager extends Store { type: 'subscription.subscribe', participantIdentity, source: VoiceTrackSource.Camera, - hasPublication: this.nativeController != null || cameraPublication != null, + hasPublication: cameraPublication != null, observedElement: element, quality: existingState ? undefined : initialQuality, }); @@ -147,7 +135,7 @@ export class VideoSubscriptionManager extends Store { reattachAfterPublish(participantIdentity: string): void { const state = this.getSubscriptionEntry(participantIdentity); - if (!state?.subscribed || (!this.room && !this.nativeController)) return; + if (!state?.subscribed || !this.room) return; this.transition({ type: 'subscription.reattachAfterPublish', participantIdentity, @@ -171,7 +159,7 @@ export class VideoSubscriptionManager extends Store { } private hasCameraPublicationForIdentity(participantIdentity: string): boolean { - return this.nativeController != null || this.findCameraPublicationForIdentity(participantIdentity) != null; + return this.findCameraPublicationForIdentity(participantIdentity) != null; } private runPublicationOperation( @@ -345,17 +333,6 @@ export class VideoSubscriptionManager extends Store { enabled: boolean, quality: VoiceMediaGraphVideoQuality, ): void { - if (this.nativeController) { - this.nativeController.setRemoteTrackSubscription( - buildVoiceMediaGraphNativeCameraSubscriptionCommand({ - participantIdentity, - subscribed: true, - enabled, - quality, - }), - ); - return; - } const publication = this.findCameraPublicationForIdentity(participantIdentity); if (!publication) { this.transition({type: 'subscription.publicationMissing', participantIdentity, source: VoiceTrackSource.Camera}); @@ -380,12 +357,6 @@ export class VideoSubscriptionManager extends Store { } private unsubscribePublication(participantIdentity: string): void { - if (this.nativeController) { - this.nativeController.setRemoteTrackSubscription( - buildVoiceMediaGraphNativeCameraSubscriptionCommand({participantIdentity, subscribed: false}), - ); - return; - } const publication = this.findCameraPublicationForIdentity(participantIdentity); if (!publication) return; const applied = this.runPublicationOperation(participantIdentity, publication, 'setSubscribed', () => { @@ -397,12 +368,6 @@ export class VideoSubscriptionManager extends Store { } private setPublicationEnabled(participantIdentity: string, enabled: boolean): void { - if (this.nativeController) { - this.nativeController.setRemoteTrackSubscription( - buildVoiceMediaGraphNativeCameraSubscriptionCommand({participantIdentity, subscribed: true, enabled}), - ); - return; - } const publication = this.findCameraPublicationForIdentity(participantIdentity); if (!publication) { this.transition({type: 'subscription.publicationMissing', participantIdentity, source: VoiceTrackSource.Camera}); @@ -417,12 +382,6 @@ export class VideoSubscriptionManager extends Store { } private setPublicationQuality(participantIdentity: string, quality: VoiceMediaGraphVideoQuality): void { - if (this.nativeController) { - this.nativeController.setRemoteTrackSubscription( - buildVoiceMediaGraphNativeCameraQualityCommand({participantIdentity, quality}), - ); - return; - } const publication = this.findCameraPublicationForIdentity(participantIdentity); if (!publication) { this.transition({type: 'subscription.publicationMissing', participantIdentity, source: VoiceTrackSource.Camera}); diff --git a/fluxer_app/src/features/voice/engine/VoiceEngineV2AppParticipantAdapterNative.test.ts b/fluxer_app/src/features/voice/engine/VoiceEngineV2AppParticipantAdapterNative.test.ts deleted file mode 100644 index c1ec475c2..000000000 --- a/fluxer_app/src/features/voice/engine/VoiceEngineV2AppParticipantAdapterNative.test.ts +++ /dev/null @@ -1,226 +0,0 @@ -// @vitest-environment happy-dom -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {VoiceConnectionQuality} from '@app/features/voice/engine/VoiceTrackSource'; -import {afterEach, beforeEach, describe, expect, it} from 'vitest'; -import type {VoiceEngineV2AppControllerHost} from './v2/VoiceEngineV2AppControllerHost'; -import {createVoiceEngineV2AppTestControllerHost} from './v2/VoiceEngineV2AppControllerHostTestUtils'; -import { - createVoiceEngineV2AppParticipantAdapter, - NATIVE_SPEAKING_HEARTBEAT_TIMEOUT_MS, - type VoiceEngineV2AppLivekitParticipantSnapshot, - type VoiceEngineV2AppParticipantAdapter, -} from './v2/VoiceEngineV2AppParticipantAdapter'; -import {createVoiceEngineV2ShadowHostPorts} from './v2/VoiceEngineV2ShadowHostPorts'; - -let host: VoiceEngineV2AppControllerHost; -let participantAdapter: VoiceEngineV2AppParticipantAdapter; - -function isParticipantSnapshotSpeaking(snapshot: VoiceEngineV2AppLivekitParticipantSnapshot | undefined): boolean { - return Boolean(snapshot?.isSpeaking || snapshot?.isAudioLevelSpeaking); -} - -beforeEach(() => { - host = createVoiceEngineV2AppTestControllerHost({ports: createVoiceEngineV2ShadowHostPorts()}); - participantAdapter = createVoiceEngineV2AppParticipantAdapter({ - controller: host.controller, - getModel: () => host.model, - now: () => 1000, - }); -}); - -afterEach(() => { - participantAdapter.clear(); - host.dispose(); -}); - -describe('VoiceEngineV2AppParticipantAdapter native snapshot methods', () => { - it('upsertParticipantFromNative creates a snapshot with parsed userId/connectionId', () => { - participantAdapter.upsertParticipantFromNative({identity: 'user_42_connA', sid: 'PA_1', name: 'Ada'}); - const snapshot = participantAdapter.getParticipant('user_42_connA'); - expect(snapshot).toBeDefined(); - expect(snapshot?.userId).toBe('42'); - expect(snapshot?.connectionId).toBe('connA'); - expect(snapshot?.sid).toBe('PA_1'); - expect(snapshot?.connectionQuality).toBe(VoiceConnectionQuality.Unknown); - expect(snapshot?.isScreenShareEnabled).toBe(false); - expect(snapshot?.isScreenShareAudioEnabled).toBe(false); - }); - - it('upsertParticipantFromNative merges over an existing snapshot (preserves prior flags)', () => { - participantAdapter.upsertParticipantFromNative({ - identity: 'user_42_connA', - sid: 'PA_1', - isScreenShareEnabled: true, - isScreenShareAudioEnabled: true, - }); - participantAdapter.upsertParticipantFromNative({identity: 'user_42_connA', sid: 'PA_1'}); - expect(participantAdapter.getParticipant('user_42_connA')?.isScreenShareEnabled).toBe(true); - expect(participantAdapter.getParticipant('user_42_connA')?.isScreenShareAudioEnabled).toBe(true); - }); - - it('patchParticipantTrackFlags flips the requested track flag only', () => { - participantAdapter.upsertParticipantFromNative({identity: 'user_42_connA', sid: 'PA_1'}); - participantAdapter.patchParticipantTrackFlags('user_42_connA', { - isCameraEnabled: true, - isScreenShareAudioEnabled: true, - }); - const snapshot = participantAdapter.getParticipant('user_42_connA'); - expect(snapshot?.isCameraEnabled).toBe(true); - expect(snapshot?.isMicrophoneEnabled).toBe(false); - expect(snapshot?.isScreenShareEnabled).toBe(false); - expect(snapshot?.isScreenShareAudioEnabled).toBe(true); - }); - - it('patchParticipantTrackFlags is a no-op for an unknown participant', () => { - participantAdapter.patchParticipantTrackFlags('user_999_x', {isMicrophoneEnabled: true}); - expect(participantAdapter.getParticipant('user_999_x')).toBeUndefined(); - }); - - it('setConnectionQualityForNative updates by sid', () => { - participantAdapter.upsertParticipantFromNative({identity: 'user_42_connA', sid: 'PA_1'}); - participantAdapter.setConnectionQualityForNative('PA_1', VoiceConnectionQuality.Poor); - expect(participantAdapter.getParticipant('user_42_connA')?.connectionQuality).toBe(VoiceConnectionQuality.Poor); - }); - - it('updateActiveSpeakersBySid marks the matching participants speaking', () => { - participantAdapter.upsertParticipantFromNative({identity: 'user_1_a', sid: 'PA_1'}); - participantAdapter.upsertParticipantFromNative({identity: 'user_2_b', sid: 'PA_2'}); - participantAdapter.updateActiveSpeakersBySid(['PA_2']); - expect(participantAdapter.getParticipant('user_1_a')?.isSpeaking).toBe(false); - expect(participantAdapter.getParticipant('user_2_b')?.isSpeaking).toBe(true); - expect(isParticipantSnapshotSpeaking(participantAdapter.getParticipant('user_2_b'))).toBe(true); - participantAdapter.updateActiveSpeakersBySid([]); - expect(participantAdapter.getParticipant('user_2_b')?.isSpeaking).toBe(false); - expect(isParticipantSnapshotSpeaking(participantAdapter.getParticipant('user_2_b'))).toBe(false); - }); - - it('applies native speaking transitions for remote participants', () => { - participantAdapter.upsertParticipantFromNative({identity: 'user_1_a', sid: 'PA_1'}); - participantAdapter.upsertParticipantFromNative({identity: 'user_2_b', sid: 'PA_2'}); - participantAdapter.applyNativeSpeakingSample( - {participantSid: 'PA_2', identity: 'user_2_b', source: 'microphone', isLocal: false, speaking: true}, - 1000, - ); - expect(participantAdapter.getParticipant('user_1_a')?.isAudioLevelSpeaking).toBe(false); - expect(participantAdapter.getParticipant('user_2_b')?.isAudioLevelSpeaking).toBe(true); - expect(participantAdapter.getParticipant('user_2_b')?.isSpeaking).toBe(false); - expect(participantAdapter.getParticipant('user_2_b')?.lastSpokeAt).toBe(1000); - - participantAdapter.applyNativeSpeakingSample( - {participantSid: 'PA_2', identity: 'user_2_b', source: 'microphone', isLocal: false, speaking: false}, - 1400, - ); - expect(participantAdapter.getParticipant('user_2_b')?.isAudioLevelSpeaking).toBe(false); - expect(isParticipantSnapshotSpeaking(participantAdapter.getParticipant('user_2_b'))).toBe(false); - }); - - it('applies native speaking transitions for the local participant', () => { - participantAdapter.upsertParticipantFromNative({identity: 'user_1_a', sid: 'PA_1', isLocal: true}); - participantAdapter.applyNativeSpeakingSample( - {participantSid: 'PA_1', identity: 'user_1_a', source: 'microphone', isLocal: true, speaking: true}, - 1000, - ); - expect(participantAdapter.getParticipant('user_1_a')?.isAudioLevelSpeaking).toBe(true); - expect(isParticipantSnapshotSpeaking(participantAdapter.getParticipant('user_1_a'))).toBe(true); - - participantAdapter.applyNativeSpeakingSample( - {participantSid: 'PA_1', identity: 'user_1_a', source: 'microphone', isLocal: true, speaking: false}, - 1500, - ); - expect(participantAdapter.getParticipant('user_1_a')?.isAudioLevelSpeaking).toBe(false); - expect(isParticipantSnapshotSpeaking(participantAdapter.getParticipant('user_1_a'))).toBe(false); - }); - - it('resolves native speaking samples by sid when the identity is unknown', () => { - participantAdapter.upsertParticipantFromNative({identity: 'user_2_b', sid: 'PA_2'}); - participantAdapter.applyNativeSpeakingSample( - {participantSid: 'PA_2', identity: 'user_renamed', source: 'microphone', isLocal: false, speaking: true}, - 1000, - ); - expect(participantAdapter.getParticipant('user_2_b')?.isAudioLevelSpeaking).toBe(true); - }); - - it('ignores native speaking samples for unknown participants', () => { - participantAdapter.applyNativeSpeakingSample( - {participantSid: 'PA_404', identity: 'user_404_x', source: 'microphone', isLocal: false, speaking: true}, - 1000, - ); - expect(participantAdapter.getParticipant('user_404_x')).toBeUndefined(); - }); - - it('keeps speaking alive on heartbeats and clears it when heartbeats stop', () => { - participantAdapter.upsertParticipantFromNative({identity: 'user_1_a', sid: 'PA_1', isLocal: true}); - participantAdapter.applyNativeSpeakingSample( - {participantSid: 'PA_1', identity: 'user_1_a', source: 'microphone', isLocal: true, speaking: true}, - 1000, - ); - participantAdapter.applyNativeSpeakingSample( - {participantSid: 'PA_1', identity: 'user_1_a', source: 'microphone', isLocal: true, speaking: true}, - 2000, - ); - participantAdapter.sweepNativeSpeakingHeartbeats(2000 + NATIVE_SPEAKING_HEARTBEAT_TIMEOUT_MS - 1); - expect(participantAdapter.getParticipant('user_1_a')?.isAudioLevelSpeaking).toBe(true); - - participantAdapter.sweepNativeSpeakingHeartbeats(2000 + NATIVE_SPEAKING_HEARTBEAT_TIMEOUT_MS); - expect(participantAdapter.getParticipant('user_1_a')?.isAudioLevelSpeaking).toBe(false); - }); - - it('does not clear speaking on sweep while heartbeats keep arriving', () => { - participantAdapter.upsertParticipantFromNative({identity: 'user_1_a', sid: 'PA_1', isLocal: true}); - participantAdapter.applyNativeSpeakingSample( - {participantSid: 'PA_1', identity: 'user_1_a', source: 'microphone', isLocal: true, speaking: true}, - 1000, - ); - for (let tick = 1; tick <= 5; tick++) { - participantAdapter.applyNativeSpeakingSample( - {participantSid: 'PA_1', identity: 'user_1_a', source: 'microphone', isLocal: true, speaking: true}, - 1000 + tick * 1000, - ); - participantAdapter.sweepNativeSpeakingHeartbeats(1000 + tick * 1000 + 500); - expect(participantAdapter.getParticipant('user_1_a')?.isAudioLevelSpeaking).toBe(true); - } - }); - - it('rejects native speaking samples from non-microphone tracks', () => { - participantAdapter.upsertParticipantFromNative({identity: 'user_1_a', sid: 'PA_1', isLocal: true}); - expect(() => - participantAdapter.applyNativeSpeakingSample( - {participantSid: 'PA_1', identity: 'user_1_a', source: 'screen_share_audio', isLocal: true, speaking: true}, - 1000, - ), - ).toThrow(); - expect(participantAdapter.getParticipant('user_1_a')?.isAudioLevelSpeaking).toBe(false); - }); - - it('ignores native speaking samples for discarded connections', () => { - participantAdapter.upsertParticipantFromNative({identity: 'user_1_staleConn', sid: 'PA_1'}); - participantAdapter.discardConnection('staleConn'); - participantAdapter.applyNativeSpeakingSample( - {participantSid: 'PA_1', identity: 'user_1_staleConn', source: 'microphone', isLocal: false, speaking: true}, - 1000, - ); - expect(participantAdapter.getParticipant('user_1_staleConn')?.isAudioLevelSpeaking ?? false).toBe(false); - }); - - it('removeParticipantBySid removes the matching participant', () => { - participantAdapter.upsertParticipantFromNative({identity: 'user_1_a', sid: 'PA_1'}); - participantAdapter.upsertParticipantFromNative({identity: 'user_2_b', sid: 'PA_2'}); - participantAdapter.removeParticipantBySid('PA_1'); - expect(participantAdapter.getParticipant('user_1_a')).toBeUndefined(); - expect(participantAdapter.getParticipant('user_2_b')).toBeDefined(); - }); - - it('does not reinsert a discarded connection from late native participant events', () => { - participantAdapter.upsertParticipantFromNative({identity: 'user_1_staleNativeConnection', sid: 'PA_1'}); - expect(participantAdapter.getParticipant('user_1_staleNativeConnection')).toBeDefined(); - - participantAdapter.discardConnection('staleNativeConnection'); - expect(participantAdapter.getParticipant('user_1_staleNativeConnection')).toBeUndefined(); - - participantAdapter.upsertParticipantFromNative({identity: 'user_1_staleNativeConnection', sid: 'PA_1'}); - participantAdapter.patchParticipantTrackFlags('user_1_staleNativeConnection', {isMicrophoneEnabled: true}); - - expect(participantAdapter.getParticipant('user_1_staleNativeConnection')).toBeUndefined(); - }); -}); diff --git a/fluxer_app/src/features/voice/engine/VoiceMediaEngineBridge.ts b/fluxer_app/src/features/voice/engine/VoiceMediaEngineBridge.ts index b859ac675..20df4ddee 100644 --- a/fluxer_app/src/features/voice/engine/VoiceMediaEngineBridge.ts +++ b/fluxer_app/src/features/voice/engine/VoiceMediaEngineBridge.ts @@ -136,13 +136,6 @@ export interface VoiceEngineV2ScreenEncodingUpdateFromMediaEngine { maxBitrateBps?: number; } -export function isVoiceEngineV2NativeProjectionActiveFromMediaEngine(): boolean { - const store = getMediaEngine(); - if (!store) return false; - if (store.room) return false; - return store.connected === true || store.voiceEngineV2Model?.connection.connected === true; -} - function getVoiceEngineV2ActiveScreenCaptureId(controller: VoiceEngineV2Controller): string | null { return controller.snapshot.screen.desired?.captureId ?? controller.snapshot.screen.published?.captureId ?? null; } diff --git a/fluxer_app/src/features/voice/engine/VoiceMediaGraph.test.ts b/fluxer_app/src/features/voice/engine/VoiceMediaGraph.test.ts index dd59e6f3e..3b109c385 100644 --- a/fluxer_app/src/features/voice/engine/VoiceMediaGraph.test.ts +++ b/fluxer_app/src/features/voice/engine/VoiceMediaGraph.test.ts @@ -32,12 +32,9 @@ import { voiceMediaGraphWatchAttemptDeadlineKey, WATCH_ATTEMPT_TIMEOUT_MS, } from '@app/features/voice/engine/VoiceMediaGraph'; -import { - resolveVoiceMediaGraphNativeTrackInfo, - resolveVoiceMediaGraphPerTrackInfo, -} from '@app/features/voice/engine/VoiceMediaGraphStats'; +import {resolveVoiceMediaGraphPerTrackInfo} from '@app/features/voice/engine/VoiceMediaGraphStats'; import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource'; -import type {VoiceEngineV2PerTrackStats, VoiceEngineV2Stats} from '@fluxer/voice_engine_v2'; +import type {VoiceEngineV2PerTrackStats} from '@fluxer/voice_engine_v2'; import {describe, expect, it} from 'vitest'; function failure(overrides: Partial): VoiceMediaGraphFailure { @@ -50,10 +47,6 @@ function failure(overrides: Partial): VoiceMediaGraphFai }; } -function nativeStats(overrides: Partial): VoiceEngineV2Stats { - return {rttMs: null, outbound: [], inbound: [], ...overrides}; -} - const STREAM_A = 'dm:channel-a:connection-a'; const STREAM_B = 'dm:channel-a:connection-b'; const STREAM_C = 'guild-a:channel-a:connection-c'; @@ -1051,30 +1044,6 @@ describe('VoiceMediaGraph stats observations', () => { }); describe('VoiceMediaGraph stats selectors', () => { - it('uses inbound native screen-share dimensions for the matching participant identity', () => { - const info = resolveVoiceMediaGraphNativeTrackInfo( - nativeStats({ - inbound: [ - { - participantSid: 'PA_1', - participantIdentity: 'user_2_connection_2', - trackSid: 'TR_remote_screen', - source: 'screen_share', - kind: 'video', - bitrateKbps: 4200, - packetsLost: 0, - width: 3840, - height: 2160, - fps: 24.7, - }, - ], - }), - {nativeSource: VoiceTrackSource.ScreenShare, participantIdentity: 'user_2_connection_2'}, - ); - - expect(info).toEqual({width: 3840, height: 2160, fps: 25}); - }); - it('matches web stats by media track id and can return fps-only observations', () => { const tracks: Array = [ { diff --git a/fluxer_app/src/features/voice/engine/VoiceMediaGraphStats.test.ts b/fluxer_app/src/features/voice/engine/VoiceMediaGraphStats.test.ts index 111250d10..907ad3a3b 100644 --- a/fluxer_app/src/features/voice/engine/VoiceMediaGraphStats.test.ts +++ b/fluxer_app/src/features/voice/engine/VoiceMediaGraphStats.test.ts @@ -8,17 +8,12 @@ import { import { buildVoiceMediaGraphStatsView, selectVoiceMediaGraphStreamTrackInfo, - voiceMediaGraphStatsObservationsFromNativeStats, voiceMediaGraphStatsObservationsFromPerTrackStats, } from '@app/features/voice/engine/VoiceMediaGraphStats'; import type {VoiceMediaGraphStatsTrackObservation} from '@app/features/voice/engine/VoiceMediaGraphStatsObservations'; -import type {VoiceEngineV2PerTrackStats, VoiceEngineV2Stats} from '@fluxer/voice_engine_v2'; +import type {VoiceEngineV2PerTrackStats} from '@fluxer/voice_engine_v2'; import {describe, expect, it} from 'vitest'; -function nativeStats(overrides: Partial): VoiceEngineV2Stats { - return {rttMs: null, outbound: [], inbound: [], ...overrides}; -} - function observation(overrides: Partial): VoiceMediaGraphStatsTrackObservation { return { trackSid: null, @@ -56,191 +51,6 @@ function snapshotWithObservations( }); } -describe('voiceMediaGraphStatsObservationsFromNativeStats', () => { - it('normalizes outbound and inbound native tracks with multi-key identity', () => { - const observations = voiceMediaGraphStatsObservationsFromNativeStats( - nativeStats({ - outbound: [ - { - trackSid: 'TR_local_screen', - source: 'screen_share', - kind: 'video', - bitrateKbps: 2000, - packetsLost: 0, - width: 2560, - height: 1440, - fps: 30, - effectiveFps: 59.6, - sourceWidth: 3840, - sourceHeight: 2160, - }, - ], - inbound: [ - { - participantIdentity: 'user_2_conn_2', - participantSid: 'PA_2', - trackSid: 'TR_remote_screen', - source: 'screen_share', - kind: 'video', - bitrateKbps: 4200, - packetsLost: 0, - width: 1920, - height: 1080, - fps: 24.7, - }, - ], - }), - ); - - expect(observations).toEqual([ - observation({ - trackSid: 'TR_local_screen', - source: 'screen_share', - direction: 'send', - fps: 59.6, - width: 2560, - height: 1440, - sourceWidth: 3840, - sourceHeight: 2160, - }), - observation({ - trackSid: 'TR_remote_screen', - participantIdentity: 'user_2_conn_2', - participantSid: 'PA_2', - source: 'screen_share', - direction: 'recv', - fps: 24.7, - width: 1920, - height: 1080, - }), - ]); - }); -}); - -describe('voiceMediaGraphStatsObservationsFromNativeStats local identity stamping', () => { - it('stamps the local participant identity onto an outbound ScreenShare video observation', () => { - const observations = voiceMediaGraphStatsObservationsFromNativeStats( - nativeStats({ - outbound: [ - { - trackSid: 'TR_local_screen', - source: 'screen_share', - kind: 'video', - bitrateKbps: 2000, - packetsLost: 0, - width: 2560, - height: 1440, - effectiveFps: 30, - }, - ], - }), - 'user_1_conn_1', - ); - - expect(observations[0]?.participantIdentity).toBe('user_1_conn_1'); - expect(observations[0]?.direction).toBe('send'); - expect(observations[0]?.source).toBe('screen_share'); - }); - - it('does not stamp the local identity onto an outbound camera observation', () => { - const observations = voiceMediaGraphStatsObservationsFromNativeStats( - nativeStats({ - outbound: [ - { - trackSid: 'TR_local_camera', - source: 'camera', - kind: 'video', - bitrateKbps: 600, - packetsLost: 0, - width: 640, - height: 480, - effectiveFps: 24, - }, - ], - }), - 'user_1_conn_1', - ); - - expect(observations[0]?.participantIdentity).toBeNull(); - }); - - it('matches the self-tile pill target via the participantIdentity fallback for outbound ScreenShare', () => { - const snapshot = snapshotWithObservations( - 'conn-1', - 'native', - voiceMediaGraphStatsObservationsFromNativeStats( - nativeStats({ - outbound: [ - { - trackSid: 'TR_local_screen', - source: 'screen_share', - kind: 'video', - bitrateKbps: 2000, - packetsLost: 0, - width: 2560, - height: 1440, - effectiveFps: 30, - }, - { - trackSid: 'TR_local_camera', - source: 'camera', - kind: 'video', - bitrateKbps: 600, - packetsLost: 0, - width: 640, - height: 480, - effectiveFps: 24, - }, - ], - }), - 'user_1_conn_1', - ), - ); - - expect( - selectVoiceMediaGraphStreamTrackInfo(snapshot, { - trackSid: 'TR_synthetic_preview', - participantIdentity: 'user_1_conn_1', - source: 'screen_share', - kind: 'video', - }), - ).toEqual({width: 2560, height: 1440, fps: 30}); - }); - - it('does not return an outbound camera observation for a ScreenShare participant target', () => { - const snapshot = snapshotWithObservations( - 'conn-1', - 'native', - voiceMediaGraphStatsObservationsFromNativeStats( - nativeStats({ - outbound: [ - { - trackSid: 'TR_local_camera', - source: 'camera', - kind: 'video', - bitrateKbps: 600, - packetsLost: 0, - width: 640, - height: 480, - effectiveFps: 24, - }, - ], - }), - 'user_1_conn_1', - ), - ); - - expect( - selectVoiceMediaGraphStreamTrackInfo(snapshot, { - trackSid: 'TR_synthetic_preview', - participantIdentity: 'user_1_conn_1', - source: 'screen_share', - kind: 'video', - }), - ).toBeNull(); - }); -}); - describe('voiceMediaGraphStatsObservationsFromPerTrackStats', () => { it('normalizes web per-track stats and skips unknown kinds', () => { const tracks: Array = [ @@ -356,55 +166,6 @@ describe('selectVoiceMediaGraphStreamTrackInfo', () => { ).toEqual({width: 2560, height: 1440, fps: 60}); }); - it('returns identical info for equivalent native and web observations', () => { - const nativeSnapshot = snapshotWithObservations( - 'conn-1', - 'native', - voiceMediaGraphStatsObservationsFromNativeStats( - nativeStats({ - inbound: [ - { - participantIdentity: 'user_2_conn_2', - trackSid: 'TR_remote_screen', - source: 'screen_share', - kind: 'video', - bitrateKbps: 4200, - packetsLost: 0, - width: 1920, - height: 1080, - fps: 30, - }, - ], - }), - ), - ); - const webSnapshot = snapshotWithObservations( - 'conn-1', - 'web', - voiceMediaGraphStatsObservationsFromPerTrackStats([ - { - direction: 'recv', - kind: 'video', - trackIdentifier: 'TR_remote_screen', - bitrateKbps: 4200, - framesPerSecond: 30, - frameWidth: 1920, - frameHeight: 1080, - }, - ]), - ); - const target = {trackSid: 'TR_remote_screen', source: 'screen_share', kind: 'video'} as const; - - expect(selectVoiceMediaGraphStreamTrackInfo(nativeSnapshot, target)).toEqual( - selectVoiceMediaGraphStreamTrackInfo(webSnapshot, target), - ); - expect(selectVoiceMediaGraphStreamTrackInfo(nativeSnapshot, target)).toEqual({ - width: 1920, - height: 1080, - fps: 30, - }); - }); - it('drops observations for a stale connectionId', () => { const snapshot = snapshotWithObservations('conn-1', 'native', [ observation({trackSid: 'TR_a', width: 1920, height: 1080, fps: 30}), diff --git a/fluxer_app/src/features/voice/engine/VoiceMediaGraphStats.ts b/fluxer_app/src/features/voice/engine/VoiceMediaGraphStats.ts index adb7437e2..71027eb37 100644 --- a/fluxer_app/src/features/voice/engine/VoiceMediaGraphStats.ts +++ b/fluxer_app/src/features/voice/engine/VoiceMediaGraphStats.ts @@ -1,14 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import assert from 'node:assert/strict'; -import { - asVoiceEngineV2StatsTrackSource, - type VoiceEngineV2InboundStats, - type VoiceEngineV2OutboundStats, - type VoiceEngineV2PerTrackStats, - type VoiceEngineV2Stats, - VoiceEngineV2StatsTrackSource, -} from '@fluxer/voice_engine_v2'; +import type {VoiceEngineV2PerTrackStats} from '@fluxer/voice_engine_v2'; import { type VoiceMediaGraphStatsEntry, type VoiceMediaGraphStatsKind, @@ -18,7 +11,7 @@ import { voiceMediaGraphStatsObservationMatchesTarget, voiceMediaGraphStatsTrackKey, } from './VoiceMediaGraphStatsObservations'; -import {asVoiceTrackSource, type VoiceTrackSource, VoiceTrackSource as VoiceTrackSourceValue} from './VoiceTrackSource'; +import type {VoiceTrackSource} from './VoiceTrackSource'; const VOICE_MEDIA_GRAPH_STATS_TRACK_LIMIT = 1024; @@ -58,62 +51,6 @@ function normalizeId(value: string | null | undefined): string | null { return trimmed ? trimmed : null; } -function isScreenShareStatsSource(source: unknown): boolean { - return asVoiceEngineV2StatsTrackSource(source) === VoiceEngineV2StatsTrackSource.ScreenShare; -} - -function nativeTrackMatches( - track: VoiceEngineV2OutboundStats | VoiceEngineV2InboundStats, - target: VoiceMediaGraphNativeStatsTarget, -): boolean { - const targetTrackSid = normalizeId(target.nativeTrackSid); - if (targetTrackSid && normalizeId(track.trackSid) === targetTrackSid) return true; - const targetIdentity = normalizeId(target.participantIdentity); - if ('participantIdentity' in track && targetIdentity && normalizeId(track.participantIdentity) === targetIdentity) { - return true; - } - return !targetTrackSid && !targetIdentity; -} - -function nativeStatsTrackToInfo( - track: VoiceEngineV2OutboundStats | VoiceEngineV2InboundStats, -): VoiceMediaGraphTrackInfo | null { - const width = track.width ?? track.sourceWidth; - const height = track.height ?? track.sourceHeight; - if (!isPositiveDimension(width)) return null; - if (!isPositiveDimension(height)) return null; - const fps = 'effectiveFps' in track && track.effectiveFps !== undefined ? track.effectiveFps : track.fps; - return {width, height, fps: Math.round(fps ?? 0)}; -} - -function resolveVoiceMediaGraphNativeTrackInfoFromTracks( - tracks: ReadonlyArray, - target: VoiceMediaGraphNativeStatsTarget, -): VoiceMediaGraphTrackInfo | null { - assert.ok(tracks.length <= VOICE_MEDIA_GRAPH_STATS_TRACK_LIMIT, 'native stats track list exceeded graph limit'); - for (const track of tracks) { - if (!isScreenShareStatsSource(track.source)) continue; - if (!nativeTrackMatches(track, target)) continue; - const info = nativeStatsTrackToInfo(track); - if (info) return info; - } - return null; -} - -export function resolveVoiceMediaGraphNativeTrackInfo( - stats: VoiceEngineV2Stats | null, - target: VoiceMediaGraphNativeStatsTarget, -): VoiceMediaGraphTrackInfo | null { - if (!stats) return null; - if (target.nativeSource != null && asVoiceTrackSource(target.nativeSource) !== VoiceTrackSourceValue.ScreenShare) { - return null; - } - return ( - resolveVoiceMediaGraphNativeTrackInfoFromTracks(stats.outbound, target) ?? - resolveVoiceMediaGraphNativeTrackInfoFromTracks(stats.inbound, target) - ); -} - function perTrackStatIdentifiers(track: VoiceEngineV2PerTrackStats): Array { const identifiers: Array = []; if (track.trackIdentifier) identifiers.push(track.trackIdentifier); @@ -190,76 +127,6 @@ function positiveOrNull(value: number | null | undefined): number | null { return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : null; } -function statsSourceOrNull(source: unknown): string | null { - return typeof source === 'string' ? normalizeId(source) : null; -} - -function nativeOutboundStatsToObservation( - track: VoiceEngineV2OutboundStats, - localParticipantIdentity: string | null, -): VoiceMediaGraphStatsTrackObservation { - const source = statsSourceOrNull(track.source); - const participantIdentity = - isScreenShareStatsSource(track.source) && track.kind === 'video' ? normalizeId(localParticipantIdentity) : null; - return { - trackSid: normalizeId(track.trackSid), - trackIdentifier: null, - mediaSourceId: null, - mid: null, - rid: null, - ssrc: null, - participantIdentity, - participantSid: null, - source, - direction: 'send', - kind: track.kind, - fps: positiveOrNull(track.effectiveFps ?? track.fps), - width: positiveOrNull(track.width), - height: positiveOrNull(track.height), - sourceFps: null, - sourceWidth: positiveOrNull(track.sourceWidth), - sourceHeight: positiveOrNull(track.sourceHeight), - }; -} - -function nativeInboundStatsToObservation(track: VoiceEngineV2InboundStats): VoiceMediaGraphStatsTrackObservation { - return { - trackSid: normalizeId(track.trackSid), - trackIdentifier: null, - mediaSourceId: null, - mid: null, - rid: null, - ssrc: null, - participantIdentity: normalizeId(track.participantIdentity), - participantSid: normalizeId(track.participantSid), - source: statsSourceOrNull(track.source), - direction: 'recv', - kind: track.kind, - fps: positiveOrNull(track.fps), - width: positiveOrNull(track.width), - height: positiveOrNull(track.height), - sourceFps: null, - sourceWidth: positiveOrNull(track.sourceWidth), - sourceHeight: positiveOrNull(track.sourceHeight), - }; -} - -export function voiceMediaGraphStatsObservationsFromNativeStats( - stats: VoiceEngineV2Stats, - localParticipantIdentity: string | null = null, -): Array { - assert.ok(stats.outbound.length <= VOICE_MEDIA_GRAPH_STATS_TRACK_LIMIT, 'native outbound stats exceeded graph limit'); - assert.ok(stats.inbound.length <= VOICE_MEDIA_GRAPH_STATS_TRACK_LIMIT, 'native inbound stats exceeded graph limit'); - const observations: Array = []; - for (const track of stats.outbound) { - observations.push(nativeOutboundStatsToObservation(track, localParticipantIdentity)); - } - for (const track of stats.inbound) { - observations.push(nativeInboundStatsToObservation(track)); - } - return observations; -} - function perTrackStatsKindOrNull(kind: VoiceEngineV2PerTrackStats['kind']): VoiceMediaGraphStatsKind | null { if (kind === 'audio') return 'audio'; if (kind === 'video') return 'video'; diff --git a/fluxer_app/src/features/voice/engine/media_engine_facade/NativeVoiceStatsSession.ts b/fluxer_app/src/features/voice/engine/media_engine_facade/NativeVoiceStatsSession.ts deleted file mode 100644 index acff3fdb9..000000000 --- a/fluxer_app/src/features/voice/engine/media_engine_facade/NativeVoiceStatsSession.ts +++ /dev/null @@ -1,117 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; -import {Logger} from '@app/features/platform/utils/AppLogger'; -import Users from '@app/features/user/state/Users'; -import {requireNativeVoiceEngine} from '@app/features/voice/engine/native_voice_engine/getVoiceEngine'; -import NativeVoiceStatsStore from '@app/features/voice/engine/native_voice_engine/NativeVoiceStatsStore'; -import {voiceMediaGraphStatsObservationsFromNativeStats} from '@app/features/voice/engine/VoiceMediaGraphStats'; -import voiceMediaGraphStore from '@app/features/voice/engine/VoiceMediaGraphStore'; -import voiceEngineV2AppConnectionHostAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter'; -import {buildVoiceParticipantIdentity} from '@app/features/voice/utils/VoiceParticipantIdentity'; -import type {VoiceEngineV2Stats} from '@fluxer/voice_engine_v2'; - -const logger = new Logger('NativeVoiceStatsSession'); -const NATIVE_VOICE_STATS_POLL_INTERVAL_MS = 1000; - -export interface NativeVoiceStatsSessionDeps { - ingestStats(stats: VoiceEngineV2Stats, timestampMs: number): void; -} - -export class NativeVoiceStatsSession { - private readonly deps: NativeVoiceStatsSessionDeps; - private pollIntervalId: number | null = null; - private pollInFlight = false; - private publishedConnectionId: string | null = null; - - constructor(deps: NativeVoiceStatsSessionDeps) { - assert.ok(deps, 'native voice stats session deps are required'); - assert.equal(typeof deps.ingestStats, 'function', 'native voice stats session requires an ingest callback'); - this.deps = deps; - } - - private stopPolling(): void { - if (this.pollIntervalId !== null) { - window.clearInterval(this.pollIntervalId); - this.pollIntervalId = null; - } - this.pollInFlight = false; - } - - private resolveLocalParticipantIdentity(connectionId: string): string | null { - const currentUserId = Users.getCurrentUser()?.id; - if (!currentUserId) return null; - return buildVoiceParticipantIdentity(currentUserId, connectionId); - } - - private publishGraphStats(stats: VoiceEngineV2Stats, connectionId: string, timestampMs: number): void { - if (this.publishedConnectionId !== connectionId) { - this.publishedConnectionId = connectionId; - voiceMediaGraphStore.transition({type: 'stats.connectionChanged', connectionId}); - } - voiceMediaGraphStore.transition({ - type: 'stats.observed', - at: timestampMs, - connectionId, - platform: 'native', - tracks: voiceMediaGraphStatsObservationsFromNativeStats( - stats, - this.resolveLocalParticipantIdentity(connectionId), - ), - }); - } - - private clearGraphStats(): void { - if (this.publishedConnectionId === null) return; - this.publishedConnectionId = null; - voiceMediaGraphStore.transition({type: 'stats.connectionChanged', connectionId: null}); - } - - private poll(): void { - NativeVoiceStatsStore.tick(voiceMediaGraphStore.nowMs()); - if (this.pollInFlight) return; - const connectionId = voiceEngineV2AppConnectionHostAdapter.connectionId; - if (!voiceEngineV2AppConnectionHostAdapter.connected || !connectionId) return; - NativeVoiceStatsStore.setConnectionId(connectionId); - this.pollInFlight = true; - const timestamp = voiceMediaGraphStore.nowMs(); - void requireNativeVoiceEngine() - .getConnectionStats() - .then((stats) => { - if ( - !voiceEngineV2AppConnectionHostAdapter.connected || - voiceEngineV2AppConnectionHostAdapter.connectionId !== connectionId - ) - return; - if (stats) { - this.deps.ingestStats(stats, timestamp); - this.publishGraphStats(stats, connectionId, timestamp); - } else { - NativeVoiceStatsStore.tick(timestamp); - } - }) - .catch((error) => { - logger.warn('Native voice engine stats poll failed', {error}); - }) - .finally(() => { - this.pollInFlight = false; - }); - } - - stop(clearStats: boolean = true): void { - this.stopPolling(); - if (clearStats) { - NativeVoiceStatsStore.clear(); - this.clearGraphStats(); - } - } - - start(): void { - this.stop(false); - NativeVoiceStatsStore.startSession(voiceMediaGraphStore.nowMs()); - this.poll(); - this.pollIntervalId = window.setInterval(() => { - this.poll(); - }, NATIVE_VOICE_STATS_POLL_INTERVAL_MS); - } -} diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeAudioDeviceModuleState.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeAudioDeviceModuleState.ts deleted file mode 100644 index 616f2ab04..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeAudioDeviceModuleState.ts +++ /dev/null @@ -1,109 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {Logger} from '@app/features/platform/utils/AppLogger'; -import {getVoiceEngineV2AppNativeBridge} from '@app/features/voice/engine/v2/VoiceEngineV2AppNativeBridge'; -import type {VoiceEngineV2BridgeApi, VoiceEngineV2BridgeAudioDeviceModuleState} from '@fluxer/voice_engine_v2/bridge'; -import {VOICE_ENGINE_V2_ADM_STATUS_EVENT_TYPE} from '@fluxer/voice_engine_v2/bridge'; - -const logger = new Logger('NativeAudioDeviceModuleState'); - -export type NativeAudioDeviceModuleStatus = 'unknown' | 'unsupported' | 'warming' | 'ready' | 'failed'; - -type Listener = (status: NativeAudioDeviceModuleStatus) => void; - -type AudioDeviceModuleStateBridge = VoiceEngineV2BridgeApi & { - getAudioDeviceModuleState: () => Promise; -}; - -function getAudioDeviceModuleStateBridge(): AudioDeviceModuleStateBridge | null { - const bridge = getVoiceEngineV2AppNativeBridge(); - if (!bridge) return null; - if (typeof bridge.getAudioDeviceModuleState !== 'function') return null; - return bridge as AudioDeviceModuleStateBridge; -} - -function isKnownAudioDeviceModuleStatus(status: string): status is 'warming' | 'ready' | 'failed' { - return status === 'warming' || status === 'ready' || status === 'failed'; -} - -class NativeAudioDeviceModuleStateStore { - private status: NativeAudioDeviceModuleStatus = 'unknown'; - private listeners = new Set(); - private eventsBound = false; - private queryPromise: Promise | null = null; - - public getStatus(): NativeAudioDeviceModuleStatus { - return this.status; - } - - public async ensureStatus(): Promise { - const bridge = getAudioDeviceModuleStateBridge(); - if (!bridge) { - return 'unsupported'; - } - this.bindEvents(bridge); - if (this.eventsBound && this.status !== 'unknown') { - return this.status; - } - if (!this.queryPromise) { - this.queryPromise = bridge - .getAudioDeviceModuleState() - .then((state) => { - if (!isKnownAudioDeviceModuleStatus(state.status)) { - logger.warn('Audio device module state query returned unknown status', {state}); - return; - } - if (this.eventsBound && this.status !== 'unknown') { - return; - } - this.setStatus(state.status); - }) - .catch((error) => { - logger.warn('Failed to query audio device module state', {error}); - }) - .finally(() => { - this.queryPromise = null; - }); - } - await this.queryPromise; - if (this.status === 'unknown') { - return 'unsupported'; - } - return this.status; - } - - public subscribe(listener: Listener): () => void { - this.listeners.add(listener); - const bridge = getAudioDeviceModuleStateBridge(); - if (bridge) { - this.bindEvents(bridge); - } - return () => { - this.listeners.delete(listener); - }; - } - - private bindEvents(bridge: AudioDeviceModuleStateBridge): void { - if (this.eventsBound) return; - if (typeof bridge.onEvent !== 'function') return; - this.eventsBound = true; - bridge.onEvent((event) => { - if (event.type !== VOICE_ENGINE_V2_ADM_STATUS_EVENT_TYPE) return; - const payload = event.payload as VoiceEngineV2BridgeAudioDeviceModuleState; - if (!isKnownAudioDeviceModuleStatus(payload.status)) { - logger.warn('Ignoring audio device module status event with unknown status', {payload}); - return; - } - this.setStatus(payload.status); - }); - } - - private setStatus(status: NativeAudioDeviceModuleStatus): void { - if (this.status === status) return; - logger.info('Audio device module status changed', {previous: this.status, next: status}); - this.status = status; - this.listeners.forEach((listener) => listener(status)); - } -} - -export const nativeAudioDeviceModuleState = new NativeAudioDeviceModuleStateStore(); diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeCameraPreviewStartGate.test.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeCameraPreviewStartGate.test.ts deleted file mode 100644 index 16501ee53..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeCameraPreviewStartGate.test.ts +++ /dev/null @@ -1,93 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {describe, expect, it} from 'vitest'; -import {NativeCameraPreviewStartGate} from './NativeCameraPreviewStartGate'; - -interface Deferred { - promise: Promise; - resolve: (value: T) => void; -} - -function createDeferred(): Deferred { - let resolveValue: ((value: T) => void) | null = null; - const promise = new Promise((resolve) => { - resolveValue = resolve; - }); - if (!resolveValue) { - throw new Error('Failed to create deferred promise'); - } - return {promise, resolve: resolveValue}; -} - -describe('NativeCameraPreviewStartGate', () => { - it('lets only the latest waiting start run after an in-flight start settles', async () => { - const gate = new NativeCameraPreviewStartGate(); - const calls: Array = []; - const firstStart = createDeferred(); - - const firstGeneration = gate.nextGeneration(); - const first = gate.runLatest(firstGeneration, () => { - calls.push('first'); - return firstStart.promise; - }); - await Promise.resolve(); - - const staleGeneration = gate.nextGeneration(); - const stale = gate.runLatest(staleGeneration, async () => { - calls.push('stale'); - return 'stale'; - }); - const latestGeneration = gate.nextGeneration(); - const latest = gate.runLatest(latestGeneration, async () => { - calls.push('latest'); - return 'latest'; - }); - - firstStart.resolve('first'); - - await expect(first).resolves.toBe('first'); - await expect(stale).resolves.toBeNull(); - await expect(latest).resolves.toBe('latest'); - expect(calls).toEqual(['first', 'latest']); - }); - - it('invalidates a waiting start without running its operation', async () => { - const gate = new NativeCameraPreviewStartGate(); - const calls: Array = []; - const firstStart = createDeferred(); - - const firstGeneration = gate.nextGeneration(); - const first = gate.runLatest(firstGeneration, () => { - calls.push('first'); - return firstStart.promise; - }); - await Promise.resolve(); - - const waitingGeneration = gate.nextGeneration(); - const waiting = gate.runLatest(waitingGeneration, async () => { - calls.push('waiting'); - return 'waiting'; - }); - gate.invalidate(); - firstStart.resolve('first'); - - await expect(first).resolves.toBe('first'); - await expect(waiting).resolves.toBeNull(); - expect(calls).toEqual(['first']); - }); - - it('clears rejected starts so a later generation can run', async () => { - const gate = new NativeCameraPreviewStartGate(); - const failingGeneration = gate.nextGeneration(); - const failure = new Error('camera failed'); - - await expect( - gate.runLatest(failingGeneration, async () => { - throw failure; - }), - ).rejects.toBe(failure); - - const nextGeneration = gate.nextGeneration(); - await expect(gate.runLatest(nextGeneration, async () => 'next')).resolves.toBe('next'); - }); -}); diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeCameraPreviewStartGate.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeCameraPreviewStartGate.ts deleted file mode 100644 index 975de84b6..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeCameraPreviewStartGate.ts +++ /dev/null @@ -1,52 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; - -export class NativeCameraPreviewStartGate { - private generation = 0; - private inFlight: Promise | null = null; - - nextGeneration(): number { - this.generation += 1; - assert.ok(Number.isInteger(this.generation), 'native camera preview generation must be an integer'); - assert.ok(this.generation > 0, 'native camera preview generation must be positive'); - return this.generation; - } - - invalidate(): void { - this.generation += 1; - assert.ok(Number.isInteger(this.generation), 'native camera preview generation must be an integer'); - assert.ok(this.generation > 0, 'native camera preview generation must be positive'); - } - - isCurrent(generation: number): boolean { - assert.ok(Number.isInteger(generation), 'native camera preview generation must be an integer'); - assert.ok(generation > 0, 'native camera preview generation must be positive'); - return generation === this.generation; - } - - async runLatest(generation: number, operation: () => Promise): Promise { - assert.ok(Number.isInteger(generation), 'native camera preview generation must be an integer'); - assert.ok(generation > 0, 'native camera preview generation must be positive'); - const previousInFlight = this.inFlight; - if (previousInFlight) { - await previousInFlight; - } - if (!this.isCurrent(generation)) { - return null; - } - const startOperation = Promise.resolve().then(operation); - const inFlight = startOperation.then( - () => undefined, - () => undefined, - ); - this.inFlight = inFlight; - try { - return await startOperation; - } finally { - if (this.inFlight === inFlight) { - this.inFlight = null; - } - } - } -} diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeCameraPreviewTrackSelection.test.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeCameraPreviewTrackSelection.test.ts deleted file mode 100644 index 0c85f477e..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeCameraPreviewTrackSelection.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {selectNativeCameraLocalPreviewTrack} from '@app/features/voice/engine/native_voice_engine/NativeCameraPreviewTrackSelection'; -import type {NativeInboundVideoTrack} from '@app/features/voice/engine/native_voice_engine/NativeVideoTileManager'; -import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource'; -import {describe, expect, it} from 'vitest'; - -function track({ - participantIdentity, - participantSid, - source = VoiceTrackSource.Camera, - trackSid, -}: { - participantIdentity?: string; - participantSid: string; - source?: string; - trackSid: string; -}): NativeInboundVideoTrack { - return { - height: 0, - participantIdentity, - participantSid, - source, - stream: {id: `stream-${trackSid}`} as MediaStream, - trackSid, - width: 0, - }; -} - -function trackMap(...tracks: Array): Readonly> { - return Object.fromEntries(tracks.map((entry) => [entry.trackSid, entry])); -} - -describe('selectNativeCameraLocalPreviewTrack', () => { - it('uses the current published camera track when the facade recorded its sid', () => { - const camera = track({ - participantIdentity: 'user_1_connection_1', - participantSid: 'PA_local', - trackSid: 'TR_camera', - }); - - const selected = selectNativeCameraLocalPreviewTrack({ - currentTrackSid: 'TR_camera', - localParticipant: null, - sessionTrackSid: null, - tracks: trackMap(camera), - }); - - expect(selected).toBe(camera); - }); - - it('falls back to the local participant identity when camera frames auto-registered the live tile', () => { - const camera = track({ - participantIdentity: 'user_1_connection_1', - participantSid: 'PA_native', - trackSid: 'TR_camera', - }); - const remote = track({ - participantIdentity: 'user_2_connection_2', - participantSid: 'PA_remote', - trackSid: 'TR_remote', - }); - - const selected = selectNativeCameraLocalPreviewTrack({ - currentTrackSid: null, - localParticipant: {identity: 'user_1_connection_1', sid: 'PA_livekit'}, - sessionTrackSid: null, - tracks: trackMap(remote, camera), - }); - - expect(selected).toBe(camera); - }); - - it('ignores the standalone preview session track while selecting the published local camera', () => { - const preview = track({ - participantSid: 'local-camera-preview', - trackSid: 'local-camera-preview', - }); - const camera = track({ - participantIdentity: 'user_1_connection_1', - participantSid: 'PA_local', - trackSid: 'TR_camera', - }); - - const selected = selectNativeCameraLocalPreviewTrack({ - currentTrackSid: null, - localParticipant: {identity: 'user_1_connection_1', sid: 'PA_local'}, - sessionTrackSid: 'local-camera-preview', - tracks: trackMap(preview, camera), - }); - - expect(selected).toBe(camera); - }); -}); diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeCameraPreviewTrackSelection.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeCameraPreviewTrackSelection.ts deleted file mode 100644 index 33aefa87a..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeCameraPreviewTrackSelection.ts +++ /dev/null @@ -1,66 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; -import type {NativeInboundVideoTrack} from '@app/features/voice/engine/native_voice_engine/NativeVideoTileManager'; -import {asVoiceTrackSource, VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource'; - -export interface NativeCameraPreviewParticipant { - identity?: string; - sid: string; -} - -export interface NativeCameraPreviewTrackSelectionInput { - currentTrackSid: string | null; - localParticipant?: NativeCameraPreviewParticipant | null; - sessionTrackSid: string | null; - tracks: Readonly>; -} - -function assertTrackSidOrNull(trackSid: string | null, label: string): void { - if (trackSid === null) return; - assert.ok(trackSid.length > 0, `${label} must be null or non-empty`); -} - -function isNativeCameraPreviewTrackCandidate(track: NativeInboundVideoTrack, sessionTrackSid: string | null): boolean { - if (sessionTrackSid !== null && track.trackSid === sessionTrackSid) return false; - return asVoiceTrackSource(track.source) === VoiceTrackSource.Camera; -} - -function hasParticipantIdentity(participant: NativeCameraPreviewParticipant): participant is { - identity: string; - sid: string; -} { - return participant.identity !== undefined && participant.identity.length > 0; -} - -function trackMatchesPreviewParticipant( - track: NativeInboundVideoTrack, - participant: NativeCameraPreviewParticipant, -): boolean { - assert.ok( - participant.sid.length > 0 || hasParticipantIdentity(participant), - 'preview participant must be identifiable', - ); - if (participant.sid.length > 0 && track.participantSid === participant.sid) return true; - if (!hasParticipantIdentity(participant)) return false; - if (track.participantIdentity === participant.identity) return true; - return track.participantSid === participant.identity; -} - -export function selectNativeCameraLocalPreviewTrack( - input: NativeCameraPreviewTrackSelectionInput, -): NativeInboundVideoTrack | null { - assertTrackSidOrNull(input.currentTrackSid, 'current native camera preview trackSid'); - assertTrackSidOrNull(input.sessionTrackSid, 'native camera preview session trackSid'); - if (input.currentTrackSid !== null) { - const current = input.tracks[input.currentTrackSid]; - if (current && isNativeCameraPreviewTrackCandidate(current, input.sessionTrackSid)) return current; - } - const participant = input.localParticipant ?? null; - if (!participant) return null; - for (const track of Object.values(input.tracks)) { - if (!isNativeCameraPreviewTrackCandidate(track, input.sessionTrackSid)) continue; - if (trackMatchesPreviewParticipant(track, participant)) return track; - } - return null; -} diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeLocalMediaReconnectPolicy.test.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeLocalMediaReconnectPolicy.test.ts deleted file mode 100644 index a22b2d893..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeLocalMediaReconnectPolicy.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {shouldSuppressVoiceEngineV2LocalTrackStateDuringReconnect} from '@fluxer/voice_engine_v2'; -import {describe, expect, it} from 'vitest'; -import {shouldSuppressNativeLocalTrackStateDuringReconnect} from './NativeLocalMediaReconnectPolicy'; - -describe('NativeLocalMediaReconnectPolicy', () => { - it('is a compatibility export for the v2 local media reconnect policy', () => { - expect(shouldSuppressNativeLocalTrackStateDuringReconnect).toBe( - shouldSuppressVoiceEngineV2LocalTrackStateDuringReconnect, - ); - }); -}); diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeLocalMediaReconnectPolicy.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeLocalMediaReconnectPolicy.ts deleted file mode 100644 index 5333019df..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeLocalMediaReconnectPolicy.ts +++ /dev/null @@ -1,6 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -export { - shouldSuppressVoiceEngineV2LocalTrackStateDuringReconnect as shouldSuppressNativeLocalTrackStateDuringReconnect, - type VoiceEngineV2LocalTrackReconnectState as NativeLocalTrackReconnectState, -} from '@fluxer/voice_engine_v2'; diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeMicrophoneOperationFailurePolicy.test.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeMicrophoneOperationFailurePolicy.test.ts deleted file mode 100644 index 094359b9f..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeMicrophoneOperationFailurePolicy.test.ts +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {getVoiceEngineV2MicrophoneOperationFailureAction} from '@fluxer/voice_engine_v2'; -import {describe, expect, it} from 'vitest'; -import {getNativeMicrophoneOperationFailureAction} from './NativeMicrophoneOperationFailurePolicy'; - -describe('NativeMicrophoneOperationFailurePolicy', () => { - it('is a compatibility export for the v2 microphone failure policy', () => { - expect(getNativeMicrophoneOperationFailureAction).toBe(getVoiceEngineV2MicrophoneOperationFailureAction); - }); -}); diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeMicrophoneOperationFailurePolicy.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeMicrophoneOperationFailurePolicy.ts deleted file mode 100644 index 456b6f4f9..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeMicrophoneOperationFailurePolicy.ts +++ /dev/null @@ -1,6 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -export { - getVoiceEngineV2MicrophoneOperationFailureAction as getNativeMicrophoneOperationFailureAction, - type VoiceEngineV2MicrophoneOperationFailureAction as NativeMicrophoneOperationFailureAction, -} from '@fluxer/voice_engine_v2'; diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVideoTileManager.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVideoTileManager.ts deleted file mode 100644 index 5b6e8247d..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVideoTileManager.ts +++ /dev/null @@ -1,486 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; -import {Logger} from '@app/features/platform/utils/AppLogger'; -import {createInboundVideoBridge} from '@app/features/voice/engine/native_voice_engine/createInboundVideoBridge'; -import {Store} from '@app/features/voice/engine/Store'; -import { - asVoiceTrackSource, - isVoiceScreenShareSource, - VoiceTrackSource, -} from '@app/features/voice/engine/VoiceTrackSource'; -import { - assertNonEmptyString, - assertOptionalNonEmptyString, -} from '@app/features/voice/engine/v2/VoiceEngineV2AppAdapterAssertions'; -import {ScreenShareWatchErrorCode, ScreenShareWatchFailures} from '@app/features/voice/state/ScreenShareWatchFailures'; -import type {VoiceEngineV2BridgeApi, VoiceEngineV2BridgeVideoFrame} from '@fluxer/voice_engine_v2/bridge'; - -const logger = new Logger('NativeVideoTileManager'); - -export const NATIVE_INBOUND_VIDEO_TRACKS_MAX = 64; -export const NATIVE_RETAINED_LAST_FRAMES_MAX = 8; -export const NATIVE_AUTO_REGISTER_SUPPRESSED_TRACKS_MAX = 64; - -const UNKNOWN_TRACK_WARNED_SIDS_MAX = 32; -const UNKNOWN_TRACK_FRAME_COUNT_MAX = 1_000_000; -const OBSERVER_FAILURE_COUNT_MAX = 1_000_000; -const OBSERVER_FAILURE_LOG_INTERVAL = 300; - -export interface NativeInboundVideoTrack { - participantSid: string; - participantIdentity?: string; - trackSid: string; - source: string; - width: number; - height: number; - stream: MediaStream; -} - -export interface NativeRetainedVideoFrame { - trackSid: string; - width: number; - height: number; - timestampUs: number; - data: ArrayBuffer; -} - -interface NativeVideoEntry { - participantSid: string; - participantIdentity?: string; - trackSid: string; - source: string; - pushFrame?: (frame: {width: number; height: number; timestampUs: number; data: ArrayBuffer}) => void; - cleanup: () => Promise; -} - -type NativeVideoFrameMetaWithSource = VoiceEngineV2BridgeVideoFrame['meta'] & { - identity?: unknown; -}; - -export interface NativeVideoTileManagerStartOptions { - onFrame?: (frame: VoiceEngineV2BridgeVideoFrame) => void; -} - -function frameKey(frame: VoiceEngineV2BridgeVideoFrame): string { - return frame.meta.trackSid; -} - -function extractParticipantConnectionId(identity: string | undefined): string | null { - if (!identity) return null; - const match = identity.match(/^user_\d+_(.+)$/); - return match ? match[1] : null; -} - -class NativeVideoTileManager extends Store { - private _tracks: Readonly> = {}; - private entries = new Map(); - private retainedLastFrames = new Map(); - private unsubscribeFrames: (() => void) | null = null; - private onFrame: ((frame: VoiceEngineV2BridgeVideoFrame) => void) | null = null; - private unknownTrackFrameCount = 0; - private unknownTrackWarnedSids = new Set(); - private observerFailureCount = 0; - private readonly autoRegisterSuppressedTrackSids = new Set(); - - get tracks(): Readonly> { - return this._tracks; - } - - get isFrameSubscriptionActive(): boolean { - return this.unsubscribeFrames !== null; - } - - getRetainedLastFrame(trackSid: string): NativeRetainedVideoFrame | undefined { - return this.retainedLastFrames.get(trackSid); - } - - get retainedLastFrameCount(): number { - return this.retainedLastFrames.size; - } - - private retainLastFrame(frame: VoiceEngineV2BridgeVideoFrame): void { - const trackSid = frame.meta.trackSid; - if (this.retainedLastFrames.has(trackSid)) { - this.retainedLastFrames.delete(trackSid); - } - this.retainedLastFrames.set(trackSid, { - trackSid, - width: frame.meta.width, - height: frame.meta.height, - timestampUs: frame.meta.timestampUs, - data: frame.data, - }); - while (this.retainedLastFrames.size > NATIVE_RETAINED_LAST_FRAMES_MAX) { - const oldestTrackSid = this.retainedLastFrames.keys().next().value; - if (oldestTrackSid === undefined) break; - this.retainedLastFrames.delete(oldestTrackSid); - } - assert.ok(this.retainedLastFrames.size >= 1, 'retained frame map must contain the frame just retained'); - assert.ok( - this.retainedLastFrames.size <= NATIVE_RETAINED_LAST_FRAMES_MAX, - 'retained frame map must stay within cap', - ); - } - - getTracksForParticipant(participantSid: string, participantIdentity?: string): Array { - const tracks: Array = []; - for (const trackSid in this._tracks) { - const track = this._tracks[trackSid]; - if (!track) continue; - if (track.participantSid === participantSid) { - tracks.push(track); - continue; - } - if (participantIdentity !== undefined && track.participantIdentity === participantIdentity) { - tracks.push(track); - } - } - return tracks; - } - - start(bridge: VoiceEngineV2BridgeApi, options: NativeVideoTileManagerStartOptions = {}): void { - this.stopFrameSubscription(); - this.resetFrameDiagnostics(); - this.onFrame = options.onFrame ?? null; - this.unsubscribeFrames = bridge.onVideoFrame((frame) => this.handleFrame(frame)); - logger.info('Inbound video manager started'); - } - - registerTrack(participantSid: string, trackSid: string, source: string, participantIdentity?: string): void { - assertNonEmptyString(participantSid, 'participantSid'); - assertNonEmptyString(trackSid, 'trackSid'); - assertNonEmptyString(source, 'source'); - assertOptionalNonEmptyString(participantIdentity, 'participantIdentity'); - const existing = this.entries.get(trackSid); - if (existing) { - const conflicting = - existing.participantSid !== participantSid || - existing.participantIdentity !== participantIdentity || - existing.source !== source; - if (conflicting) { - logger.error('Conflicting re-registration for existing inbound video track; keeping existing entry', { - trackSid, - existingParticipantSid: existing.participantSid, - existingParticipantIdentity: existing.participantIdentity, - existingSource: existing.source, - requestedParticipantSid: participantSid, - requestedParticipantIdentity: participantIdentity, - requestedSource: source, - }); - } - return; - } - if (!this.hasTrackCapacity('registerTrack', participantSid, trackSid, source)) { - return; - } - const bridge = createInboundVideoBridge(`${participantSid}:${trackSid}`); - if (!bridge) { - this.reportBridgeUnavailable(participantSid, trackSid, source); - return; - } - const entry: NativeVideoEntry = { - participantSid, - participantIdentity, - trackSid, - source, - pushFrame: bridge.pushFrame, - cleanup: () => bridge.cleanup(), - }; - this.forgetSuppressedAutoRegisterTrack(trackSid); - this.entries.set(trackSid, entry); - this.update(() => { - this._tracks = { - ...this._tracks, - [trackSid]: { - participantSid, - participantIdentity, - trackSid, - source, - width: 0, - height: 0, - stream: bridge.stream, - }, - }; - }); - logger.debug('Registered inbound video track', {participantSid, participantIdentity, trackSid, source}); - } - - private hasTrackCapacity(operation: string, participantSid: string, trackSid: string, source: string): boolean { - assert.ok(this.entries.size <= NATIVE_INBOUND_VIDEO_TRACKS_MAX, 'tracked entries must stay within cap'); - if (this.entries.size < NATIVE_INBOUND_VIDEO_TRACKS_MAX) { - return true; - } - logger.error('Refusing inbound video track registration beyond cap; a stale entry may have leaked', { - operation, - participantSid, - trackSid, - source, - trackedCount: this.entries.size, - cap: NATIVE_INBOUND_VIDEO_TRACKS_MAX, - }); - return false; - } - - private reportBridgeUnavailable(participantSid: string, trackSid: string, source: string): void { - if (isVoiceScreenShareSource(source)) { - logger.warn('Could not build inbound video bridge; tile will be unavailable', {participantSid, trackSid}); - ScreenShareWatchFailures.reportFailure({ - participantSid, - trackSid, - source: VoiceTrackSource.ScreenShare, - code: ScreenShareWatchErrorCode.NativeInboundBridgeUnavailable, - reason: 'native-inbound-bridge-unavailable', - }); - return; - } - logger.error('Could not build inbound camera video bridge; camera tile will be unavailable', { - participantSid, - trackSid, - source, - }); - } - - registerLocalPreviewTrack({ - participantSid, - participantIdentity, - trackSid, - source, - width, - height, - stream, - cleanup, - }: { - participantSid: string; - participantIdentity?: string; - trackSid: string; - source: string; - width: number; - height: number; - stream: MediaStream; - cleanup: () => Promise; - }): void { - assertNonEmptyString(participantSid, 'participantSid'); - assertNonEmptyString(trackSid, 'trackSid'); - assertNonEmptyString(source, 'source'); - assertOptionalNonEmptyString(participantIdentity, 'participantIdentity'); - this.unregisterTrack(trackSid); - if (!this.hasTrackCapacity('registerLocalPreviewTrack', participantSid, trackSid, source)) { - void cleanup().catch((error) => { - logger.warn('Failed to clean up refused local preview track', {trackSid, error}); - }); - return; - } - const entry: NativeVideoEntry = {participantSid, participantIdentity, trackSid, source, cleanup}; - this.forgetSuppressedAutoRegisterTrack(trackSid); - this.entries.set(trackSid, entry); - this.update(() => { - this._tracks = { - ...this._tracks, - [trackSid]: { - participantSid, - participantIdentity, - trackSid, - source, - width, - height, - stream, - }, - }; - }); - logger.debug('Registered local native preview track', { - participantSid, - participantIdentity, - trackSid, - source, - width, - height, - }); - } - - unregisterTrack(trackSid: string): void { - const entry = this.entries.get(trackSid); - if (!entry) return; - this.entries.delete(trackSid); - this.rememberSuppressedAutoRegisterTrack(trackSid); - this.retainedLastFrames.delete(trackSid); - void entry.cleanup().catch((error) => { - logger.warn('Failed to clean up inbound video bridge', {trackSid, error}); - }); - this.update(() => { - if (!(trackSid in this._tracks)) return; - const next = {...this._tracks}; - delete next[trackSid]; - this._tracks = next; - }); - logger.debug('Unregistered inbound video track', {trackSid}); - } - - unregisterParticipant(participantSid: string): void { - for (const entry of [...this.entries.values()]) { - if (entry.participantSid === participantSid) { - this.unregisterTrack(entry.trackSid); - } - } - } - - unregisterConnection(connectionId: string): Array { - assertNonEmptyString(connectionId, 'connectionId'); - const trackSids: Array = []; - for (const entry of [...this.entries.values()]) { - const identityConnectionId = extractParticipantConnectionId(entry.participantIdentity); - if (identityConnectionId === connectionId) { - trackSids.push(entry.trackSid); - continue; - } - const sidConnectionId = extractParticipantConnectionId(entry.participantSid); - if (sidConnectionId === connectionId) { - trackSids.push(entry.trackSid); - } - } - for (const trackSid of trackSids) { - this.unregisterTrack(trackSid); - } - return trackSids; - } - - clear(): void { - this.stopFrameSubscription(); - this.resetFrameDiagnostics(); - for (const entry of [...this.entries.values()]) { - this.unregisterTrack(entry.trackSid); - } - this.retainedLastFrames.clear(); - assert.equal(this.entries.size, 0, 'tracked entries must be empty after clear'); - assert.equal(this.retainedLastFrames.size, 0, 'retained frames must be empty after clear'); - } - - private rememberSuppressedAutoRegisterTrack(trackSid: string): void { - assertNonEmptyString(trackSid, 'auto-register suppressed trackSid'); - if (this.autoRegisterSuppressedTrackSids.has(trackSid)) { - this.autoRegisterSuppressedTrackSids.delete(trackSid); - } - while (this.autoRegisterSuppressedTrackSids.size >= NATIVE_AUTO_REGISTER_SUPPRESSED_TRACKS_MAX) { - const oldestTrackSid = this.autoRegisterSuppressedTrackSids.keys().next().value; - if (oldestTrackSid === undefined) break; - this.autoRegisterSuppressedTrackSids.delete(oldestTrackSid); - } - this.autoRegisterSuppressedTrackSids.add(trackSid); - assert.ok( - this.autoRegisterSuppressedTrackSids.size <= NATIVE_AUTO_REGISTER_SUPPRESSED_TRACKS_MAX, - 'auto-register suppression set must stay within cap', - ); - } - - private forgetSuppressedAutoRegisterTrack(trackSid: string): void { - assertNonEmptyString(trackSid, 'auto-register resumed trackSid'); - this.autoRegisterSuppressedTrackSids.delete(trackSid); - assert.ok( - this.autoRegisterSuppressedTrackSids.size <= NATIVE_AUTO_REGISTER_SUPPRESSED_TRACKS_MAX, - 'auto-register suppression set must stay within cap after delete', - ); - } - - private stopFrameSubscription(): void { - this.unsubscribeFrames?.(); - this.unsubscribeFrames = null; - this.onFrame = null; - } - - private resetFrameDiagnostics(): void { - this.unknownTrackFrameCount = 0; - this.unknownTrackWarnedSids.clear(); - this.observerFailureCount = 0; - this.autoRegisterSuppressedTrackSids.clear(); - assert.equal(this.unknownTrackWarnedSids.size, 0, 'unknown-track warn latch must be empty after reset'); - assert.equal(this.observerFailureCount, 0, 'observer failure count must be zero after reset'); - assert.equal(this.autoRegisterSuppressedTrackSids.size, 0, 'auto-register suppression set must reset'); - } - - private noteObserverFailure(error: unknown): void { - this.observerFailureCount = Math.min(this.observerFailureCount + 1, OBSERVER_FAILURE_COUNT_MAX); - assert.ok(this.observerFailureCount >= 1, 'observer failure count must advance'); - assert.ok(this.observerFailureCount <= OBSERVER_FAILURE_COUNT_MAX, 'observer failure count must stay bounded'); - const shouldLog = - this.observerFailureCount === 1 || this.observerFailureCount % OBSERVER_FAILURE_LOG_INTERVAL === 0; - if (shouldLog) { - logger.warn('Inbound video frame observer failed', {failureCount: this.observerFailureCount, error}); - } - } - - private noteUnknownTrackFrame(trackSid: string): void { - this.unknownTrackFrameCount = Math.min(this.unknownTrackFrameCount + 1, UNKNOWN_TRACK_FRAME_COUNT_MAX); - assert.ok(this.unknownTrackFrameCount >= 1, 'unknown-track frame count must advance'); - assert.ok(this.unknownTrackFrameCount <= UNKNOWN_TRACK_FRAME_COUNT_MAX, 'unknown-track count must stay bounded'); - if (this.unknownTrackWarnedSids.has(trackSid)) { - return; - } - if (this.unknownTrackWarnedSids.size >= UNKNOWN_TRACK_WARNED_SIDS_MAX) { - return; - } - this.unknownTrackWarnedSids.add(trackSid); - logger.warn('Dropping inbound video frames for unregistered track', { - trackSid, - droppedFrameCount: this.unknownTrackFrameCount, - registeredTrackCount: this.entries.size, - }); - } - - private registerCameraTrackFromFrame(frame: VoiceEngineV2BridgeVideoFrame): NativeVideoEntry | null { - const meta = frame.meta as NativeVideoFrameMetaWithSource; - const source = asVoiceTrackSource(meta.source); - if (source !== VoiceTrackSource.Camera) return null; - if (typeof meta.participantSid !== 'string' || meta.participantSid.length === 0) return null; - if (typeof meta.trackSid !== 'string' || meta.trackSid.length === 0) return null; - if (this.autoRegisterSuppressedTrackSids.has(meta.trackSid)) return null; - const participantIdentity = - typeof meta.participantIdentity === 'string' && meta.participantIdentity.length > 0 - ? meta.participantIdentity - : typeof meta.identity === 'string' && meta.identity.length > 0 - ? meta.identity - : undefined; - this.registerTrack(meta.participantSid, meta.trackSid, source, participantIdentity); - return this.entries.get(meta.trackSid) ?? null; - } - - private handleFrame(frame: VoiceEngineV2BridgeVideoFrame): void { - try { - this.onFrame?.(frame); - } catch (error) { - this.noteObserverFailure(error); - } - const key = frameKey(frame); - const entry = this.entries.get(key) ?? this.registerCameraTrackFromFrame(frame); - if (!entry) { - this.noteUnknownTrackFrame(key); - return; - } - entry.pushFrame?.({ - width: frame.meta.width, - height: frame.meta.height, - timestampUs: frame.meta.timestampUs, - data: frame.data, - }); - this.retainLastFrame(frame); - const existing = this._tracks[key]; - if (existing && (existing.width !== frame.meta.width || existing.height !== frame.meta.height)) { - this.update(() => { - this._tracks = { - ...this._tracks, - [key]: {...existing, width: frame.meta.width, height: frame.meta.height}, - }; - }); - } - } -} - -const instance = new NativeVideoTileManager(); - -( - window as typeof window & { - _nativeVideoTileManager?: NativeVideoTileManager; - } -)._nativeVideoTileManager = instance; - -export default instance; -export {NativeVideoTileManager}; diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceConnectRetryPolicy.test.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceConnectRetryPolicy.test.ts deleted file mode 100644 index 52e4fc48b..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceConnectRetryPolicy.test.ts +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {shouldRetryVoiceEngineV2NativeConnectTimeout} from '@fluxer/voice_engine_v2'; -import {describe, expect, it} from 'vitest'; -import {shouldRetryNativeVoiceConnectTimeout} from './NativeVoiceConnectRetryPolicy'; - -describe('NativeVoiceConnectRetryPolicy', () => { - it('is a compatibility export for the v2 native connect retry policy', () => { - expect(shouldRetryNativeVoiceConnectTimeout).toBe(shouldRetryVoiceEngineV2NativeConnectTimeout); - }); -}); diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceConnectRetryPolicy.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceConnectRetryPolicy.ts deleted file mode 100644 index 5fee27749..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceConnectRetryPolicy.ts +++ /dev/null @@ -1,6 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -export { - shouldRetryVoiceEngineV2NativeConnectTimeout as shouldRetryNativeVoiceConnectTimeout, - type VoiceEngineV2NativeConnectRetryPolicyInput as NativeVoiceConnectRetryPolicyInput, -} from '@fluxer/voice_engine_v2'; diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceDeviceSync.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceDeviceSync.ts deleted file mode 100644 index 66fb73697..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceDeviceSync.ts +++ /dev/null @@ -1,233 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; -import {Logger} from '@app/features/platform/utils/AppLogger'; -import {getStreamKey} from '@app/features/voice/components/StreamKeys'; -import {computeNativeParticipantVolume} from '@app/features/voice/engine/native_voice_engine/nativeVoiceVolume'; -import type {VoiceEngine} from '@app/features/voice/engine/native_voice_engine/VoiceEngine'; -import {getEffectiveAudioState} from '@app/features/voice/engine/VoiceEffectiveAudioState'; -import {getVoiceConnectionContextFromMediaEngine} from '@app/features/voice/engine/VoiceMediaEngineBridge'; -import { - getLocalSpeakingThresholdRms, - getRemoteSpeakingThresholdRms, -} from '@app/features/voice/engine/VoiceSpeakingThreshold'; -import type {VoiceEngineV2AppParticipantSnapshot} from '@app/features/voice/engine/v2/VoiceEngineV2AppSelectors'; -import LocalVoiceState from '@app/features/voice/state/LocalVoiceState'; -import ParticipantVolume from '@app/features/voice/state/ParticipantVolume'; -import StreamAudioPrefs from '@app/features/voice/state/StreamAudioPrefs'; -import VoiceSettings from '@app/features/voice/state/VoiceSettings'; -import {type VoiceDeviceState, voiceDeviceManager} from '@app/features/voice/utils/VoiceDeviceManager'; - -const logger = new Logger('NativeVoiceDeviceSync'); - -export const NATIVE_VOICE_VOLUME_SYNC_PARTICIPANTS_MAX = 256; - -const NATIVE_PARTICIPANT_GAIN_MAX = 2; - -export {computeNativeParticipantVolume}; - -interface NativeVoiceStreamConnectionContext { - guildId: string | null; - channelId: string | null; -} - -interface NativeVoiceDeviceSyncDeps { - engine: VoiceEngine; - getParticipants: () => Readonly>; - subscribeParticipants?: (listener: () => void) => () => void; - getEffectiveDeaf?: () => boolean; - getStreamConnectionContext?: () => NativeVoiceStreamConnectionContext; - subscribeLocalVoiceState?: (listener: () => void) => () => void; - subscribeDeviceState?: (listener: (state: VoiceDeviceState) => void) => () => void; -} - -function defaultStreamConnectionContext(): NativeVoiceStreamConnectionContext { - const context = getVoiceConnectionContextFromMediaEngine(); - return {guildId: context?.guildId ?? null, channelId: context?.channelId ?? null}; -} - -function releaseStaleOutputSelection(state: VoiceDeviceState): void { - const outputDeviceId = VoiceSettings.getOutputDeviceId(); - if (!outputDeviceId || outputDeviceId === 'default') { - return; - } - if (state.outputDevices.length === 0) { - return; - } - const stillPresent = state.outputDevices.some((device) => device.deviceId === outputDeviceId); - if (stillPresent) { - return; - } - logger.warn('Selected audio output device no longer available; falling back to default', { - deviceId: outputDeviceId, - }); - VoiceSettings.updateSettings({outputDeviceId: 'default'}); -} - -function resolveStreamKey( - context: NativeVoiceStreamConnectionContext, - participant: VoiceEngineV2AppParticipantSnapshot, -): string | null { - if (!participant.connectionId) return null; - return getStreamKey(context.guildId, context.channelId, participant.connectionId); -} - -function applyParticipantVolumes( - engine: VoiceEngine, - participants: Readonly>, - effectiveDeaf: boolean, - streamContext: NativeVoiceStreamConnectionContext, -): void { - assert.equal(typeof effectiveDeaf, 'boolean', 'effectiveDeaf must be a boolean'); - const outputVolumePercent = VoiceSettings.getOutputVolume(); - let participantCount = 0; - for (const identity in participants) { - const participant = participants[identity]; - if (!participant) continue; - participantCount += 1; - if (participantCount > NATIVE_VOICE_VOLUME_SYNC_PARTICIPANTS_MAX) { - logger.error('Participant volume sync exceeds cap; truncating', { - count: participantCount, - cap: NATIVE_VOICE_VOLUME_SYNC_PARTICIPANTS_MAX, - }); - break; - } - if (participant.isLocal) continue; - if (!participant.sid || !participant.userId) continue; - const streamKey = resolveStreamKey(streamContext, participant); - const streamVolumePercent = streamKey ? StreamAudioPrefs.getVolume(streamKey) : undefined; - const streamMuted = streamKey ? StreamAudioPrefs.isMuted(streamKey) : undefined; - const volume = computeNativeParticipantVolume({ - userVolumePercent: ParticipantVolume.getVolume(participant.userId), - outputVolumePercent, - locallyMuted: ParticipantVolume.isLocalMuted(participant.userId), - effectiveDeaf, - streamVolumePercent, - streamMuted, - }); - assert.ok(volume >= 0, 'participant volume must be non-negative'); - assert.ok(volume <= NATIVE_PARTICIPANT_GAIN_MAX, 'participant volume must not exceed gain cap'); - if (effectiveDeaf) { - assert.equal(volume, 0, 'deafened local user must push volume 0 for every participant'); - } - void engine.setParticipantVolume(participant.sid, volume).catch((error) => { - logger.warn('Native participant-volume sync failed', {sid: participant.sid, error}); - }); - } -} - -function buildParticipantSidSnapshot( - participants: Readonly>, -): string { - let snapshot = ''; - for (const identity in participants) { - const sid = participants[identity]?.sid; - if (!sid) continue; - snapshot = snapshot ? `${snapshot},${sid}` : sid; - } - return snapshot; -} - -export function bindNativeVoiceDeviceSync(deps: NativeVoiceDeviceSyncDeps): () => void { - const {engine, getParticipants} = deps; - const getEffectiveDeaf = deps.getEffectiveDeaf ?? (() => getEffectiveAudioState().effectiveDeaf); - const getStreamConnectionContext = deps.getStreamConnectionContext ?? defaultStreamConnectionContext; - const subscribeLocalVoiceState = - deps.subscribeLocalVoiceState ?? ((listener: () => void) => LocalVoiceState.subscribe(listener)); - const subscribeDeviceState = - deps.subscribeDeviceState ?? - ((listener: (state: VoiceDeviceState) => void) => voiceDeviceManager.subscribe(listener)); - - const applyOutputDevice = (deviceId: string): void => { - if (!deviceId) return; - void engine.setAudioOutputDevice(deviceId).catch((error) => { - logger.warn('Native output-device sync failed', {deviceId, error}); - }); - }; - - const applyVolumes = (): void => { - applyParticipantVolumes(engine, getParticipants(), getEffectiveDeaf(), getStreamConnectionContext()); - }; - - const applySpeakingDetection = (vadThreshold: number): void => { - assert.ok(Number.isFinite(vadThreshold), 'vad threshold must be finite'); - void engine - .setSpeakingDetection({ - localThresholdRms: getLocalSpeakingThresholdRms(vadThreshold), - remoteThresholdRms: getRemoteSpeakingThresholdRms(vadThreshold), - }) - .catch((error) => { - logger.warn('Native speaking-detection sync failed', {error}); - }); - }; - - applyOutputDevice(VoiceSettings.getOutputDeviceId()); - applyVolumes(); - applySpeakingDetection(VoiceSettings.getVadThreshold()); - - let previousVadThreshold = VoiceSettings.getVadThreshold(); - const syncSpeakingDetection = () => { - const vadThreshold = VoiceSettings.getVadThreshold(); - if (vadThreshold === previousVadThreshold) return; - previousVadThreshold = vadThreshold; - applySpeakingDetection(vadThreshold); - }; - - let previousOutputDeviceId = VoiceSettings.getOutputDeviceId(); - const getVolumeSnapshot = () => { - const streamContext = getStreamConnectionContext(); - return { - volumes: ParticipantVolume.volumes, - mutes: ParticipantVolume.localMutes, - outputVolume: VoiceSettings.getOutputVolume(), - effectiveDeaf: getEffectiveDeaf(), - participantSids: buildParticipantSidSnapshot(getParticipants()), - streamGuildId: streamContext.guildId, - streamChannelId: streamContext.channelId, - streamPrefsRevision: StreamAudioPrefs.audioPrefsRevision, - }; - }; - let previousVolumeSnapshot = getVolumeSnapshot(); - const syncOutputDevice = () => { - const deviceId = VoiceSettings.getOutputDeviceId(); - if (deviceId === previousOutputDeviceId) return; - previousOutputDeviceId = deviceId; - applyOutputDevice(deviceId); - }; - const syncVolumes = () => { - const current = getVolumeSnapshot(); - if ( - current.volumes === previousVolumeSnapshot.volumes && - current.mutes === previousVolumeSnapshot.mutes && - current.outputVolume === previousVolumeSnapshot.outputVolume && - current.effectiveDeaf === previousVolumeSnapshot.effectiveDeaf && - current.participantSids === previousVolumeSnapshot.participantSids && - current.streamGuildId === previousVolumeSnapshot.streamGuildId && - current.streamChannelId === previousVolumeSnapshot.streamChannelId && - current.streamPrefsRevision === previousVolumeSnapshot.streamPrefsRevision - ) { - return; - } - previousVolumeSnapshot = current; - applyVolumes(); - }; - const settingsDisposer = VoiceSettings.subscribe(() => { - syncOutputDevice(); - syncVolumes(); - syncSpeakingDetection(); - }); - const volumeDisposer = ParticipantVolume.subscribe(syncVolumes); - const streamPrefsDisposer = StreamAudioPrefs.subscribe(syncVolumes); - const localVoiceStateDisposer = subscribeLocalVoiceState(syncVolumes); - const participantDisposer = deps.subscribeParticipants?.(syncVolumes) ?? (() => {}); - const deviceStateDisposer = subscribeDeviceState(releaseStaleOutputSelection); - - return () => { - settingsDisposer(); - volumeDisposer(); - streamPrefsDisposer(); - localVoiceStateDisposer(); - participantDisposer(); - deviceStateDisposer(); - }; -} diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceE2EEStore.test.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceE2EEStore.test.ts deleted file mode 100644 index 48ee6739d..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceE2EEStore.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import { - NativeVoiceE2EEStore, - normalizeParticipantE2EEState, -} from '@app/features/voice/engine/native_voice_engine/NativeVoiceE2EEStore'; -import {beforeEach, describe, expect, it} from 'vitest'; - -let store: NativeVoiceE2EEStore; - -beforeEach(() => { - store = new NativeVoiceE2EEStore(); -}); - -describe('normalizeParticipantE2EEState', () => { - it('maps engine spellings onto the closed set', () => { - expect(normalizeParticipantE2EEState('encrypted')).toBe('encrypted'); - expect(normalizeParticipantE2EEState('ok')).toBe('encrypted'); - expect(normalizeParticipantE2EEState('decryption_failed')).toBe('decryption_failed'); - expect(normalizeParticipantE2EEState('decryptionFailed')).toBe('decryption_failed'); - expect(normalizeParticipantE2EEState('error')).toBe('decryption_failed'); - expect(normalizeParticipantE2EEState('missing_key')).toBe('unencrypted'); - expect(normalizeParticipantE2EEState('unencrypted')).toBe('unencrypted'); - expect(normalizeParticipantE2EEState('???')).toBe('unknown'); - expect(normalizeParticipantE2EEState(undefined)).toBe('unknown'); - }); -}); - -describe('NativeVoiceE2EEStore', () => { - it('starts empty with a none aggregate', () => { - expect(store.hasAnyState()).toBe(false); - expect(store.aggregateStatus()).toBe('none'); - }); - - it('records and normalises per-participant state', () => { - store.setState('PA_1', 'ok'); - expect(store.getStateForSid('PA_1')).toBe('encrypted'); - expect(store.hasAnyState()).toBe(true); - }); - - it('ignores an empty sid', () => { - store.setState('', 'encrypted'); - expect(store.hasAnyState()).toBe(false); - }); - - it('aggregates to encrypted when every participant is encrypted', () => { - store.setState('PA_1', 'encrypted'); - store.setState('PA_2', 'ok'); - expect(store.aggregateStatus()).toBe('encrypted'); - }); - - it('aggregates to broken when any participant is not encrypted', () => { - store.setState('PA_1', 'encrypted'); - store.setState('PA_2', 'decryption_failed'); - expect(store.aggregateStatus()).toBe('broken'); - }); - - it('removes one participant and clears all', () => { - store.setState('PA_1', 'encrypted'); - store.setState('PA_2', 'encrypted'); - store.remove('PA_1'); - expect(store.getStateForSid('PA_1')).toBeUndefined(); - expect(store.aggregateStatus()).toBe('encrypted'); - store.clear(); - expect(store.hasAnyState()).toBe(false); - expect(store.aggregateStatus()).toBe('none'); - }); -}); diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceE2EEStore.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceE2EEStore.ts deleted file mode 100644 index fbfa8aa74..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceE2EEStore.ts +++ /dev/null @@ -1,71 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {Store} from '@app/features/voice/engine/Store'; -import { - normalizeVoiceEngineV2ParticipantE2eeState, - type VoiceEngineV2ParticipantE2eeState, -} from '@fluxer/voice_engine_v2'; - -export type NativeParticipantE2EEState = VoiceEngineV2ParticipantE2eeState; - -export function normalizeParticipantE2EEState(raw: unknown): NativeParticipantE2EEState { - return normalizeVoiceEngineV2ParticipantE2eeState(raw); -} - -class NativeVoiceE2EEStore extends Store { - private _stateBySid: Readonly> = {}; - - get stateBySid(): Readonly> { - return this._stateBySid; - } - - hasAnyState(): boolean { - for (const _ in this._stateBySid) return true; - return false; - } - - getStateForSid(sid: string): NativeParticipantE2EEState | undefined { - return this._stateBySid[sid]; - } - - setState(sid: string, raw: unknown): void { - if (!sid) return; - const next = normalizeParticipantE2EEState(raw); - if (this._stateBySid[sid] === next) return; - this.update(() => { - this._stateBySid = {...this._stateBySid, [sid]: next}; - }); - } - - remove(sid: string): void { - if (!(sid in this._stateBySid)) return; - this.update(() => { - const next = {...this._stateBySid}; - delete next[sid]; - this._stateBySid = next; - }); - } - - clear(): void { - if (!this.hasAnyState()) return; - this.update(() => { - this._stateBySid = {}; - }); - } - - aggregateStatus(): 'encrypted' | 'broken' | 'none' { - let sawAny = false; - for (const sid in this._stateBySid) { - const state = this._stateBySid[sid]; - if (!state) continue; - sawAny = true; - if (state !== 'encrypted') return 'broken'; - } - return sawAny ? 'encrypted' : 'none'; - } -} - -const instance = new NativeVoiceE2EEStore(); - -export default instance; -export {NativeVoiceE2EEStore}; diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceEngine.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceEngine.ts deleted file mode 100644 index f7a81c61b..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceEngine.ts +++ /dev/null @@ -1,835 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; -import {Logger} from '@app/features/platform/utils/AppLogger'; -import { - type NativeAudioDeviceModuleStatus, - nativeAudioDeviceModuleState, -} from '@app/features/voice/engine/native_voice_engine/NativeAudioDeviceModuleState'; -import {getNativeVoiceEngineConnectionEventAction} from '@app/features/voice/engine/native_voice_engine/nativeVoiceEngineEventMapper'; -import type {VoiceEngine} from '@app/features/voice/engine/native_voice_engine/VoiceEngine'; -import { - createVoiceEngineV2OperationFailure, - VOICE_ENGINE_V2_CAMERA_EFFECT_STRENGTH_MAX, - VOICE_ENGINE_V2_CAMERA_EFFECT_STRENGTH_MIN, - VOICE_ENGINE_V2_OPERATION_SUCCESS, - type VoiceEngineV2BridgeApi, - type VoiceEngineV2BridgeAudioInputDevice, - type VoiceEngineV2BridgeAudioOutputDevice, - type VoiceEngineV2BridgeCameraDevice, - type VoiceEngineV2BridgeCameraPreviewInfo, - type VoiceEngineV2BridgeCapabilities, - type VoiceEngineV2BridgeConnectOptions, - type VoiceEngineV2BridgeEvent, - type VoiceEngineV2BridgeOperationResult, - type VoiceEngineV2BridgePcmFrame, - type VoiceEngineV2BridgeProcessedCameraFrame, - type VoiceEngineV2BridgePublishCameraOptions, - type VoiceEngineV2BridgePublishDataOptions, - type VoiceEngineV2BridgePublishDeviceScreenShareOptions, - type VoiceEngineV2BridgePublishMicrophoneOptions, - type VoiceEngineV2BridgePublishNativeCameraSinkResult, - type VoiceEngineV2BridgePublishProcessedCameraOptions, - type VoiceEngineV2BridgePublishProcessedCameraResult, - type VoiceEngineV2BridgePublishScreenAudioOptions, - type VoiceEngineV2BridgePublishScreenOptions, - type VoiceEngineV2BridgeRemoteTrackSubscriptionOptions, - type VoiceEngineV2BridgeSpeakingDetectionOptions, - type VoiceEngineV2BridgeStartCameraPreviewOptions, - type VoiceEngineV2BridgeStats, - type VoiceEngineV2BridgeUpdateCameraCaptureOptions, - type VoiceEngineV2BridgeUpdateScreenShareEncodingOptions, -} from '@fluxer/voice_engine_v2/bridge'; - -const logger = new Logger('NativeVoiceEngine'); - -const PCM_FRAME_CHANNELS_MIN = 1; -const PCM_FRAME_CHANNELS_MAX = 8; - -function assertCameraEffectStrengthInvariant(value: number, field: string, operation: string): void { - assert.ok(Number.isInteger(value), `${operation} ${field} must be an integer`); - assert.ok(value >= VOICE_ENGINE_V2_CAMERA_EFFECT_STRENGTH_MIN, `${operation} ${field} must be at least 0`); - assert.ok(value <= VOICE_ENGINE_V2_CAMERA_EFFECT_STRENGTH_MAX, `${operation} ${field} must be at most 100`); -} - -function assertCameraOptionInvariants(params: VoiceEngineV2BridgePublishCameraOptions, operation: string): void { - if (params.width != null) { - assert.ok(Number.isFinite(params.width), `${operation} width must be finite`); - assert.ok(params.width > 0, `${operation} width must be positive`); - } - if (params.height != null) { - assert.ok(Number.isFinite(params.height), `${operation} height must be finite`); - assert.ok(params.height > 0, `${operation} height must be positive`); - } - if (params.frameRate != null) { - assert.ok(Number.isFinite(params.frameRate), `${operation} frameRate must be finite`); - assert.ok(params.frameRate > 0, `${operation} frameRate must be positive`); - } - if (params.backgroundBlurStrength != null) { - assertCameraEffectStrengthInvariant(params.backgroundBlurStrength, 'backgroundBlurStrength', operation); - } -} - -export function floatToPcm16Buffer(channelData: Float32Array): ArrayBuffer { - const buffer = new ArrayBuffer(channelData.length * 2); - const view = new Uint8Array(buffer); - for (let i = 0; i < channelData.length; i++) { - const clamped = Math.max(-1, Math.min(1, channelData[i] ?? 0)); - const pcm = (clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff) | 0; - const byteOffset = i * 2; - view[byteOffset] = pcm & 0xff; - view[byteOffset + 1] = (pcm >> 8) & 0xff; - } - return buffer; -} - -function toOwnedArrayBuffer(payload: ArrayBuffer | ArrayBufferView): ArrayBuffer { - if (payload instanceof ArrayBuffer) return payload.slice(0); - const copy = new Uint8Array(payload.byteLength); - copy.set(new Uint8Array(payload.buffer, payload.byteOffset, payload.byteLength)); - return copy.buffer; -} - -function createNativeMicrophoneNotConnectedResult(): VoiceEngineV2BridgeOperationResult { - return createVoiceEngineV2OperationFailure({ - code: 'not-connected', - capability: 'microphoneCapture', - message: 'Native voice engine is not connected', - }); -} - -function isNativeAudioDeviceModuleReadyForMicrophone(status: NativeAudioDeviceModuleStatus): boolean { - assert.equal(typeof status, 'string', 'native audio device module status must be a string'); - return status === 'ready' || status === 'unsupported'; -} - -function createNativeMicrophoneAudioDeviceModuleNotReadyResult( - status: NativeAudioDeviceModuleStatus, -): VoiceEngineV2BridgeOperationResult { - assert.equal(typeof status, 'string', 'native audio device module status must be a string'); - return createVoiceEngineV2OperationFailure({ - code: 'native-error', - capability: 'microphoneCapture', - message: `Native audio device module is ${status}`, - }); -} - -function isCameraAlreadyPublishedError(error: unknown): boolean { - return error instanceof Error && /camera already published/i.test(error.message); -} - -type NativeVoiceEngineScreenShareVideoPublication = - | { - readonly kind: 'display'; - readonly configKey: string; - readonly options: VoiceEngineV2BridgePublishScreenOptions; - } - | { - readonly kind: 'device'; - readonly configKey: string; - }; - -type NativeVoiceEngineScreenShareVideoState = - | { - readonly kind: 'idle'; - } - | { - readonly kind: 'publishing'; - readonly target: NativeVoiceEngineScreenShareVideoPublication; - readonly promise: Promise; - } - | { - readonly kind: 'published'; - readonly publication: NativeVoiceEngineScreenShareVideoPublication; - }; - -type NativeVoiceEngineScreenShareAudioState = - | { - readonly kind: 'idle'; - } - | { - readonly kind: 'published'; - readonly configKey: string; - }; - -export class NativeVoiceEngine implements VoiceEngine { - readonly kind = 'native' as const; - private readonly bridge: VoiceEngineV2BridgeApi; - private connected = false; - private connectionOperationGeneration = 0; - private prewarmPromise: Promise | null = null; - private microphonePublished = false; - private microphoneConfigKey: string | null = null; - private cameraPublished = false; - private cameraConfigKey: string | null = null; - private cameraTrackSid: string | null = null; - private cameraPublishInFlight: Promise | null = null; - private cameraPublishInFlightKey: string | null = null; - private screenShareVideoState: NativeVoiceEngineScreenShareVideoState = {kind: 'idle'}; - private screenShareAudioState: NativeVoiceEngineScreenShareAudioState = {kind: 'idle'}; - - constructor(bridge: VoiceEngineV2BridgeApi) { - this.bridge = bridge; - } - - async getCapabilities(): Promise { - return this.bridge.getCapabilities(); - } - - private async runPrewarm(): Promise { - logger.info('Native voice engine prewarm requested'); - await this.bridge.prewarm(); - logger.info('Native voice engine prewarm acknowledged'); - } - - async prewarm(): Promise { - if (!this.prewarmPromise) { - this.prewarmPromise = this.runPrewarm().finally(() => { - this.prewarmPromise = null; - }); - } - return this.prewarmPromise; - } - - async connect(params: VoiceEngineV2BridgeConnectOptions): Promise { - const generation = ++this.connectionOperationGeneration; - logger.info('Native voice engine connect requested', {hasE2EE: params.e2eeKey != null}); - await this.prewarm(); - if (generation !== this.connectionOperationGeneration) { - logger.warn('Skipping stale native voice engine connect after prewarm', {hasE2EE: params.e2eeKey != null}); - return; - } - await this.bridge.connect(params); - if (generation !== this.connectionOperationGeneration) { - logger.warn('Ignoring stale native voice engine connect acknowledgement', {hasE2EE: params.e2eeKey != null}); - return; - } - this.connected = true; - logger.info('Native voice engine connect acknowledged', {hasE2EE: params.e2eeKey != null}); - } - - async disconnect(): Promise { - this.connectionOperationGeneration++; - this.connected = false; - this.clearPublishedMediaState(); - await this.bridge.disconnect(); - } - - private clearPublishedMediaState(): void { - this.microphonePublished = false; - this.microphoneConfigKey = null; - this.cameraPublished = false; - this.cameraConfigKey = null; - this.cameraTrackSid = null; - this.cameraPublishInFlight = null; - this.cameraPublishInFlightKey = null; - this.clearScreenShareVideoState(); - this.clearScreenShareAudioState(); - } - - isConnected(): boolean { - return this.connected; - } - - isPublishingCamera(): boolean { - return this.cameraPublished || this.cameraPublishInFlight !== null; - } - - async publishMicrophone( - params: VoiceEngineV2BridgePublishMicrophoneOptions = {}, - ): Promise { - const capabilities = await this.getCapabilities(); - if (!capabilities.microphoneCapture) { - this.microphonePublished = false; - this.microphoneConfigKey = null; - return createVoiceEngineV2OperationFailure({ - code: 'unsupported-capability', - capability: 'microphoneCapture', - message: 'Native voice engine addon does not support device microphone capture', - }); - } - if (!this.connected) { - this.microphonePublished = false; - this.microphoneConfigKey = null; - return createNativeMicrophoneNotConnectedResult(); - } - const audioDeviceModuleStatus = await nativeAudioDeviceModuleState.ensureStatus(); - if (!isNativeAudioDeviceModuleReadyForMicrophone(audioDeviceModuleStatus)) { - this.microphonePublished = false; - this.microphoneConfigKey = null; - return createNativeMicrophoneAudioDeviceModuleNotReadyResult(audioDeviceModuleStatus); - } - const requestedDeviceId = params.deviceId?.trim() || null; - const publishOptions = { - deviceId: requestedDeviceId ?? undefined, - echoCancellation: params.echoCancellation, - noiseSuppression: params.noiseSuppression, - autoGainControl: params.autoGainControl, - deepFilter: params.deepFilter, - deepFilterNoiseReductionLevel: params.deepFilterNoiseReductionLevel, - maxBitrateBps: params.maxBitrateBps, - }; - const configKey = JSON.stringify(publishOptions); - if (this.microphonePublished && this.microphoneConfigKey === configKey) return VOICE_ENGINE_V2_OPERATION_SUCCESS; - const result = await this.bridge.publishMicrophone(publishOptions); - if (!result.ok) { - this.microphonePublished = false; - this.microphoneConfigKey = null; - return result; - } - this.microphonePublished = true; - this.microphoneConfigKey = configKey; - logger.info('Native voice engine microphone publish requested', { - hasDeviceId: requestedDeviceId != null, - }); - return result; - } - - private async waitForInFlightPublish(inFlight: Promise, operation: string): Promise { - try { - await inFlight; - } catch (error) { - logger.warn('Native voice engine in-flight publish failed while being awaited', {operation, error}); - } - } - - private getCameraPublishOptionsConfigKey(params: VoiceEngineV2BridgePublishCameraOptions): string { - return JSON.stringify({ - deviceId: params.deviceId, - width: params.width, - height: params.height, - frameRate: params.frameRate, - mirror: params.mirror, - backgroundMode: params.backgroundMode, - backgroundCustomMediaPath: params.backgroundCustomMediaPath, - backgroundCustomMediaKind: params.backgroundCustomMediaKind, - backgroundBlurStrength: params.backgroundBlurStrength, - }); - } - - private buildCameraCaptureOptions( - params: VoiceEngineV2BridgePublishCameraOptions, - ): VoiceEngineV2BridgePublishCameraOptions { - return { - deviceId: params.deviceId, - width: params.width, - height: params.height, - frameRate: params.frameRate, - ...(params.mirror != null ? {mirror: params.mirror} : {}), - ...(params.backgroundMode ? {backgroundMode: params.backgroundMode} : {}), - ...(params.backgroundCustomMediaPath ? {backgroundCustomMediaPath: params.backgroundCustomMediaPath} : {}), - ...(params.backgroundCustomMediaKind ? {backgroundCustomMediaKind: params.backgroundCustomMediaKind} : {}), - ...(params.backgroundBlurStrength != null ? {backgroundBlurStrength: params.backgroundBlurStrength} : {}), - }; - } - - private getScreenShareAudioConfigKey(params: VoiceEngineV2BridgePublishScreenAudioOptions): string { - return JSON.stringify({ - sampleRate: params.sampleRate, - numChannels: params.numChannels, - route: params.route, - captureId: params.captureId, - tapId: params.tapId, - }); - } - - private getScreenSharePublishOptionsConfigKey(params: VoiceEngineV2BridgePublishScreenOptions): string { - return JSON.stringify({ - captureId: params.captureId, - width: params.width, - height: params.height, - codec: params.codec, - maxBitrateBps: params.maxBitrateBps, - maxFramerate: params.maxFramerate, - adaptiveSend: params.adaptiveSend, - minVideoFps: params.minVideoFps, - maxAudioBufferMs: params.maxAudioBufferMs, - pacing: params.pacing, - trackName: params.trackName, - }); - } - - private getPublishedScreenShareVideo(): NativeVoiceEngineScreenShareVideoPublication | null { - return this.screenShareVideoState.kind === 'published' ? this.screenShareVideoState.publication : null; - } - - private getPublishingScreenShareVideo(): Extract< - NativeVoiceEngineScreenShareVideoState, - {kind: 'publishing'} - > | null { - return this.screenShareVideoState.kind === 'publishing' ? this.screenShareVideoState : null; - } - - private clearScreenShareVideoState(): void { - this.screenShareVideoState = {kind: 'idle'}; - } - - private clearScreenShareAudioState(): void { - this.screenShareAudioState = {kind: 'idle'}; - } - - private isPublishedScreenShareVideoConfig(configKey: string): boolean { - assert.ok(configKey.length > 0, 'screen-share video config key must be non-empty'); - const publication = this.getPublishedScreenShareVideo(); - return publication?.configKey === configKey; - } - - private canUpdateScreenShareInPlace( - previous: VoiceEngineV2BridgePublishScreenOptions, - next: VoiceEngineV2BridgePublishScreenOptions, - ): boolean { - return ( - previous.captureId === next.captureId && - (previous.codec ?? '') === (next.codec ?? '') && - previous.adaptiveSend === next.adaptiveSend && - previous.minVideoFps === next.minVideoFps && - previous.maxAudioBufferMs === next.maxAudioBufferMs && - previous.pacing === next.pacing && - previous.trackName === next.trackName - ); - } - - async publishScreenShare(params: VoiceEngineV2BridgePublishScreenOptions): Promise { - assert.ok(params.captureId.length > 0, 'screen-share publish captureId must be non-empty'); - assert.ok(Number.isFinite(params.width), 'screen-share publish width must be finite'); - assert.ok(params.width > 0, 'screen-share publish width must be positive'); - assert.ok(Number.isFinite(params.height), 'screen-share publish height must be finite'); - assert.ok(params.height > 0, 'screen-share publish height must be positive'); - if (params.codec != null) { - assert.ok(params.codec.length > 0, 'screen-share publish codec must be non-empty'); - } - const configKey = this.getScreenSharePublishOptionsConfigKey(params); - const target: NativeVoiceEngineScreenShareVideoPublication = {kind: 'display', configKey, options: params}; - if (this.isPublishedScreenShareVideoConfig(configKey)) return; - const publishing = this.getPublishingScreenShareVideo(); - if (publishing) { - if (publishing.target.configKey === configKey) { - return publishing.promise; - } - await this.waitForInFlightPublish(publishing.promise, 'publishScreenShare'); - if (this.isPublishedScreenShareVideoConfig(configKey)) return; - } - const published = this.getPublishedScreenShareVideo(); - if (published?.kind === 'display' && this.canUpdateScreenShareInPlace(published.options, params)) { - await this.updateScreenShareEncoding({ - captureId: params.captureId, - width: params.width, - height: params.height, - frameRate: params.maxFramerate, - maxBitrateBps: params.maxBitrateBps, - }); - return; - } - const publish = (async (): Promise => { - if (this.getPublishedScreenShareVideo()) { - await this.bridge.unpublishScreen(); - this.clearScreenShareVideoState(); - } - await this.bridge.publishScreen({ - captureId: params.captureId, - width: params.width, - height: params.height, - codec: params.codec, - maxBitrateBps: params.maxBitrateBps, - maxFramerate: params.maxFramerate, - adaptiveSend: params.adaptiveSend, - minVideoFps: params.minVideoFps, - maxAudioBufferMs: params.maxAudioBufferMs, - pacing: params.pacing, - ...(params.trackName ? {trackName: params.trackName} : {}), - }); - this.screenShareVideoState = {kind: 'published', publication: target}; - logger.info('Native voice engine screen-share publish requested', {captureId: params.captureId}); - })(); - this.screenShareVideoState = {kind: 'publishing', target, promise: publish}; - try { - await publish; - } finally { - const latest = this.getPublishingScreenShareVideo(); - if (latest?.promise === publish) { - this.clearScreenShareVideoState(); - } - } - } - - async updateScreenShareEncoding(params: VoiceEngineV2BridgeUpdateScreenShareEncodingOptions): Promise { - assert.ok(params.captureId.length > 0, 'screen-share encoding update captureId must be non-empty'); - const publishing = this.getPublishingScreenShareVideo(); - if (publishing) { - await this.waitForInFlightPublish(publishing.promise, 'updateScreenShareEncoding'); - } - const published = this.getPublishedScreenShareVideo(); - if (published?.kind !== 'display') { - throw new Error('Native screen-share encoding update requires an active display screen share'); - } - if (published.options.captureId !== params.captureId) { - throw new Error('Native screen-share encoding update captureId does not match the active display share'); - } - await this.bridge.updateScreenShareEncoding({ - captureId: params.captureId, - width: params.width, - height: params.height, - frameRate: params.frameRate, - maxBitrateBps: params.maxBitrateBps, - }); - const options = { - ...published.options, - width: params.width, - height: params.height, - maxBitrateBps: params.maxBitrateBps ?? published.options.maxBitrateBps, - maxFramerate: params.frameRate ?? published.options.maxFramerate, - }; - const configKey = this.getScreenSharePublishOptionsConfigKey(options); - this.screenShareVideoState = {kind: 'published', publication: {kind: 'display', configKey, options}}; - logger.info('Native voice engine screen-share encoding update requested', { - captureId: params.captureId, - width: params.width, - height: params.height, - frameRate: params.frameRate, - maxBitrateBps: params.maxBitrateBps, - }); - } - - async unpublishScreenShare(): Promise { - const publishing = this.getPublishingScreenShareVideo(); - if (publishing) { - await this.waitForInFlightPublish(publishing.promise, 'unpublishScreenShare'); - } - this.clearScreenShareVideoState(); - this.clearScreenShareAudioState(); - await this.bridge.unpublishScreen(); - } - - async publishScreenShareAudio(params: VoiceEngineV2BridgePublishScreenAudioOptions): Promise { - const configKey = this.getScreenShareAudioConfigKey(params); - if (this.screenShareAudioState.kind === 'published' && this.screenShareAudioState.configKey === configKey) return; - await this.bridge.publishScreenAudio({ - sampleRate: params.sampleRate, - numChannels: params.numChannels, - }); - this.screenShareAudioState = {kind: 'published', configKey}; - logger.info('Native voice engine screen-share audio publish requested'); - } - - async pushScreenShareAudioPcm(frame: VoiceEngineV2BridgePcmFrame): Promise { - assert.ok(Number.isFinite(frame.sampleRate), 'pcm frame sampleRate must be finite'); - assert.ok(frame.sampleRate > 0, 'pcm frame sampleRate must be positive'); - assert.ok(frame.numChannels >= PCM_FRAME_CHANNELS_MIN, 'pcm frame must have at least one channel'); - assert.ok(frame.numChannels <= PCM_FRAME_CHANNELS_MAX, 'pcm frame must have at most eight channels'); - assert.ok(frame.samples.byteLength > 0, 'pcm frame samples must be non-empty'); - if (this.screenShareAudioState.kind !== 'published') return false; - return this.bridge.pushScreenAudioPcm({ - sampleRate: frame.sampleRate, - numChannels: frame.numChannels, - samples: frame.samples, - }); - } - - async unpublishScreenShareAudio(): Promise { - this.clearScreenShareAudioState(); - await this.bridge.unpublishScreenAudio(); - } - - async setMicEnabled(enabled: boolean): Promise { - if (!this.connected) { - return enabled ? createNativeMicrophoneNotConnectedResult() : VOICE_ENGINE_V2_OPERATION_SUCCESS; - } - if (enabled && !this.microphonePublished) { - const publishResult = await this.publishMicrophone(); - if (!publishResult.ok) return publishResult; - } - const result = await this.bridge.setMicEnabled(enabled); - if (!result.ok && !enabled) { - logger.warn('Native voice engine failed to disable microphone', {error: result.error}); - } - return result; - } - - async setSpeakingDetection(options: VoiceEngineV2BridgeSpeakingDetectionOptions): Promise { - assert.ok(Number.isFinite(options.localThresholdRms), 'local speaking threshold must be finite'); - assert.ok(Number.isFinite(options.remoteThresholdRms), 'remote speaking threshold must be finite'); - assert.ok(options.localThresholdRms > 0, 'local speaking threshold must be positive'); - assert.ok(options.remoteThresholdRms > 0, 'remote speaking threshold must be positive'); - await this.bridge.setSpeakingDetection(options); - } - - async publishCamera(params: VoiceEngineV2BridgePublishCameraOptions): Promise { - assertCameraOptionInvariants(params, 'camera publish'); - const publishOptions = this.buildCameraCaptureOptions(params); - const configKey = this.getCameraPublishOptionsConfigKey(params); - if (this.cameraPublished && this.cameraConfigKey === configKey) return; - if (this.cameraPublishInFlight) { - if (this.cameraPublishInFlightKey === configKey) { - await this.cameraPublishInFlight; - return; - } - await this.waitForInFlightPublish(this.cameraPublishInFlight, 'publishCamera'); - if (this.cameraPublished && this.cameraConfigKey === configKey) return; - } - const publish = (async (): Promise => { - if (this.cameraPublished) { - await this.bridge.unpublishCamera(); - this.cameraPublished = false; - this.cameraConfigKey = null; - this.cameraTrackSid = null; - } - try { - await this.bridge.publishCamera(publishOptions); - this.cameraTrackSid = null; - } catch (error) { - if (!isCameraAlreadyPublishedError(error)) { - throw error; - } - logger.warn('Native voice engine camera was already published; republishing requested camera config'); - this.cameraPublished = true; - this.cameraConfigKey = null; - this.cameraTrackSid = null; - await this.bridge.unpublishCamera(); - this.cameraPublished = false; - this.cameraConfigKey = null; - this.cameraTrackSid = null; - await this.bridge.publishCamera(publishOptions); - this.cameraTrackSid = null; - } - this.cameraPublished = true; - this.cameraConfigKey = configKey; - logger.info('Native voice engine camera publish requested', { - hasDeviceId: params.deviceId != null, - backgroundMode: params.backgroundMode ?? 'none', - mirror: params.mirror ?? false, - }); - })(); - this.cameraPublishInFlight = publish; - this.cameraPublishInFlightKey = configKey; - try { - await publish; - } finally { - if (this.cameraPublishInFlight === publish) { - this.cameraPublishInFlight = null; - this.cameraPublishInFlightKey = null; - } - } - } - - async updateCameraCapture(params: VoiceEngineV2BridgeUpdateCameraCaptureOptions): Promise { - assertCameraOptionInvariants(params, 'camera capture update'); - if (this.cameraPublishInFlight) { - await this.waitForInFlightPublish(this.cameraPublishInFlight, 'updateCameraCapture'); - } - const updateOptions = this.buildCameraCaptureOptions(params); - const configKey = this.getCameraPublishOptionsConfigKey(params); - assert.ok(configKey.length > 0, 'camera capture update config key must be non-empty'); - if (this.cameraPublished && this.cameraConfigKey === configKey) return; - await this.bridge.updateCameraCapture(updateOptions); - this.cameraPublished = true; - this.cameraConfigKey = configKey; - logger.info('Native voice engine camera capture update requested', { - hasDeviceId: params.deviceId != null, - backgroundMode: params.backgroundMode ?? 'none', - mirror: params.mirror ?? false, - }); - } - - async publishNativeCameraSink( - params: VoiceEngineV2BridgePublishCameraOptions, - ): Promise { - const result = await this.bridge.publishNativeCameraSink(params); - this.cameraPublished = true; - this.cameraConfigKey = `sink:${this.getCameraPublishOptionsConfigKey(params)}`; - this.cameraTrackSid = result.trackSid; - return result; - } - - async publishProcessedCamera( - params: VoiceEngineV2BridgePublishProcessedCameraOptions, - ): Promise { - const publishOptions = { - width: params.width, - height: params.height, - frameRate: params.frameRate, - }; - const configKey = `processed:${JSON.stringify(publishOptions)}`; - if (this.cameraPublished && this.cameraConfigKey === configKey && this.cameraTrackSid) { - return {trackSid: this.cameraTrackSid}; - } - if (this.cameraPublishInFlight) { - await this.waitForInFlightPublish(this.cameraPublishInFlight, 'publishProcessedCamera'); - if (this.cameraPublished && this.cameraConfigKey === configKey && this.cameraTrackSid) { - return {trackSid: this.cameraTrackSid}; - } - } - const publish = (async (): Promise => { - if (this.cameraPublished) { - await this.bridge.unpublishCamera(); - this.cameraPublished = false; - this.cameraConfigKey = null; - this.cameraTrackSid = null; - } - let result: VoiceEngineV2BridgePublishProcessedCameraResult; - try { - result = await this.bridge.publishProcessedCamera(publishOptions); - } catch (error) { - if (!isCameraAlreadyPublishedError(error)) { - throw error; - } - logger.warn('Native voice engine camera was already published; republishing processed camera config'); - this.cameraPublished = true; - this.cameraConfigKey = null; - this.cameraTrackSid = null; - await this.bridge.unpublishCamera(); - this.cameraPublished = false; - this.cameraConfigKey = null; - result = await this.bridge.publishProcessedCamera(publishOptions); - } - this.cameraPublished = true; - this.cameraConfigKey = configKey; - this.cameraTrackSid = result.trackSid; - logger.info('Native voice engine processed camera publish requested', {trackSid: result.trackSid}); - return result; - })(); - this.cameraPublishInFlight = publish; - this.cameraPublishInFlightKey = configKey; - try { - return await publish; - } finally { - if (this.cameraPublishInFlight === publish) { - this.cameraPublishInFlight = null; - this.cameraPublishInFlightKey = null; - } - } - } - - async pushProcessedCameraFrame(frame: VoiceEngineV2BridgeProcessedCameraFrame): Promise { - if (!this.cameraPublished) return false; - return this.bridge.pushProcessedCameraFrame(frame); - } - - async publishDeviceScreenShare(params: VoiceEngineV2BridgePublishDeviceScreenShareOptions): Promise { - const publishOptions = { - deviceId: params.deviceId, - width: params.width, - height: params.height, - frameRate: params.frameRate, - codec: params.codec, - maxBitrateBps: params.maxBitrateBps, - maxFramerate: params.maxFramerate, - }; - const configKey = `device:${JSON.stringify(publishOptions)}`; - const target: NativeVoiceEngineScreenShareVideoPublication = {kind: 'device', configKey}; - if (this.isPublishedScreenShareVideoConfig(configKey)) return; - const publishing = this.getPublishingScreenShareVideo(); - if (publishing) { - if (publishing.target.configKey === configKey) { - return publishing.promise; - } - await this.waitForInFlightPublish(publishing.promise, 'publishDeviceScreenShare'); - if (this.isPublishedScreenShareVideoConfig(configKey)) return; - } - const publish = (async (): Promise => { - if (this.getPublishedScreenShareVideo()) { - await this.bridge.unpublishScreen(); - this.clearScreenShareVideoState(); - } - await this.bridge.publishDeviceScreenShare(publishOptions); - this.screenShareVideoState = {kind: 'published', publication: target}; - logger.info('Native voice engine device screen-share publish requested', { - hasDeviceId: params.deviceId != null, - }); - })(); - this.screenShareVideoState = {kind: 'publishing', target, promise: publish}; - try { - await publish; - } finally { - const latest = this.getPublishingScreenShareVideo(); - if (latest?.promise === publish) { - this.clearScreenShareVideoState(); - } - } - } - - async listCameraDevices(): Promise> { - return this.bridge.listCameraDevices(); - } - - async unpublishCamera(): Promise { - if (this.cameraPublishInFlight) { - await this.waitForInFlightPublish(this.cameraPublishInFlight, 'unpublishCamera'); - } - await this.bridge.unpublishCamera(); - this.cameraPublished = false; - this.cameraConfigKey = null; - this.cameraTrackSid = null; - this.cameraPublishInFlight = null; - this.cameraPublishInFlightKey = null; - } - - async startCameraPreview( - params: VoiceEngineV2BridgeStartCameraPreviewOptions, - ): Promise { - assertCameraOptionInvariants(params, 'camera preview'); - const info = await this.bridge.startCameraPreview(params); - logger.info('Native voice engine camera preview started', { - trackSid: info.trackSid, - width: info.width, - height: info.height, - backgroundMode: params.backgroundMode ?? 'none', - mirror: params.mirror ?? false, - }); - return info; - } - - async stopCameraPreview(): Promise { - await this.bridge.stopCameraPreview(); - } - - async listAudioInputDevices(): Promise> { - return this.bridge.listAudioInputDevices(); - } - - async listAudioOutputDevices(): Promise> { - return this.bridge.listAudioOutputDevices(); - } - - async setAudioOutputDevice(deviceId: string): Promise { - await this.bridge.setAudioOutputDevice(deviceId); - } - - async setParticipantVolume(participantSid: string, volume: number): Promise { - const clamped = Math.max(0, Math.min(2, volume)); - await this.bridge.setParticipantVolume(participantSid, clamped); - } - - async setRemoteTrackSubscription(params: VoiceEngineV2BridgeRemoteTrackSubscriptionOptions): Promise { - await this.bridge.setRemoteTrackSubscription(params); - } - - async publishData(params: VoiceEngineV2BridgePublishDataOptions): Promise { - await this.bridge.publishData({ - payload: toOwnedArrayBuffer(params.payload), - reliable: params.reliable, - topic: params.topic, - destinationIdentities: params.destinationIdentities, - }); - } - - async getConnectionStats(): Promise { - return this.bridge.getConnectionStats(); - } - - onEvent(listener: (event: VoiceEngineV2BridgeEvent) => void): () => void { - return this.bridge.onEvent((event: VoiceEngineV2BridgeEvent) => { - const connectionAction = getNativeVoiceEngineConnectionEventAction(event); - if (connectionAction === 'connected' || connectionAction === 'reconnected') { - this.connected = true; - } else if (connectionAction === 'disconnected') { - this.connected = false; - this.clearPublishedMediaState(); - } - try { - listener(event); - } catch (error) { - logger.error('Native voice engine event listener threw', {type: event.type, error}); - } - }); - } -} diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceEngineSelection.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceEngineSelection.ts deleted file mode 100644 index 407959ce3..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceEngineSelection.ts +++ /dev/null @@ -1,272 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import RuntimeCrash from '@app/features/app/state/RuntimeCrash'; -import {isElectronPlatform} from '@app/features/platform/types/Platform'; -import {resetNativeHardwareEncoderCapabilities} from '@app/features/voice/utils/NativeHardwareEncoderCapabilities'; -import { - VOICE_ENGINE_V2_BRIDGE_METHODS, - VOICE_ENGINE_V2_BRIDGE_VERSION, - VOICE_ENGINE_V2_ENGINE_READY_EVENT_TYPE, - type VoiceEngineV2BridgeApi, - type VoiceEngineV2BridgeCapabilities, - type VoiceEngineV2BridgeMethodName, - type VoiceEngineV2BridgeReadiness, -} from '@fluxer/voice_engine_v2/bridge'; - -export const NATIVE_VOICE_ENGINE_BRIDGE_VERSION = VOICE_ENGINE_V2_BRIDGE_VERSION; -export const NATIVE_VOICE_ENGINE_READINESS_TIMEOUT_MS = 8000; - -export type NativeVoiceEngineUpgradeBlockReason = - | 'missing-bridge' - | 'missing-bridge-version' - | 'unsupported-bridge-version' - | 'incomplete-bridge'; - -export type NativeVoiceEngineSelectionSnapshot = - | {state: 'web'} - | {state: 'uninitialized'} - | {state: 'checking'} - | {state: 'native'} - | {state: 'blocked'; reason: NativeVoiceEngineUpgradeBlockReason} - | {state: 'unsupported'}; - -const REQUIRED_NATIVE_VOICE_ENGINE_METHODS: ReadonlyArray = - VOICE_ENGINE_V2_BRIDGE_METHODS; - -export class NativeVoiceEngineUpgradeRequiredError extends Error { - readonly reason: NativeVoiceEngineUpgradeBlockReason; - - constructor(reason: NativeVoiceEngineUpgradeBlockReason) { - super(`Native voice engine bridge is not available in this desktop app build: ${reason}`); - this.name = 'NativeVoiceEngineUpgradeRequiredError'; - this.reason = reason; - } -} - -class NativeVoiceEngineInvariantError extends Error { - constructor(message: string, cause?: unknown) { - super(cause instanceof Error ? `${message}: ${cause.message}` : message); - this.name = 'NativeVoiceEngineInvariantError'; - } -} - -let nativeSupportState: 'unknown' | 'checking' | 'supported' | 'unsupported' = 'unknown'; -let nativeSupportPromise: Promise | null = null; -let nativePrewarmPromise: Promise | null = null; -let nativeCapabilitiesSnapshot: VoiceEngineV2BridgeCapabilities | null = null; - -const NATIVE_VOICE_ENGINE_FORCE_DISABLED: boolean = true; - -function isNativeVoiceEngineRequired(): boolean { - if (NATIVE_VOICE_ENGINE_FORCE_DISABLED) return false; - return isElectronPlatform(); -} - -function getNativeVoiceEngineBridgeCandidate(): unknown { - return window.electron?.voiceEngine ?? null; -} - -function getNativeVoiceEngineBridgeUpgradeBlockReason(): NativeVoiceEngineUpgradeBlockReason | null { - if (!isElectronPlatform()) return null; - const bridge = getNativeVoiceEngineBridgeCandidate(); - if (!bridge || typeof bridge !== 'object') return 'missing-bridge'; - const record = bridge as Record; - const bridgeVersion = record.bridgeVersion; - if (typeof bridgeVersion !== 'number') return 'missing-bridge-version'; - if (bridgeVersion !== NATIVE_VOICE_ENGINE_BRIDGE_VERSION) return 'unsupported-bridge-version'; - if (REQUIRED_NATIVE_VOICE_ENGINE_METHODS.some((method) => typeof record[method] !== 'function')) { - return 'incomplete-bridge'; - } - return null; -} - -function getNativeVoiceEngineBridge(): VoiceEngineV2BridgeApi | null { - if (!isElectronPlatform()) return null; - if (getNativeVoiceEngineBridgeUpgradeBlockReason()) return null; - return getNativeVoiceEngineBridgeCandidate() as VoiceEngineV2BridgeApi | null; -} - -function shouldSelectNativeFromCachedState(): boolean { - return isNativeVoiceEngineRequired(); -} - -function crashNativeVoiceEngineInvariant(message: string, cause?: unknown): never { - nativeSupportState = 'unsupported'; - const error = new NativeVoiceEngineInvariantError(message, cause); - throw RuntimeCrash.triggerFatalCrash(error); -} - -function prewarmNativeVoiceEngineBridge(bridge: VoiceEngineV2BridgeApi): Promise { - if (!nativePrewarmPromise) { - nativePrewarmPromise = bridge.prewarm().catch((error: unknown) => { - nativePrewarmPromise = null; - throw error; - }); - } - return nativePrewarmPromise; -} - -export function isNativeVoiceEngineSelected(): boolean { - if (!shouldSelectNativeFromCachedState()) return false; - const bridge = getNativeVoiceEngineBridge(); - if (!bridge) return false; - return nativeSupportState === 'supported' && shouldSelectNativeFromCachedState(); -} - -export function isNativeVoiceEngineSelectionPending(): boolean { - if (!shouldSelectNativeFromCachedState()) return false; - if (getNativeVoiceEngineBridgeUpgradeBlockReason()) return true; - return nativeSupportState === 'checking' || nativeSupportState === 'unknown'; -} - -export function getNativeVoiceEngineUpgradeBlockReason(): NativeVoiceEngineUpgradeBlockReason | null { - if (!shouldSelectNativeFromCachedState()) return null; - return getNativeVoiceEngineBridgeUpgradeBlockReason(); -} - -export function getNativeVoiceEngineSelectionSnapshot(): NativeVoiceEngineSelectionSnapshot { - if (!shouldSelectNativeFromCachedState()) return {state: 'web'}; - const upgradeBlockReason = getNativeVoiceEngineBridgeUpgradeBlockReason(); - if (upgradeBlockReason) return {state: 'blocked', reason: upgradeBlockReason}; - if (nativeSupportState === 'supported') return {state: 'native'}; - if (nativeSupportState === 'checking') return {state: 'checking'}; - if (nativeSupportState === 'unsupported') return {state: 'unsupported'}; - return {state: 'uninitialized'}; -} - -export function getNativeVoiceEngineCapabilitiesSnapshot(): VoiceEngineV2BridgeCapabilities | null { - return nativeCapabilitiesSnapshot; -} - -export async function refreshNativeVoiceEngineCapabilitiesSnapshot(): Promise { - if (nativeCapabilitiesSnapshot) return nativeCapabilitiesSnapshot; - const bridge = getNativeVoiceEngineBridge(); - if (!bridge) return null; - const capabilities = await bridge.getCapabilities(); - nativeCapabilitiesSnapshot = capabilities; - return capabilities; -} - -function waitForNativeVoiceEngineReadyEvent( - bridge: VoiceEngineV2BridgeApi, - initialReason: string | undefined, - timeoutMs: number, -): Promise { - return new Promise((resolve) => { - let settled = false; - let dispose: (() => void) | null = null; - let timeoutId: ReturnType | null = null; - const settle = (readiness: VoiceEngineV2BridgeReadiness): void => { - if (settled) return; - settled = true; - if (timeoutId !== null) clearTimeout(timeoutId); - dispose?.(); - resolve(readiness); - }; - dispose = bridge.onEvent((event) => { - if (event.type === VOICE_ENGINE_V2_ENGINE_READY_EVENT_TYPE) { - settle({ready: true}); - } - }); - timeoutId = setTimeout(() => { - settle({ready: false, reason: initialReason ?? 'readiness-timeout'}); - }, timeoutMs); - void bridge - .getVoiceEngineReadiness() - .then((readiness) => { - if (readiness.ready) settle(readiness); - }) - .catch(() => {}); - }); -} - -export async function awaitNativeVoiceEngineReadiness( - timeoutMs: number = NATIVE_VOICE_ENGINE_READINESS_TIMEOUT_MS, -): Promise { - const bridge = getNativeVoiceEngineBridge(); - if (!bridge) return {ready: false, reason: 'missing-bridge'}; - let initial: VoiceEngineV2BridgeReadiness; - try { - initial = await bridge.getVoiceEngineReadiness(); - } catch (error) { - return {ready: false, reason: error instanceof Error ? error.message : 'readiness-query-failed'}; - } - if (initial.ready) return initial; - return waitForNativeVoiceEngineReadyEvent(bridge, initial.reason, timeoutMs); -} - -export function isNativeVoiceEngineUpgradeRequiredError( - error: unknown, -): error is NativeVoiceEngineUpgradeRequiredError { - return error instanceof NativeVoiceEngineUpgradeRequiredError; -} - -export async function initializeNativeVoiceEngineSelectionForStartup(): Promise { - if (!shouldSelectNativeFromCachedState()) return getNativeVoiceEngineSelectionSnapshot(); - const selected = await shouldUseNativeVoiceEngine(); - if (!selected) { - throw new Error('Native voice engine startup selection did not select native in Electron'); - } - return getNativeVoiceEngineSelectionSnapshot(); -} - -export async function shouldUseNativeVoiceEngine(): Promise { - if (!shouldSelectNativeFromCachedState()) return false; - const upgradeBlockReason = getNativeVoiceEngineBridgeUpgradeBlockReason(); - if (upgradeBlockReason) { - throw new NativeVoiceEngineUpgradeRequiredError(upgradeBlockReason); - } - const bridge = getNativeVoiceEngineBridge(); - if (!bridge) { - throw new NativeVoiceEngineUpgradeRequiredError('missing-bridge'); - } - if (nativeSupportState === 'supported') { - return true; - } - if (nativeSupportState === 'unsupported') { - crashNativeVoiceEngineInvariant('Native voice engine bridge is current but the native backend is unsupported'); - } - if (nativeSupportPromise) return nativeSupportPromise; - nativeSupportState = 'checking'; - nativeSupportPromise = bridge - .isSupported() - .then(async (supported) => { - if (!supported) { - crashNativeVoiceEngineInvariant('Native voice engine bridge is current but support probing returned false'); - } - const capabilities = await bridge.getCapabilities(); - if (!capabilities.microphoneCapture) { - crashNativeVoiceEngineInvariant( - 'Native voice engine bridge is current but the native backend cannot capture device microphone audio', - ); - } - nativeCapabilitiesSnapshot = capabilities; - await prewarmNativeVoiceEngineBridge(bridge); - const readiness = await bridge.getVoiceEngineReadiness(); - if (!readiness?.ready) { - crashNativeVoiceEngineInvariant( - `Native voice engine reported not-ready after a successful prewarm: ${readiness?.reason ?? 'unknown'}`, - ); - } - nativeSupportState = 'supported'; - return true; - }) - .catch((error) => { - if (error instanceof NativeVoiceEngineInvariantError) { - throw error; - } - crashNativeVoiceEngineInvariant('Native voice engine support probing failed', error); - }) - .finally(() => { - nativeSupportPromise = null; - }); - return nativeSupportPromise; -} - -export function resetNativeVoiceEngineSelectionForTesting(): void { - nativeSupportState = 'unknown'; - nativeSupportPromise = null; - nativePrewarmPromise = null; - nativeCapabilitiesSnapshot = null; - resetNativeHardwareEncoderCapabilities(); -} diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceFrameStatsBatcher.test.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceFrameStatsBatcher.test.ts deleted file mode 100644 index 026260962..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceFrameStatsBatcher.test.ts +++ /dev/null @@ -1,160 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import { - NativeVoiceFrameStatsBatcher, - type NativeVoiceFrameStatsScheduler, -} from '@app/features/voice/engine/native_voice_engine/NativeVoiceFrameStatsBatcher'; -import { - VOICE_ENGINE_V2_FRAME_STATS_FLUSH_INTERVAL_MS, - VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP, - type VoiceEngineV2FrameStatsEvent, -} from '@fluxer/voice_engine_v2'; -import type {VoiceEngineV2BridgeVideoFrame} from '@fluxer/voice_engine_v2/bridge'; -import {describe, expect, it} from 'vitest'; - -interface ManualScheduler extends NativeVoiceFrameStatsScheduler { - fire(): void; - readonly activeIntervalCount: number; - readonly scheduledIntervalsMs: Array; -} - -function createManualScheduler(): ManualScheduler { - const callbacks = new Map void>(); - const scheduledIntervalsMs: Array = []; - let nextHandle = 1; - return { - setInterval(callback, ms) { - const handle = nextHandle++; - callbacks.set(handle, callback); - scheduledIntervalsMs.push(ms); - return handle; - }, - clearInterval(handle) { - callbacks.delete(handle as number); - }, - fire() { - for (const callback of [...callbacks.values()]) { - callback(); - } - }, - get activeIntervalCount() { - return callbacks.size; - }, - scheduledIntervalsMs, - }; -} - -function makeBridgeFrame( - overrides: Partial = {}, -): VoiceEngineV2BridgeVideoFrame { - return { - meta: { - participantSid: 'PA_alice', - trackSid: 'TR_screen', - width: 1280, - height: 720, - timestampUs: 33_333, - ...overrides, - }, - data: new ArrayBuffer(16), - }; -} - -function createBatcher() { - const scheduler = createManualScheduler(); - const dispatched: Array = []; - const droppedReports: Array = []; - const batcher = new NativeVoiceFrameStatsBatcher({ - dispatch: (event) => dispatched.push(event), - scheduler, - onDroppedUpdates: (count) => droppedReports.push(count), - }); - return {batcher, scheduler, dispatched, droppedReports}; -} - -describe('NativeVoiceFrameStatsBatcher', () => { - it('does not dispatch per frame and emits one absolute stats event per track on the flush cadence', () => { - const {batcher, scheduler, dispatched} = createBatcher(); - for (let index = 1; index <= 30; index += 1) { - batcher.recordFrame(makeBridgeFrame({timestampUs: index * 33_333})); - } - batcher.recordFrame(makeBridgeFrame({trackSid: 'TR_camera', timestampUs: 999})); - expect(dispatched).toHaveLength(0); - expect(scheduler.scheduledIntervalsMs).toEqual([VOICE_ENGINE_V2_FRAME_STATS_FLUSH_INTERVAL_MS]); - - scheduler.fire(); - expect(dispatched).toHaveLength(2); - expect(dispatched[0]).toEqual({ - type: 'inboundVideo.frameStats', - stats: { - participantSid: 'PA_alice', - trackSid: 'TR_screen', - width: 1280, - height: 720, - frameCount: 30, - lastFrameTimestampUs: 30 * 33_333, - lastFrameByteLength: 16, - }, - }); - - scheduler.fire(); - expect(dispatched).toHaveLength(2); - }); - - it('flushes pending counts when a track is removed', () => { - const {batcher, scheduler, dispatched} = createBatcher(); - batcher.recordFrame(makeBridgeFrame()); - batcher.removeTrack('TR_screen'); - expect(dispatched).toHaveLength(1); - expect(dispatched[0]?.stats.frameCount).toBe(1); - expect(batcher.trackedTracksCount).toBe(0); - - scheduler.fire(); - expect(dispatched).toHaveLength(1); - }); - - it('flushes final counts on teardown, cancels the flush timer, and clears tracked state', () => { - const {batcher, scheduler, dispatched} = createBatcher(); - batcher.recordFrame(makeBridgeFrame({timestampUs: 1000})); - batcher.recordFrame(makeBridgeFrame({timestampUs: 2000})); - expect(batcher.isFlushTimerActive).toBe(true); - expect(scheduler.activeIntervalCount).toBe(1); - - batcher.teardown(); - expect(dispatched).toHaveLength(1); - expect(dispatched[0]?.stats.frameCount).toBe(2); - expect(dispatched[0]?.stats.lastFrameTimestampUs).toBe(2000); - expect(batcher.isFlushTimerActive).toBe(false); - expect(scheduler.activeIntervalCount).toBe(0); - expect(batcher.trackedTracksCount).toBe(0); - - scheduler.fire(); - expect(dispatched).toHaveLength(1); - batcher.teardown(); - expect(dispatched).toHaveLength(1); - }); - - it('keeps recording after teardown by starting a fresh flush timer', () => { - const {batcher, scheduler, dispatched} = createBatcher(); - batcher.recordFrame(makeBridgeFrame({timestampUs: 1000})); - batcher.teardown(); - expect(dispatched).toHaveLength(1); - - batcher.recordFrame(makeBridgeFrame({timestampUs: 5000})); - expect(batcher.isFlushTimerActive).toBe(true); - scheduler.fire(); - expect(dispatched).toHaveLength(2); - expect(dispatched[1]?.stats.frameCount).toBe(1); - }); - - it('reports the first dropped update beyond the track cap', () => { - const {batcher, droppedReports} = createBatcher(); - for (let index = 0; index < VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP; index += 1) { - batcher.recordFrame(makeBridgeFrame({trackSid: `TR_${index}`})); - } - expect(droppedReports).toEqual([]); - batcher.recordFrame(makeBridgeFrame({trackSid: 'TR_overflow'})); - expect(droppedReports).toEqual([1]); - expect(batcher.droppedUpdatesCount).toBe(1); - }); -}); diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceFrameStatsBatcher.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceFrameStatsBatcher.ts deleted file mode 100644 index cb704e6ac..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceFrameStatsBatcher.ts +++ /dev/null @@ -1,136 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; -import { - VOICE_ENGINE_V2_FRAME_STATS_FLUSH_INTERVAL_MS, - VoiceEngineV2FrameStatsAccumulator, - type VoiceEngineV2FrameStatsEvent, - type VoiceEngineV2InboundVideoFrame, -} from '@fluxer/voice_engine_v2'; -import type {VoiceEngineV2BridgeVideoFrame} from '@fluxer/voice_engine_v2/bridge'; - -export const NATIVE_VOICE_FRAME_STATS_DROP_REPORT_INTERVAL = 300; - -type NativeVoiceFrameStatsIntervalHandle = unknown; - -export interface NativeVoiceFrameStatsScheduler { - setInterval(callback: () => void, ms: number): NativeVoiceFrameStatsIntervalHandle; - clearInterval(handle: NativeVoiceFrameStatsIntervalHandle): void; -} - -export interface NativeVoiceFrameStatsBatcherOptions { - dispatch: (event: VoiceEngineV2FrameStatsEvent) => void; - scheduler?: NativeVoiceFrameStatsScheduler; - onDroppedUpdates?: (droppedUpdatesCount: number) => void; -} - -function defaultScheduler(): NativeVoiceFrameStatsScheduler { - return { - setInterval(callback, ms): NativeVoiceFrameStatsIntervalHandle { - return globalThis.setInterval(callback, ms); - }, - clearInterval(handle): void { - globalThis.clearInterval(handle as ReturnType); - }, - }; -} - -export class NativeVoiceFrameStatsBatcher { - private readonly dispatch: (event: VoiceEngineV2FrameStatsEvent) => void; - private readonly scheduler: NativeVoiceFrameStatsScheduler; - private readonly onDroppedUpdates: ((droppedUpdatesCount: number) => void) | null; - private readonly accumulator = new VoiceEngineV2FrameStatsAccumulator(); - private readonly scratchFrame: VoiceEngineV2InboundVideoFrame = { - participantSid: '', - trackSid: '', - width: 0, - height: 0, - timestampUs: 0, - byteLength: 0, - }; - private intervalHandle: NativeVoiceFrameStatsIntervalHandle | null = null; - - constructor(options: NativeVoiceFrameStatsBatcherOptions) { - assert.ok(options != null, 'frame stats batcher requires options'); - assert.equal(typeof options.dispatch, 'function', 'frame stats batcher requires a dispatch function'); - this.dispatch = options.dispatch; - this.scheduler = options.scheduler ?? defaultScheduler(); - assert.equal(typeof this.scheduler.setInterval, 'function', 'frame stats scheduler must expose setInterval'); - assert.equal(typeof this.scheduler.clearInterval, 'function', 'frame stats scheduler must expose clearInterval'); - this.onDroppedUpdates = options.onDroppedUpdates ?? null; - } - - get isFlushTimerActive(): boolean { - return this.intervalHandle !== null; - } - - get droppedUpdatesCount(): number { - return this.accumulator.droppedUpdatesCount; - } - - get trackedTracksCount(): number { - return this.accumulator.trackedTracksCount; - } - - recordFrame(frame: VoiceEngineV2BridgeVideoFrame): void { - assert.ok(frame != null, 'recordFrame requires a frame'); - assert.ok(frame.meta != null, 'recordFrame requires frame metadata'); - const scratch = this.scratchFrame; - scratch.participantSid = frame.meta.participantSid; - scratch.trackSid = frame.meta.trackSid; - scratch.width = frame.meta.width; - scratch.height = frame.meta.height; - scratch.timestampUs = frame.meta.timestampUs; - scratch.byteLength = frame.data.byteLength; - const recorded = this.accumulator.record(scratch); - if (!recorded) { - this.noteDroppedUpdate(); - } - this.ensureFlushTimer(); - } - - flush(): number { - if (this.accumulator.dirtyTracksCount === 0) return 0; - const flushedCount = this.accumulator.flushDirty(this.dispatch); - assert.ok(flushedCount >= 1, 'flush of a dirty accumulator must emit at least one event'); - assert.equal(this.accumulator.dirtyTracksCount, 0, 'flush must leave no dirty tracks behind'); - return flushedCount; - } - - removeTrack(trackSid: string): void { - assert.equal(typeof trackSid, 'string', 'removeTrack requires a string trackSid'); - if (trackSid.length === 0) return; - this.flush(); - this.accumulator.removeTrack(trackSid); - } - - teardown(): void { - try { - this.flush(); - } finally { - if (this.intervalHandle !== null) { - this.scheduler.clearInterval(this.intervalHandle); - this.intervalHandle = null; - } - this.accumulator.clear(); - } - assert.equal(this.intervalHandle, null, 'teardown must cancel the flush timer'); - assert.equal(this.accumulator.dirtyTracksCount, 0, 'teardown must leave no dirty tracks behind'); - } - - private ensureFlushTimer(): void { - if (this.intervalHandle !== null) return; - this.intervalHandle = this.scheduler.setInterval(() => { - this.flush(); - }, VOICE_ENGINE_V2_FRAME_STATS_FLUSH_INTERVAL_MS); - assert.ok(this.intervalHandle != null, 'frame stats scheduler must return an interval handle'); - } - - private noteDroppedUpdate(): void { - const droppedUpdatesCount = this.accumulator.droppedUpdatesCount; - assert.ok(droppedUpdatesCount >= 1, 'dropped update accounting requires at least one drop'); - if (droppedUpdatesCount === 1 || droppedUpdatesCount % NATIVE_VOICE_FRAME_STATS_DROP_REPORT_INTERVAL === 0) { - this.onDroppedUpdates?.(droppedUpdatesCount); - } - } -} diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceStatsStore.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceStatsStore.ts deleted file mode 100644 index cbecb707f..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/NativeVoiceStatsStore.ts +++ /dev/null @@ -1,251 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import { - mapNativeStatsToOverrides, - mapNativeStatsToPerTrackStats, -} from '@app/features/voice/engine/native_voice_engine/nativeVoiceStatsMapper'; -import {Store} from '@app/features/voice/engine/Store'; -import { - createVoiceLatencySnapshot, - type LatencyDataPoint, - observeVoiceLatencyGap, - recordVoiceLatencySample, - resetVoiceLatencySnapshot, - startVoiceLatencyTracking, - type VoiceLatencySnapshot, - type VoiceLatencyStatus, -} from '@app/features/voice/engine/VoiceLatencyTracker'; -import {createInitialVoiceStats} from '@app/features/voice/engine/VoiceStatsStateMachine'; -import type { - VoiceEngineV2PerTrackStats, - VoiceEngineV2Stats, - VoiceEngineV2StatsSample, - VoiceEngineV2VoiceStats, -} from '@fluxer/voice_engine_v2'; - -interface NativeVoiceStatsStoreSetStatsOptions { - connectionId?: string | null; - mergeSparseTrackStats?: boolean; -} - -interface NativeVoiceStatsStoreVoiceStatsOptions { - participantCount?: number; -} - -function mergeNativeVoiceStats( - previous: VoiceEngineV2Stats | null, - stats: VoiceEngineV2Stats, - options: NativeVoiceStatsStoreSetStatsOptions, -): VoiceEngineV2Stats { - const preserveOutbound = - options.mergeSparseTrackStats === true && stats.outbound.length === 0 && (previous?.outbound.length ?? 0) > 0; - const preserveInbound = - options.mergeSparseTrackStats === true && stats.inbound.length === 0 && (previous?.inbound.length ?? 0) > 0; - return { - ...stats, - outbound: preserveOutbound ? (previous?.outbound ?? []) : stats.outbound, - inbound: preserveInbound ? (previous?.inbound ?? []) : stats.inbound, - droppedVideoFrameCallbacks: stats.droppedVideoFrameCallbacks ?? previous?.droppedVideoFrameCallbacks, - droppedNativeVideoFrames: stats.droppedNativeVideoFrames ?? previous?.droppedNativeVideoFrames, - send: stats.send === undefined ? previous?.send : stats.send, - }; -} - -function nativeStatsToVoiceStats( - stats: VoiceEngineV2Stats | null, - duration: number, - currentLatency: number | null, - participantCount: number, -): VoiceEngineV2VoiceStats { - const initial = createInitialVoiceStats(); - if (!stats) return {...initial, duration, participantCount}; - const overrides = mapNativeStatsToOverrides(stats); - return { - audioSendBitrate: overrides.network.audioSendBitrateKbps, - audioRecvBitrate: overrides.network.audioRecvBitrateKbps, - videoSendBitrate: overrides.network.videoSendBitrateKbps, - videoRecvBitrate: overrides.network.videoRecvBitrateKbps, - audioPacketLoss: overrides.network.audioPacketLossPercent, - videoPacketLoss: overrides.network.videoPacketLossPercent, - rtt: currentLatency ?? overrides.network.rttMs ?? 0, - jitter: overrides.network.jitterMs, - participantCount, - duration, - }; -} - -function appendNativeStatsSample( - timeSeries: ReadonlyArray, - sample: VoiceEngineV2StatsSample, - limit: number, -): Array { - if (limit <= 0) return []; - const retainedCount = Math.min(timeSeries.length, limit - 1); - const nextTimeSeries = new Array(retainedCount + 1); - const startIndex = timeSeries.length - retainedCount; - for (let i = 0; i < retainedCount; i += 1) { - nextTimeSeries[i] = timeSeries[startIndex + i]; - } - nextTimeSeries[retainedCount] = sample; - return nextTimeSeries; -} - -function replaceLastNativeStatsSample( - timeSeries: ReadonlyArray, - sample: VoiceEngineV2StatsSample, -): Array { - if (timeSeries.length === 0) return [sample]; - const nextTimeSeries = new Array(timeSeries.length); - for (let i = 0; i < timeSeries.length - 1; i += 1) { - nextTimeSeries[i] = timeSeries[i]; - } - nextTimeSeries[timeSeries.length - 1] = sample; - return nextTimeSeries; -} - -class NativeVoiceStatsStore extends Store { - private _stats: VoiceEngineV2Stats | null = null; - private _connectionId: string | null = null; - private _latency: VoiceLatencySnapshot = createVoiceLatencySnapshot(); - private _sessionStartedAt: number | null = null; - private _clockNow: number = Date.now(); - private _statsTimeSeries: Array = []; - - get stats(): VoiceEngineV2Stats | null { - return this._stats; - } - - get connectionId(): string | null { - return this._connectionId; - } - - get currentLatency(): number | null { - return this._latency.currentLatency; - } - - get averageLatency(): number | null { - return this._latency.averageLatency; - } - - get latencyHistory(): Array { - return this._latency.latencyHistory; - } - - get latencyStatus(): VoiceLatencyStatus { - return this._latency.status; - } - - get duration(): number { - return this._sessionStartedAt ? Math.floor((this._clockNow - this._sessionStartedAt) / 1000) : 0; - } - - get statsTimeSeries(): Array { - return this._statsTimeSeries; - } - - get perTrackStats(): Array { - return this._stats ? mapNativeStatsToPerTrackStats(this._stats) : []; - } - - getVoiceStats(options: NativeVoiceStatsStoreVoiceStatsOptions = {}): VoiceEngineV2VoiceStats { - return nativeStatsToVoiceStats(this._stats, this.duration, this.currentLatency, options.participantCount ?? 0); - } - - startSession(now: number = Date.now()): void { - this.update(() => { - this._sessionStartedAt = now; - this._clockNow = now; - this._latency = resetVoiceLatencySnapshot({tracking: true, now}); - this._stats = null; - this._statsTimeSeries = []; - }); - } - - tick(timestamp: number = Date.now()): void { - this.update(() => { - this._clockNow = timestamp; - if (this._sessionStartedAt != null) { - this._latency = observeVoiceLatencyGap(this._latency, timestamp); - } - }); - } - - setConnectionId(connectionId: string | null): void { - if (this._connectionId === connectionId) return; - this.update(() => { - this._connectionId = connectionId; - this._stats = null; - }); - } - - setStats( - stats: VoiceEngineV2Stats | null, - timestamp: number = Date.now(), - options: NativeVoiceStatsStoreSetStatsOptions = {}, - ): void { - if (options.connectionId !== undefined) { - if (this._connectionId !== null && options.connectionId !== this._connectionId) return; - } - this.update(() => { - if (options.connectionId !== undefined && this._connectionId === null) { - this._connectionId = options.connectionId; - } - this._clockNow = timestamp; - this._stats = stats ? mergeNativeVoiceStats(this._stats, stats, options) : stats; - const latency = - stats && !this._latency.tracking ? startVoiceLatencyTracking(this._latency, timestamp) : this._latency; - this._latency = - stats && typeof stats.rttMs === 'number' - ? recordVoiceLatencySample(latency, timestamp, stats.rttMs) - : observeVoiceLatencyGap(latency, timestamp); - if (stats && this._sessionStartedAt == null) { - this._sessionStartedAt = timestamp; - } - if (this._stats) { - const overrides = mapNativeStatsToOverrides(this._stats); - const sample: VoiceEngineV2StatsSample = { - timestamp, - rtt: this.currentLatency ?? this._stats.rttMs ?? 0, - jitter: overrides.network.jitterMs, - audioPacketLoss: overrides.network.audioPacketLossPercent, - videoPacketLoss: overrides.network.videoPacketLossPercent, - audioSendBitrate: overrides.network.audioSendBitrateKbps, - audioRecvBitrate: overrides.network.audioRecvBitrateKbps, - videoSendBitrate: overrides.network.videoSendBitrateKbps, - videoRecvBitrate: overrides.network.videoRecvBitrateKbps, - }; - const previous = this._statsTimeSeries.at(-1); - if (previous && Math.floor(previous.timestamp / 1000) === Math.floor(timestamp / 1000)) { - this._statsTimeSeries = replaceLastNativeStatsSample(this._statsTimeSeries, sample); - } else { - this._statsTimeSeries = appendNativeStatsSample(this._statsTimeSeries, sample, 60); - } - } - }); - } - - clear(): void { - if ( - this._stats == null && - this._connectionId == null && - this._latency.latencyHistory.length === 0 && - this._sessionStartedAt == null && - this._statsTimeSeries.length === 0 - ) { - return; - } - this.update(() => { - this._stats = null; - this._connectionId = null; - this._latency = resetVoiceLatencySnapshot(); - this._sessionStartedAt = null; - this._clockNow = Date.now(); - this._statsTimeSeries = []; - }); - } -} - -const instance = new NativeVoiceStatsStore(); - -export default instance; -export {NativeVoiceStatsStore}; diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/VoiceEngine.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/VoiceEngine.ts deleted file mode 100644 index 276b6e0d1..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/VoiceEngine.ts +++ /dev/null @@ -1,64 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import type { - VoiceEngineV2BridgeAudioInputDevice, - VoiceEngineV2BridgeAudioOutputDevice, - VoiceEngineV2BridgeCameraDevice, - VoiceEngineV2BridgeCapabilities, - VoiceEngineV2BridgeConnectOptions, - VoiceEngineV2BridgeEvent, - VoiceEngineV2BridgeOperationResult, - VoiceEngineV2BridgePcmFrame, - VoiceEngineV2BridgeProcessedCameraFrame, - VoiceEngineV2BridgePublishCameraOptions, - VoiceEngineV2BridgePublishDataOptions, - VoiceEngineV2BridgePublishDeviceScreenShareOptions, - VoiceEngineV2BridgePublishMicrophoneOptions, - VoiceEngineV2BridgePublishNativeCameraSinkResult, - VoiceEngineV2BridgePublishProcessedCameraOptions, - VoiceEngineV2BridgePublishProcessedCameraResult, - VoiceEngineV2BridgePublishScreenAudioOptions, - VoiceEngineV2BridgePublishScreenOptions, - VoiceEngineV2BridgeRemoteTrackSubscriptionOptions, - VoiceEngineV2BridgeSpeakingDetectionOptions, - VoiceEngineV2BridgeStats, - VoiceEngineV2BridgeUpdateScreenShareEncodingOptions, -} from '@fluxer/voice_engine_v2/bridge'; - -export interface VoiceEngine { - readonly kind: 'native'; - getCapabilities(): Promise; - prewarm(): Promise; - connect(params: VoiceEngineV2BridgeConnectOptions): Promise; - disconnect(): Promise; - isConnected(): boolean; - publishMicrophone(params?: VoiceEngineV2BridgePublishMicrophoneOptions): Promise; - publishScreenShare(params: VoiceEngineV2BridgePublishScreenOptions): Promise; - updateScreenShareEncoding(params: VoiceEngineV2BridgeUpdateScreenShareEncodingOptions): Promise; - unpublishScreenShare(): Promise; - publishScreenShareAudio(params: VoiceEngineV2BridgePublishScreenAudioOptions): Promise; - pushScreenShareAudioPcm(frame: VoiceEngineV2BridgePcmFrame): Promise; - unpublishScreenShareAudio(): Promise; - setMicEnabled(enabled: boolean): Promise; - setSpeakingDetection(options: VoiceEngineV2BridgeSpeakingDetectionOptions): Promise; - publishCamera(params: VoiceEngineV2BridgePublishCameraOptions): Promise; - publishNativeCameraSink( - params: VoiceEngineV2BridgePublishCameraOptions, - ): Promise; - publishProcessedCamera( - params: VoiceEngineV2BridgePublishProcessedCameraOptions, - ): Promise; - pushProcessedCameraFrame(frame: VoiceEngineV2BridgeProcessedCameraFrame): Promise; - publishDeviceScreenShare(params: VoiceEngineV2BridgePublishDeviceScreenShareOptions): Promise; - listCameraDevices(): Promise>; - unpublishCamera(): Promise; - isPublishingCamera(): boolean; - listAudioInputDevices(): Promise>; - listAudioOutputDevices(): Promise>; - setAudioOutputDevice(deviceId: string): Promise; - setParticipantVolume(participantSid: string, volume: number): Promise; - setRemoteTrackSubscription(options: VoiceEngineV2BridgeRemoteTrackSubscriptionOptions): Promise; - publishData(params: VoiceEngineV2BridgePublishDataOptions): Promise; - getConnectionStats(): Promise; - onEvent(listener: (event: VoiceEngineV2BridgeEvent) => void): () => void; -} diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/createInboundVideoBridge.test.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/createInboundVideoBridge.test.ts deleted file mode 100644 index d3f8ccdd4..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/createInboundVideoBridge.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -// @vitest-environment happy-dom -// SPDX-License-Identifier: AGPL-3.0-or-later - -import { - createInboundVideoBridge, - i420VideoFrameLayout, - tightI420ByteLength, -} from '@app/features/voice/engine/native_voice_engine/createInboundVideoBridge'; -import {afterEach, beforeEach, describe, expect, it} from 'vitest'; - -class FakeVideoFrame { - closed = false; - constructor( - public readonly data: unknown, - public readonly init: {format: string; codedWidth: number; codedHeight: number; layout?: unknown}, - ) {} - close(): void { - this.closed = true; - } -} - -class FakeWriter { - written: Array = []; - closed = false; - write(chunk: FakeVideoFrame): Promise { - this.written.push(chunk); - return Promise.resolve(); - } - close(): Promise { - this.closed = true; - return Promise.resolve(); - } -} - -class FakeGeneratorTrack { - kind = 'video'; - stopped = false; - readonly writer = new FakeWriter(); - readonly writable = {getWriter: () => this.writer}; - stop(): void { - this.stopped = true; - } -} - -class FakeMediaStream { - constructor(public readonly tracks: Array) {} -} - -beforeEach(() => { - Reflect.set(window, 'VideoFrame', FakeVideoFrame); - Reflect.set(window, 'MediaStreamTrackGenerator', FakeGeneratorTrack); - Reflect.set(window, 'MediaStream', FakeMediaStream); -}); - -afterEach(() => { - Reflect.deleteProperty(window, 'VideoFrame'); - Reflect.deleteProperty(window, 'MediaStreamTrackGenerator'); - Reflect.deleteProperty(window, 'MediaStream'); -}); - -describe('i420VideoFrameLayout', () => { - it('lays out tightly packed Y/U/V planes', () => { - const layout = i420VideoFrameLayout(320, 240); - expect(layout).toEqual([ - {offset: 0, stride: 320}, - {offset: 320 * 240, stride: 160}, - {offset: 320 * 240 + 160 * 120, stride: 160}, - ]); - }); -}); - -describe('tightI420ByteLength', () => { - it('computes Y + U + V for the dims', () => { - expect(tightI420ByteLength(320, 240)).toBe(320 * 240 + 2 * 160 * 120); - expect(tightI420ByteLength(2, 2)).toBe(4 + 2 * 1 * 1); - }); -}); - -describe('createInboundVideoBridge', () => { - it('returns null when WebCodecs primitives are unavailable', () => { - Reflect.deleteProperty(window, 'VideoFrame'); - expect(createInboundVideoBridge('k')).toBeNull(); - }); - - it('builds a track + stream and converts a valid I420 frame to a VideoFrame', async () => { - const bridge = createInboundVideoBridge('PA_1:TR_9'); - expect(bridge).not.toBeNull(); - expect(bridge!.stream).toBeInstanceOf(FakeMediaStream); - const generator = bridge!.track as unknown as FakeGeneratorTrack; - bridge!.pushFrame({ - width: 320, - height: 240, - timestampUs: 1000, - data: new ArrayBuffer(tightI420ByteLength(320, 240)), - }); - await Promise.resolve(); - await Promise.resolve(); - expect(generator.writer.written.length).toBe(1); - const frame = generator.writer.written[0] as unknown as FakeVideoFrame; - expect(frame.init.format).toBe('I420'); - expect(frame.init.codedWidth).toBe(320); - expect(frame.init.codedHeight).toBe(240); - }); - - it('drops frames whose buffer is too small for the dims', async () => { - const bridge = createInboundVideoBridge('PA_1:TR_9'); - const generator = bridge!.track as unknown as FakeGeneratorTrack; - bridge!.pushFrame({width: 320, height: 240, timestampUs: 0, data: new ArrayBuffer(10)}); - await Promise.resolve(); - expect(generator.writer.written.length).toBe(0); - }); - - it('drops frames after cleanup', async () => { - const bridge = createInboundVideoBridge('PA_1:TR_9'); - const generator = bridge!.track as unknown as FakeGeneratorTrack; - await bridge!.cleanup(); - bridge!.pushFrame({width: 320, height: 240, timestampUs: 0, data: new ArrayBuffer(tightI420ByteLength(320, 240))}); - await Promise.resolve(); - expect(generator.writer.written.length).toBe(0); - expect(generator.stopped).toBe(true); - }); -}); diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/createInboundVideoBridge.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/createInboundVideoBridge.ts deleted file mode 100644 index 225f90c76..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/createInboundVideoBridge.ts +++ /dev/null @@ -1,231 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; -import {Logger} from '@app/features/platform/utils/AppLogger'; -import { - getGeneratorVideoCtor, - getVideoFrameCtor, - MAX_GENERATOR_PENDING_VIDEO_FRAMES, - type RendererVideoFrame, - type VideoFrameInit, -} from '@app/features/voice/utils/native_screen_capture_bridge/shared'; - -const logger = new Logger('InboundVideoBridge'); - -const DISPOSE_FAILURE_COUNT_MAX = 1_000_000; -const DISPOSE_FAILURE_LOG_INTERVAL = 100; - -export function i420VideoFrameLayout(width: number, height: number): Array<{offset: number; stride: number}> { - const chromaWidth = width >> 1; - const yPlaneBytes = width * height; - const uPlaneBytes = chromaWidth * (height >> 1); - return [ - {offset: 0, stride: width}, - {offset: yPlaneBytes, stride: chromaWidth}, - {offset: yPlaneBytes + uPlaneBytes, stride: chromaWidth}, - ]; -} - -export function tightI420ByteLength(width: number, height: number): number { - const chromaWidth = width >> 1; - const chromaHeight = height >> 1; - return width * height + 2 * chromaWidth * chromaHeight; -} - -export interface InboundVideoFrameInput { - width: number; - height: number; - timestampUs: number; - data: ArrayBuffer; -} - -export interface InboundVideoBridgeHandle { - track: MediaStreamTrack; - stream: MediaStream; - pushFrame: (frame: InboundVideoFrameInput) => void; - cleanup: () => Promise; -} - -class InboundVideoFrameRing { - private readonly frames: Array; - private head = 0; - private tail = 0; - private count = 0; - - constructor() { - assert.ok(MAX_GENERATOR_PENDING_VIDEO_FRAMES > 0, 'inbound video queue capacity must be positive'); - this.frames = new Array(MAX_GENERATOR_PENDING_VIDEO_FRAMES).fill(null); - } - - get length(): number { - return this.count; - } - - push(frame: RendererVideoFrame, closeFrame: (frame: RendererVideoFrame) => void): void { - assert.ok(frame); - assert.equal(typeof closeFrame, 'function'); - if (this.count >= MAX_GENERATOR_PENDING_VIDEO_FRAMES) { - const dropped = this.frames[this.head]; - this.frames[this.head] = null; - this.head = (this.head + 1) % MAX_GENERATOR_PENDING_VIDEO_FRAMES; - this.count -= 1; - if (dropped) closeFrame(dropped); - } - this.frames[this.tail] = frame; - this.tail = (this.tail + 1) % MAX_GENERATOR_PENDING_VIDEO_FRAMES; - this.count += 1; - assert.ok(this.count <= MAX_GENERATOR_PENDING_VIDEO_FRAMES, 'inbound video queue exceeded its cap'); - } - - pop(): RendererVideoFrame | null { - if (this.count === 0) return null; - const frame = this.frames[this.head]; - this.frames[this.head] = null; - this.head = (this.head + 1) % MAX_GENERATOR_PENDING_VIDEO_FRAMES; - this.count -= 1; - assert.ok(this.count >= 0, 'inbound video queue count underflowed'); - return frame; - } - - drain(closeFrame: (frame: RendererVideoFrame) => void): void { - assert.equal(typeof closeFrame, 'function'); - let frame = this.pop(); - while (frame) { - closeFrame(frame); - frame = this.pop(); - } - } -} - -function isValidInboundFrame(frame: InboundVideoFrameInput): boolean { - if (!Number.isSafeInteger(frame.width) || frame.width <= 0 || frame.width % 2 !== 0) return false; - if (!Number.isSafeInteger(frame.height) || frame.height <= 0 || frame.height % 2 !== 0) return false; - if (!(frame.data instanceof ArrayBuffer)) return false; - return frame.data.byteLength >= tightI420ByteLength(frame.width, frame.height); -} - -export function createInboundVideoBridge(diagnosticKey: string): InboundVideoBridgeHandle | null { - const Generator = getGeneratorVideoCtor(); - const VideoFrameImpl = getVideoFrameCtor(); - if (!Generator || !VideoFrameImpl) { - logger.warn('MediaStreamTrackGenerator path unavailable for inbound native video', {diagnosticKey}); - return null; - } - const generator = new Generator({kind: 'video'}); - const writer = generator.writable.getWriter(); - const stream = new MediaStream([generator]); - let cleanedUp = false; - let activeWrite: Promise | null = null; - const pendingFrames = new InboundVideoFrameRing(); - let disposeFailureCount = 0; - const noteDisposeFailure = (operation: string, error: unknown): void => { - disposeFailureCount = Math.min(disposeFailureCount + 1, DISPOSE_FAILURE_COUNT_MAX); - const shouldLog = disposeFailureCount === 1 || disposeFailureCount % DISPOSE_FAILURE_LOG_INTERVAL === 0; - if (shouldLog) { - logger.debug('Inbound video bridge disposal failed', { - diagnosticKey, - operation, - failureCount: disposeFailureCount, - error, - }); - } - }; - const closeFrame = (frame: RendererVideoFrame): void => { - try { - frame.close(); - } catch (error) { - noteDisposeFailure('frame.close', error); - } - }; - let cachedFrameInit: VideoFrameInit | null = null; - const getFrameInit = (width: number, height: number, timestamp: number): VideoFrameInit => { - if (!cachedFrameInit || cachedFrameInit.codedWidth !== width || cachedFrameInit.codedHeight !== height) { - cachedFrameInit = { - format: 'I420', - codedWidth: width, - codedHeight: height, - displayWidth: width, - displayHeight: height, - timestamp, - layout: i420VideoFrameLayout(width, height), - }; - } - cachedFrameInit.timestamp = timestamp; - return cachedFrameInit; - }; - const pumpPendingFrames = (): void => { - if (cleanedUp || activeWrite || pendingFrames.length === 0) return; - const videoFrame = pendingFrames.pop(); - if (!videoFrame) return; - try { - activeWrite = writer - .write(videoFrame) - .catch((error) => { - logger.warn('Failed to write inbound video frame to generator', {diagnosticKey, error}); - }) - .finally(() => { - closeFrame(videoFrame); - activeWrite = null; - pumpPendingFrames(); - }); - } catch (error) { - logger.warn('Failed to write inbound video frame to generator', {diagnosticKey, error}); - closeFrame(videoFrame); - activeWrite = null; - pumpPendingFrames(); - } - }; - const enqueueVideoFrame = (videoFrame: RendererVideoFrame): void => { - if (cleanedUp) { - closeFrame(videoFrame); - return; - } - pendingFrames.push(videoFrame, closeFrame); - pumpPendingFrames(); - }; - const buildVideoFrame = (frame: InboundVideoFrameInput): RendererVideoFrame | null => { - try { - const view = new Uint8Array(frame.data); - const timestamp = Math.max(0, Math.round(frame.timestampUs)); - return new VideoFrameImpl(view, getFrameInit(frame.width, frame.height, timestamp)); - } catch (error) { - logger.warn('Failed to construct inbound I420 VideoFrame', {diagnosticKey, error}); - return null; - } - }; - const pushFrame = (frame: InboundVideoFrameInput): void => { - if (cleanedUp) return; - if (!isValidInboundFrame(frame)) { - logger.debug('Dropping invalid inbound video frame', { - diagnosticKey, - width: frame.width, - height: frame.height, - byteLength: frame.data?.byteLength, - }); - return; - } - const videoFrame = buildVideoFrame(frame); - if (videoFrame) { - enqueueVideoFrame(videoFrame); - } - }; - const cleanup = async (): Promise => { - if (cleanedUp) return; - cleanedUp = true; - pendingFrames.drain(closeFrame); - if (activeWrite) { - await activeWrite; - } - try { - await writer.close(); - } catch (error) { - noteDisposeFailure('writer.close', error); - } - try { - generator.stop(); - } catch (error) { - noteDisposeFailure('generator.stop', error); - } - }; - return {track: generator, stream, pushFrame, cleanup}; -} diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/getVoiceEngine.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/getVoiceEngine.ts deleted file mode 100644 index c64d8118d..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/getVoiceEngine.ts +++ /dev/null @@ -1,45 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {NativeVoiceEngine} from '@app/features/voice/engine/native_voice_engine/NativeVoiceEngine'; -import { - getNativeVoiceEngineUpgradeBlockReason, - isNativeVoiceEngineSelected, - NativeVoiceEngineUpgradeRequiredError, - resetNativeVoiceEngineSelectionForTesting, -} from '@app/features/voice/engine/native_voice_engine/NativeVoiceEngineSelection'; - -export { - getNativeVoiceEngineUpgradeBlockReason, - isNativeVoiceEngineSelected, - isNativeVoiceEngineSelectionPending, - isNativeVoiceEngineUpgradeRequiredError, - NATIVE_VOICE_ENGINE_BRIDGE_VERSION, - shouldUseNativeVoiceEngine, -} from '@app/features/voice/engine/native_voice_engine/NativeVoiceEngineSelection'; - -let cachedEngine: NativeVoiceEngine | null = null; - -export function requireNativeVoiceEngine(): NativeVoiceEngine { - const upgradeBlockReason = getNativeVoiceEngineUpgradeBlockReason(); - if (upgradeBlockReason) { - throw new NativeVoiceEngineUpgradeRequiredError(upgradeBlockReason); - } - if (!isNativeVoiceEngineSelected()) { - throw new Error('Native voice engine is not selected'); - } - if (cachedEngine) { - return cachedEngine; - } - const bridge = window.electron?.voiceEngine; - if (!bridge) { - throw new Error('Native voice engine bridge is not available'); - } - const engine = new NativeVoiceEngine(bridge); - cachedEngine = engine; - return engine; -} - -export function resetVoiceEngineForTesting(): void { - cachedEngine = null; - resetNativeVoiceEngineSelectionForTesting(); -} diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/nativeVoiceEngineEventMapper.test.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/nativeVoiceEngineEventMapper.test.ts deleted file mode 100644 index f66a2107c..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/nativeVoiceEngineEventMapper.test.ts +++ /dev/null @@ -1,23 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {collectNativeVoiceEngineConnectedRosterPublishedTracks} from '@app/features/voice/engine/native_voice_engine/nativeVoiceEngineEventMapper'; -import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource'; -import {describe, expect, it} from 'vitest'; - -describe('collectNativeVoiceEngineConnectedRosterPublishedTracks', () => { - it('includes screen-share audio publications for subscription reattach', () => { - const tracks = collectNativeVoiceEngineConnectedRosterPublishedTracks({ - participants: [ - { - identity: 'user_1', - tracks: [{source: 'screen_share_audio'}, {source: 'screen_share'}, {source: 'microphone'}], - }, - ], - }); - - expect(tracks).toEqual([ - {identity: 'user_1', source: VoiceTrackSource.ScreenShareAudio}, - {identity: 'user_1', source: VoiceTrackSource.ScreenShare}, - ]); - }); -}); diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/nativeVoiceEngineEventMapper.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/nativeVoiceEngineEventMapper.ts deleted file mode 100644 index 7ebf3b1b3..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/nativeVoiceEngineEventMapper.ts +++ /dev/null @@ -1,680 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {Logger} from '@app/features/platform/utils/AppLogger'; -import { - asVoiceConnectionQuality, - asVoiceTrackSource, - type VoiceConnectionQuality, - type VoiceTrackSource, - VoiceTrackSource as VoiceTrackSourceValue, -} from '@app/features/voice/engine/VoiceTrackSource'; -import {ScreenShareWatchErrorCode, ScreenShareWatchFailures} from '@app/features/voice/state/ScreenShareWatchFailures'; -import type {VoiceEngineV2Stats} from '@fluxer/voice_engine_v2'; -import {coerceVoiceEngineV2BridgeStats, type VoiceEngineV2BridgeEvent} from '@fluxer/voice_engine_v2/bridge'; - -const logger = new Logger('NativeVoiceEngineV2BridgeEventMapper'); - -export interface NativeVoiceEngineParticipantFields { - identity: string; - sid: string; - name?: string; - isLocal?: boolean; - metadata?: string; - attributes?: Readonly>; - isMicrophoneEnabled?: boolean; - isCameraEnabled?: boolean; - isScreenShareEnabled?: boolean; - isScreenShareAudioEnabled?: boolean; -} - -export interface NativeVoiceEngineLocalTrackParticipant { - participantSid?: string; - participantIdentity?: string; -} - -export interface NativeVoiceEngineLocalTrackPublication { - trackSid?: string; - trackName?: string; -} - -interface NativeVoiceEngineParticipantTrackFlags { - isMicrophoneEnabled?: boolean; - isCameraEnabled?: boolean; - isScreenShareEnabled?: boolean; - isScreenShareAudioEnabled?: boolean; -} - -export interface NativeVoiceEngineV2BridgeEventManagers { - participants: { - upsertParticipantFromNative: (fields: NativeVoiceEngineParticipantFields) => void; - patchParticipantTrackFlags: (identity: string, flags: NativeVoiceEngineParticipantTrackFlags) => void; - setConnectionQualityForNative: (sid: string, quality: VoiceConnectionQuality) => void; - updateActiveSpeakersBySid: (sids: ReadonlyArray) => void; - applyNativeSpeakingSample: ( - sample: { - participantSid: string; - identity: string; - source: string; - isLocal: boolean; - speaking: boolean; - }, - nowMs?: number, - ) => void; - sweepNativeSpeakingHeartbeats: (nowMs?: number) => void; - removeParticipant: (identity: string) => void; - removeParticipantBySid: (sid: string) => void; - }; - inboundVideo: { - registerTrack: (participantSid: string, trackSid: string, source: string, participantIdentity?: string) => void; - unregisterTrack: (trackSid: string) => void; - unregisterParticipant: (participantSid: string) => void; - }; - localMedia: { - onLocalTrackPublished: ( - source: VoiceTrackSource, - trackSid?: string, - participant?: NativeVoiceEngineLocalTrackParticipant, - publication?: NativeVoiceEngineLocalTrackPublication, - ) => void; - onLocalTrackUnpublished?: ( - source: VoiceTrackSource, - trackSid?: string, - participant?: NativeVoiceEngineLocalTrackParticipant, - publication?: NativeVoiceEngineLocalTrackPublication, - ) => void; - onLocalTrackRepublished?: ( - source: VoiceTrackSource, - trackSid?: string, - participant?: NativeVoiceEngineLocalTrackParticipant, - publication?: NativeVoiceEngineLocalTrackPublication, - ) => void; - }; - e2ee: { - setState: (sid: string, raw: unknown) => void; - remove: (sid: string) => void; - }; - stats: { - setStats: (stats: VoiceEngineV2Stats, timestampMs?: number) => void; - }; -} - -export function ingestNativeVoiceEngineV2BridgeStats( - stats: VoiceEngineV2Stats, - managers: NativeVoiceEngineV2BridgeEventManagers, - timestampMs: number = Date.now(), -): void { - managers.stats.setStats(stats, timestampMs); - managers.participants.sweepNativeSpeakingHeartbeats(timestampMs); -} - -function asString(value: unknown): string | undefined { - return typeof value === 'string' && value.length > 0 ? value : undefined; -} - -function asBoolean(value: unknown): boolean | undefined { - return typeof value === 'boolean' ? value : undefined; -} - -function asStringArray(value: unknown): Array { - if (!Array.isArray(value)) return []; - return value.filter((entry): entry is string => typeof entry === 'string'); -} - -function asOptionalString(value: unknown): string | undefined { - return typeof value === 'string' ? value : undefined; -} - -function asStringRecord(value: unknown): Readonly> | undefined { - if (value == null || typeof value !== 'object' || Array.isArray(value)) return undefined; - const entries = Object.entries(value).filter((entry): entry is [string, string] => typeof entry[1] === 'string'); - return Object.freeze(Object.fromEntries(entries)); -} - -function asRecordArray(value: unknown): Array> { - if (!Array.isArray(value)) return []; - return value.filter((entry): entry is Record => entry != null && typeof entry === 'object'); -} - -export function parseConnectionQuality(value: unknown): VoiceConnectionQuality { - return asVoiceConnectionQuality(value); -} - -function trackFlagsForSource(source: string | undefined, present: boolean): NativeVoiceEngineParticipantTrackFlags { - switch (asVoiceTrackSource(source)) { - case VoiceTrackSourceValue.Microphone: - return {isMicrophoneEnabled: present}; - case VoiceTrackSourceValue.Camera: - return {isCameraEnabled: present}; - case VoiceTrackSourceValue.ScreenShare: - return {isScreenShareEnabled: present}; - case VoiceTrackSourceValue.ScreenShareAudio: - return {isScreenShareAudioEnabled: present}; - default: - return {}; - } -} - -function participantNameFromPayload(payload: Record): string | undefined { - return asString(payload.participantName) ?? asString(payload.name); -} - -function upsertParticipantFromPayload( - payload: Record, - managers: NativeVoiceEngineV2BridgeEventManagers, - sidKey: 'sid' | 'participantSid', -): string | undefined { - const identity = asString(payload.identity); - const sid = asString(payload[sidKey]); - if (!identity || !sid) return identity; - const name = participantNameFromPayload(payload); - const fields: NativeVoiceEngineParticipantFields = { - identity, - sid, - }; - if (name) fields.name = name; - const metadata = asOptionalString(payload.metadata); - if (metadata != null) fields.metadata = metadata; - const attributes = asStringRecord(payload.attributes); - if (attributes) fields.attributes = attributes; - managers.participants.upsertParticipantFromNative(fields); - return identity; -} - -function upsertLocalParticipantFromTrackPayload( - payload: Record, - managers: NativeVoiceEngineV2BridgeEventManagers, -): string | undefined { - const identity = asString(payload.identity); - const sid = asString(payload.participantSid); - if (!identity || !sid) return identity; - const name = participantNameFromPayload(payload); - managers.participants.upsertParticipantFromNative( - name ? {identity, sid, name, isLocal: true} : {identity, sid, isLocal: true}, - ); - return identity; -} - -function localTrackParticipantFromPayload( - payload: Record, -): NativeVoiceEngineLocalTrackParticipant | undefined { - const participantSid = asString(payload.participantSid); - const participantIdentity = asString(payload.identity); - if (!participantSid && !participantIdentity) return undefined; - return { - ...(participantSid ? {participantSid} : {}), - ...(participantIdentity ? {participantIdentity} : {}), - }; -} - -function localTrackPublicationFromPayload( - payload: Record, -): NativeVoiceEngineLocalTrackPublication | undefined { - const trackName = asString(payload.trackName); - if (!trackName) return undefined; - const trackSid = asString(payload.trackSid); - return { - ...(trackSid ? {trackSid} : {}), - trackName, - }; -} - -function emitLocalTrackPublished( - managers: NativeVoiceEngineV2BridgeEventManagers, - source: VoiceTrackSource, - trackSid: string | undefined, - participant: NativeVoiceEngineLocalTrackParticipant | undefined, - publication: NativeVoiceEngineLocalTrackPublication | undefined, -): void { - if (publication) { - managers.localMedia.onLocalTrackPublished(source, trackSid, participant, publication); - return; - } - if (participant) { - managers.localMedia.onLocalTrackPublished(source, trackSid, participant); - return; - } - managers.localMedia.onLocalTrackPublished(source, trackSid); -} - -function emitLocalTrackUnpublished( - managers: NativeVoiceEngineV2BridgeEventManagers, - source: VoiceTrackSource, - trackSid: string | undefined, - participant: NativeVoiceEngineLocalTrackParticipant | undefined, - publication: NativeVoiceEngineLocalTrackPublication | undefined, -): void { - if (publication) { - managers.localMedia.onLocalTrackUnpublished?.(source, trackSid, participant, publication); - return; - } - if (participant) { - managers.localMedia.onLocalTrackUnpublished?.(source, trackSid, participant); - return; - } - managers.localMedia.onLocalTrackUnpublished?.(source, trackSid); -} - -function emitLocalTrackRepublished( - managers: NativeVoiceEngineV2BridgeEventManagers, - source: VoiceTrackSource, - trackSid: string | undefined, - participant: NativeVoiceEngineLocalTrackParticipant | undefined, - publication: NativeVoiceEngineLocalTrackPublication | undefined, -): void { - if (publication) { - managers.localMedia.onLocalTrackRepublished?.(source, trackSid, participant, publication); - return; - } - if (participant) { - managers.localMedia.onLocalTrackRepublished?.(source, trackSid, participant); - return; - } - managers.localMedia.onLocalTrackRepublished?.(source, trackSid); -} - -function isPublishedTrackEnabled(payload: Record, present: boolean): boolean { - return present && asBoolean(payload.muted) !== true; -} - -function patchParticipantTrackFlagsFromPayload( - payload: Record, - managers: NativeVoiceEngineV2BridgeEventManagers, - present: boolean, - enabledOverride?: boolean, -): void { - const identity = upsertParticipantFromPayload(payload, managers, 'participantSid'); - if (!identity) return; - const source = asString(payload.source); - managers.participants.patchParticipantTrackFlags( - identity, - trackFlagsForSource(source, enabledOverride ?? isPublishedTrackEnabled(payload, present)), - ); -} - -function isCameraVideoTrackPayload(payload: Record): boolean { - if (asString(payload.kind) !== 'video') return false; - return asVoiceTrackSource(payload.source) === VoiceTrackSourceValue.Camera; -} - -function unregisterMutedCameraTrack( - payload: Record, - managers: NativeVoiceEngineV2BridgeEventManagers, -): void { - if (!isCameraVideoTrackPayload(payload)) return; - const trackSid = asString(payload.trackSid); - if (!trackSid) return; - managers.inboundVideo.unregisterTrack(trackSid); -} - -function registerUnmutedCameraTrack( - payload: Record, - managers: NativeVoiceEngineV2BridgeEventManagers, -): void { - if (!isCameraVideoTrackPayload(payload)) return; - const participantSid = asString(payload.participantSid); - const participantIdentity = asString(payload.identity); - const trackSid = asString(payload.trackSid); - if (!participantSid || !trackSid) { - logger.warn('trackUnmuted camera missing participantSid/trackSid', {payload}); - return; - } - managers.inboundVideo.registerTrack(participantSid, trackSid, VoiceTrackSourceValue.Camera, participantIdentity); -} - -function patchLocalParticipantTrackFlagsFromPayload( - payload: Record, - managers: NativeVoiceEngineV2BridgeEventManagers, - present: boolean, -): void { - const identity = upsertLocalParticipantFromTrackPayload(payload, managers); - if (!identity) return; - managers.participants.patchParticipantTrackFlags( - identity, - trackFlagsForSource(asString(payload.source), isPublishedTrackEnabled(payload, present)), - ); -} - -const CONNECTED_ROSTER_PARTICIPANTS_MAX = 1024; -const CONNECTED_ROSTER_TRACKS_PER_PARTICIPANT_MAX = 16; - -export function applyNativeVoiceEngineConnectedRoster( - payload: Record, - managers: NativeVoiceEngineV2BridgeEventManagers, -): void { - const participants = payload.participants; - if (!Array.isArray(participants)) return; - if (participants.length > CONNECTED_ROSTER_PARTICIPANTS_MAX) { - logger.warn('Native voice engine connected roster exceeds cap; extra participants ignored', { - count: participants.length, - cap: CONNECTED_ROSTER_PARTICIPANTS_MAX, - }); - } - for (const entry of participants.slice(0, CONNECTED_ROSTER_PARTICIPANTS_MAX)) { - if (typeof entry !== 'object' || entry === null) continue; - const record = entry as Record; - const identity = asString(record.identity); - const sid = asString(record.sid); - if (!identity || !sid) { - logger.warn('Native voice engine connected roster entry missing identity/sid', {entry: record}); - continue; - } - const name = asString(record.name); - managers.participants.upsertParticipantFromNative(name ? {identity, sid, name} : {identity, sid}); - const tracks = record.tracks; - if (!Array.isArray(tracks)) continue; - for (const track of tracks.slice(0, CONNECTED_ROSTER_TRACKS_PER_PARTICIPANT_MAX)) { - if (typeof track !== 'object' || track === null) continue; - patchParticipantTrackFlagsFromPayload(track as Record, managers, true); - } - } -} - -export interface NativeVoiceEngineConnectedRosterPublishedTrack { - identity: string; - source: VoiceTrackSource; -} - -export function collectNativeVoiceEngineConnectedRosterPublishedTracks( - payload: Record, -): Array { - const participants = payload.participants; - if (!Array.isArray(participants)) return []; - const tracks: Array = []; - const seen = new Set(); - for (const entry of participants.slice(0, CONNECTED_ROSTER_PARTICIPANTS_MAX)) { - if (typeof entry !== 'object' || entry === null) continue; - const record = entry as Record; - const identity = asString(record.identity); - if (!identity) continue; - const publications = record.tracks; - if (!Array.isArray(publications)) continue; - for (const publication of publications.slice(0, CONNECTED_ROSTER_TRACKS_PER_PARTICIPANT_MAX)) { - if (typeof publication !== 'object' || publication === null) continue; - const source = asVoiceTrackSource((publication as Record).source); - if ( - source !== VoiceTrackSourceValue.Camera && - source !== VoiceTrackSourceValue.ScreenShare && - source !== VoiceTrackSourceValue.ScreenShareAudio - ) { - continue; - } - const key = `${identity}:${source}`; - if (seen.has(key)) continue; - seen.add(key); - tracks.push({identity, source}); - } - } - return tracks; -} - -export function isFacadeOwnedConnectionEvent(type: string): boolean { - return ( - type === 'connected' || - type === 'Connected' || - type === 'disconnected' || - type === 'Disconnected' || - type === 'connectionState' || - type === 'Reconnecting' || - type === 'Reconnected' - ); -} - -export type NativeVoiceEngineConnectionEventAction = 'connected' | 'disconnected' | 'reconnecting' | 'reconnected'; - -export function getNativeVoiceEngineConnectionEventAction( - event: Pick, -): NativeVoiceEngineConnectionEventAction | null { - const payload = event.payload as Record; - switch (event.type) { - case 'connected': - case 'Connected': - return 'connected'; - case 'disconnected': - case 'Disconnected': - return 'disconnected'; - case 'Reconnecting': - return 'reconnecting'; - case 'Reconnected': - return 'reconnected'; - case 'connectionState': - switch (asString(payload.state)) { - case 'connected': - return 'connected'; - case 'disconnected': - return 'reconnecting'; - case 'reconnecting': - case 'signalReconnecting': - case 'signalreconnecting': - return 'reconnecting'; - default: - return null; - } - default: - return null; - } -} - -export function mapNativeVoiceEngineV2BridgeEvent( - event: VoiceEngineV2BridgeEvent, - managers: NativeVoiceEngineV2BridgeEventManagers, -): void { - const payload = event.payload as Record; - switch (event.type) { - case 'connectionState': - logger.debug('Native engine connection state', {state: payload.state}); - break; - case 'participantJoined': { - const identity = asString(payload.identity); - const sid = asString(payload.sid); - if (!identity || !sid) { - logger.warn('participantJoined missing identity/sid', {payload}); - break; - } - managers.participants.upsertParticipantFromNative({identity, sid, name: asString(payload.name)}); - break; - } - case 'participantNameChanged': { - const identity = asString(payload.identity); - const sid = asString(payload.sid); - const name = asString(payload.name); - if (!identity || !sid || !name) { - logger.warn('participantNameChanged missing identity/sid/name', {payload}); - break; - } - managers.participants.upsertParticipantFromNative({identity, sid, name}); - break; - } - case 'participantMetadataChanged': - case 'participantAttributesChanged': { - const identity = upsertParticipantFromPayload(payload, managers, 'sid'); - if (!identity) { - logger.warn(`${event.type} missing identity/sid`, {payload}); - } - break; - } - case 'participantLeft': { - const identity = asString(payload.identity); - const sid = asString(payload.sid); - if (identity) { - managers.participants.removeParticipant(identity); - } else if (sid) { - managers.participants.removeParticipantBySid(sid); - } - if (sid) { - managers.inboundVideo.unregisterParticipant(sid); - managers.e2ee.remove(sid); - } - break; - } - case 'trackPublished': { - patchParticipantTrackFlagsFromPayload(payload, managers, true); - break; - } - case 'trackUnpublished': { - patchParticipantTrackFlagsFromPayload(payload, managers, false); - const trackSid = asString(payload.trackSid); - if (trackSid) { - managers.inboundVideo.unregisterTrack(trackSid); - } - break; - } - case 'trackSubscribed': { - const participantSid = asString(payload.participantSid); - const participantIdentity = asString(payload.identity); - const trackSid = asString(payload.trackSid); - const kind = asString(payload.kind); - const source = asString(payload.source); - if (!participantSid || !trackSid) { - logger.warn('trackSubscribed missing participantSid/trackSid', {payload}); - break; - } - patchParticipantTrackFlagsFromPayload(payload, managers, true); - if (kind === 'video') { - managers.inboundVideo.registerTrack(participantSid, trackSid, asVoiceTrackSource(source), participantIdentity); - } - break; - } - case 'trackUnsubscribed': { - const participantSid = asString(payload.participantSid); - const trackSid = asString(payload.trackSid); - if (!trackSid) { - logger.warn('trackUnsubscribed missing trackSid', {payload}); - break; - } - managers.inboundVideo.unregisterTrack(trackSid); - if (!participantSid) { - logger.debug('trackUnsubscribed without participantSid; tile torn down by trackSid only', {trackSid}); - } - break; - } - case 'trackMuted': - case 'trackUnmuted': { - const unmuted = event.type === 'trackUnmuted'; - patchParticipantTrackFlagsFromPayload(payload, managers, unmuted, unmuted); - if (event.type === 'trackMuted') { - unregisterMutedCameraTrack(payload, managers); - } else { - registerUnmutedCameraTrack(payload, managers); - } - break; - } - case 'trackSubscriptionFailed': { - const trackSid = asString(payload.trackSid); - if (trackSid) { - managers.inboundVideo.unregisterTrack(trackSid); - } - const source = asVoiceTrackSource(payload.source); - if (source === VoiceTrackSourceValue.ScreenShare) { - ScreenShareWatchFailures.reportFailure({ - participantIdentity: asString(payload.identity) ?? asString(payload.participantIdentity), - participantSid: asString(payload.participantSid), - trackSid, - source, - code: ScreenShareWatchErrorCode.RemoteTrackSubscriptionFailed, - reason: 'remote-track-subscription-failed', - error: payload.error ?? payload.reason ?? payload.message, - }); - } - break; - } - case 'activeSpeakers': { - for (const participant of asRecordArray(payload.participants)) { - const identity = asString(participant.identity); - const sid = asString(participant.sid); - if (identity && sid) { - const name = asString(participant.name); - managers.participants.upsertParticipantFromNative(name ? {identity, sid, name} : {identity, sid}); - } - } - managers.participants.updateActiveSpeakersBySid(asStringArray(payload.sids)); - break; - } - case 'speakingChanged': { - const participantSid = asString(payload.participantSid); - const identity = asString(payload.identity); - const source = asString(payload.source); - const isLocal = asBoolean(payload.isLocal); - const speaking = asBoolean(payload.speaking); - if (!identity || isLocal === undefined || speaking === undefined) { - logger.warn('speakingChanged missing identity/isLocal/speaking', {payload}); - break; - } - if (source !== 'microphone') { - logger.warn('speakingChanged from non-microphone track ignored', {payload}); - break; - } - managers.participants.applyNativeSpeakingSample({ - participantSid: participantSid ?? '', - identity, - source, - isLocal, - speaking, - }); - break; - } - case 'connectionQuality': { - const sid = asString(payload.sid); - if (!sid) { - logger.warn('connectionQuality missing sid', {payload}); - break; - } - upsertParticipantFromPayload(payload, managers, 'sid'); - managers.participants.setConnectionQualityForNative(sid, parseConnectionQuality(payload.quality)); - break; - } - case 'e2eeState': { - const sid = asString(payload.sid); - if (!sid) { - logger.warn('e2eeState missing sid', {payload}); - break; - } - upsertParticipantFromPayload(payload, managers, 'sid'); - managers.e2ee.setState(sid, payload.state); - break; - } - case 'stats': { - const stats = coerceVoiceEngineV2BridgeStats(payload); - ingestNativeVoiceEngineV2BridgeStats(stats, managers); - break; - } - case 'audioPlaybackUnavailable': { - logger.warn('Native voice engine audio playback unavailable', {message: asString(payload.message)}); - break; - } - case 'localTrackPublished': { - const source = asVoiceTrackSource(payload.source); - const trackSid = asString(payload.trackSid); - const participant = localTrackParticipantFromPayload(payload); - const publication = localTrackPublicationFromPayload(payload); - patchLocalParticipantTrackFlagsFromPayload(payload, managers, true); - emitLocalTrackPublished(managers, source, trackSid, participant, publication); - break; - } - case 'localTrackUnpublished': { - const source = asVoiceTrackSource(payload.source); - const trackSid = asString(payload.trackSid); - const participant = localTrackParticipantFromPayload(payload); - const publication = localTrackPublicationFromPayload(payload); - patchLocalParticipantTrackFlagsFromPayload(payload, managers, false); - emitLocalTrackUnpublished(managers, source, trackSid, participant, publication); - break; - } - case 'localTrackRepublished': { - const source = asVoiceTrackSource(payload.source); - const trackSid = asString(payload.trackSid); - const participant = localTrackParticipantFromPayload(payload); - const publication = localTrackPublicationFromPayload(payload); - patchLocalParticipantTrackFlagsFromPayload(payload, managers, true); - emitLocalTrackRepublished(managers, source, trackSid, participant, publication); - emitLocalTrackPublished(managers, source, trackSid, participant, publication); - break; - } - case 'TokenRefreshed': - case 'ParticipantsUpdated': - case 'RoomUpdated': - break; - default: - logger.debug('Native voice engine event (unmapped)', {type: event.type}); - break; - } -} diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/nativeVoiceStatsMapper.test.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/nativeVoiceStatsMapper.test.ts deleted file mode 100644 index 886af3bde..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/nativeVoiceStatsMapper.test.ts +++ /dev/null @@ -1,263 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {mapNativeStatsToOverrides} from '@app/features/voice/engine/native_voice_engine/nativeVoiceStatsMapper'; -import type {VoiceEngineV2Stats} from '@fluxer/voice_engine_v2'; -import {coerceVoiceEngineV2BridgeStats} from '@fluxer/voice_engine_v2/bridge'; -import {describe, expect, it} from 'vitest'; - -function stats(overrides: Partial): VoiceEngineV2Stats { - return {rttMs: null, outbound: [], inbound: [], ...overrides}; -} - -describe('mapNativeStatsToOverrides', () => { - it('splits outbound video into camera vs screen share by source', () => { - const result = mapNativeStatsToOverrides( - stats({ - outbound: [ - {trackSid: 'cam', source: 'camera', kind: 'video', bitrateKbps: 500, packetsLost: 0, fps: 30}, - {trackSid: 'ss', source: 'screen_share', kind: 'video', bitrateKbps: 3000, packetsLost: 1, fps: 60}, - ], - }), - ); - expect(result.localVideo?.trackIdentifier).toBe('cam'); - expect(result.localVideo?.framesPerSecond).toBe(30); - expect(result.localScreenShare?.trackIdentifier).toBe('ss'); - expect(result.localScreenShare?.framesPerSecond).toBe(60); - expect(result.network.videoSendBitrateKbps).toBe(3500); - }); - - it('treats screenshare (no underscore) as a screen-share source too', () => { - const result = mapNativeStatsToOverrides( - stats({ - outbound: [{trackSid: 'ss', source: 'screenshare', kind: 'video', bitrateKbps: 1000, packetsLost: 0}], - }), - ); - expect(result.localScreenShare?.trackIdentifier).toBe('ss'); - expect(result.localVideo).toBeNull(); - }); - - it('preserves native screen-share codec, dimensions, fps, and frame diagnostics from raw stats payloads', () => { - const result = mapNativeStatsToOverrides( - coerceVoiceEngineV2BridgeStats({ - rttMs: 1, - outbound: [ - { - trackSid: 'TR_screen', - source: 'screenshare', - kind: 'video', - codec: 'video/H265', - bitrateKbps: 2500, - packetsLost: 0, - fps: 14, - width: 1280, - height: 720, - sourceWidth: 2176, - sourceHeight: 1200, - targetBitrateKbps: 50000, - configuredFps: 60, - targetFps: 60, - effectiveFps: 58.7, - framesProduced: 120, - framesAccepted: 119, - framesDropped: 1, - framesCoalesced: 0, - framesCaptured: 118, - captureFailures: 0, - maxQueueAgeMs: 12.5, - maxPushLatencyMs: 8.25, - adaptiveSendTier: 'full', - adaptiveSendReason: 'disabled', - }, - ], - }), - ); - expect(result.localScreenShare).toMatchObject({ - trackIdentifier: 'TR_screen', - codec: 'video/H265', - framesPerSecond: 58.7, - configuredFramesPerSecond: 60, - targetFramesPerSecond: 60, - effectiveFramesPerSecond: 58.7, - frameWidth: 1280, - frameHeight: 720, - sourceFrameWidth: 2176, - sourceFrameHeight: 1200, - targetBitrateKbps: 50000, - framesProduced: 120, - framesAccepted: 119, - framesDropped: 1, - framesCaptured: 118, - maxQueueAgeMs: 12.5, - maxPushLatencyMs: 8.25, - adaptiveSendTier: 'full', - adaptiveSendReason: 'disabled', - }); - }); - - it('coalesces native outbound rows for one screen-share publication before selecting the local screen-share row', () => { - const result = mapNativeStatsToOverrides( - stats({ - outbound: [ - { - trackSid: 'TR_screen', - source: 'screen_share', - kind: 'video', - codec: 'video/H265', - bitrateKbps: 269.9, - packetsLost: 0, - fps: 3, - effectiveFps: 53.9, - configuredFps: 60, - targetFps: 60, - width: 2176, - height: 1200, - sourceWidth: 2176, - sourceHeight: 1200, - targetBitrateKbps: 50_000, - framesProduced: 1904, - framesAccepted: 1904, - framesDropped: 64, - framesCoalesced: 64, - framesCaptured: 1839, - captureFailures: 0, - maxQueueAgeMs: 18, - maxPushLatencyMs: 18, - adaptiveSendTier: 'full', - adaptiveSendReason: 'adaptiveDisabled', - }, - { - trackSid: 'TR_screen', - source: 'screenshare', - kind: 'video', - codec: 'video/H265', - bitrateKbps: 4614.1, - packetsLost: 0, - fps: 30, - }, - ], - }), - ); - expect(result.network.videoSendBitrateKbps).toBe(4884); - expect(result.localScreenShare).toMatchObject({ - trackIdentifier: 'TR_screen', - codec: 'video/H265', - bitrateKbps: 4884, - framesPerSecond: 53.9, - effectiveFramesPerSecond: 53.9, - configuredFramesPerSecond: 60, - targetFramesPerSecond: 60, - frameWidth: 2176, - frameHeight: 1200, - targetBitrateKbps: 50_000, - adaptiveSendTier: 'full', - adaptiveSendReason: 'adaptiveDisabled', - }); - expect(result.perTrackStats).toContainEqual(expect.objectContaining({trackIdentifier: 'TR_screen'})); - }); - - it('splits native screen-share audio from microphone audio', () => { - const result = mapNativeStatsToOverrides( - stats({ - outbound: [ - {trackSid: 'mic', source: 'microphone', kind: 'audio', bitrateKbps: 48, packetsLost: 0}, - {trackSid: 'ss-audio', source: 'screen_share_audio', kind: 'audio', bitrateKbps: 96, packetsLost: 0}, - ], - }), - ); - expect(result.localAudio?.trackIdentifier).toBe('mic'); - expect(result.localScreenShareAudio?.trackIdentifier).toBe('ss-audio'); - expect(result.network.audioSendBitrateKbps).toBe(144); - }); - - it('maps inbound audio + video and rolls up recv bitrate, jitter, rtt', () => { - const result = mapNativeStatsToOverrides( - stats({ - rttMs: 25, - droppedVideoFrameCallbacks: 3, - inbound: [ - {participantSid: 'PA_1', trackSid: 'a', kind: 'audio', bitrateKbps: 32, packetsLost: 2, jitterMs: 8}, - {participantSid: 'PA_1', trackSid: 'v', kind: 'video', bitrateKbps: 900, packetsLost: 0}, - ], - }), - ); - expect(result.network.rttMs).toBe(25); - expect(result.network.audioRecvBitrateKbps).toBe(32); - expect(result.network.videoRecvBitrateKbps).toBe(900); - expect(result.network.jitterMs).toBe(8); - expect(result.network.droppedVideoFrameCallbacks).toBe(3); - expect(result.remoteAudio?.trackIdentifier).toBe('a'); - expect(result.remoteVideo?.trackIdentifier).toBe('v'); - expect(result.remoteScreenShare).toBeNull(); - }); - - it('maps native inbound screen-share source, dimensions, and fps into remote screen-share stats', () => { - const result = mapNativeStatsToOverrides( - coerceVoiceEngineV2BridgeStats({ - rttMs: 20, - inbound: [ - { - participantSid: 'PA_1', - participantIdentity: 'user_2_connection_2', - trackSid: 'TR_remote_screen', - source: 'screen_share', - kind: 'video', - codec: 'video/H265', - bitrateKbps: 4200, - packetsLost: 0, - width: 3840, - height: 2160, - sourceWidth: 3840, - sourceHeight: 2160, - fps: 24.7, - }, - ], - }), - ); - - expect(result.remoteVideo).toBeNull(); - expect(result.remoteScreenShare).toMatchObject({ - trackIdentifier: 'TR_remote_screen', - codec: 'video/H265', - bitrateKbps: 4200, - framesPerSecond: 24.7, - frameWidth: 3840, - frameHeight: 2160, - sourceFrameWidth: 3840, - sourceFrameHeight: 2160, - }); - }); - - it('reports worst-case packet loss across the relevant direction', () => { - const result = mapNativeStatsToOverrides( - stats({ - outbound: [ - {trackSid: 'a', source: 'microphone', kind: 'audio', bitrateKbps: 30, packetsLost: 5, packetsSent: 100}, - ], - inbound: [ - {participantSid: 'PA_1', trackSid: 'b', kind: 'audio', bitrateKbps: 30, packetsLost: 12, packetsReceived: 88}, - ], - }), - ); - expect(result.network.audioPacketLossPercent).toBe(12); - }); - - it('handles empty stats without throwing and yields null tracks / zero network', () => { - const result = mapNativeStatsToOverrides(stats({rttMs: null})); - expect(result.localAudio).toBeNull(); - expect(result.remoteVideo).toBeNull(); - expect(result.network.rttMs).toBeNull(); - expect(result.network.videoSendBitrateKbps).toBe(0); - }); - - it('prefers the first track with non-zero bitrate when several exist', () => { - const result = mapNativeStatsToOverrides( - stats({ - outbound: [ - {trackSid: 'silent', source: 'microphone', kind: 'audio', bitrateKbps: 0, packetsLost: 0}, - {trackSid: 'live', source: 'microphone', kind: 'audio', bitrateKbps: 40, packetsLost: 0}, - ], - }), - ); - expect(result.localAudio?.trackIdentifier).toBe('live'); - }); -}); diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/nativeVoiceStatsMapper.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/nativeVoiceStatsMapper.ts deleted file mode 100644 index 1bed245d2..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/nativeVoiceStatsMapper.ts +++ /dev/null @@ -1,111 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import { - summarizeVoiceEngineV2Stats, - type VoiceEngineV2PerTrackStats, - type VoiceEngineV2Stats, - type VoiceEngineV2StatsSummary, - type VoiceEngineV2StatsTrackSummary, -} from '@fluxer/voice_engine_v2'; - -export interface NativeStatsForNerdsOverrides { - network: { - audioSendBitrateKbps: number; - audioRecvBitrateKbps: number; - videoSendBitrateKbps: number; - videoRecvBitrateKbps: number; - audioPacketLossPercent: number; - videoPacketLossPercent: number; - jitterMs: number; - rttMs: number | null; - droppedVideoFrameCallbacks?: number; - }; - localVideo: VoiceEngineV2PerTrackStats | null; - localAudio: VoiceEngineV2PerTrackStats | null; - localScreenShare: VoiceEngineV2PerTrackStats | null; - localScreenShareAudio: VoiceEngineV2PerTrackStats | null; - remoteVideo: VoiceEngineV2PerTrackStats | null; - remoteScreenShare: VoiceEngineV2PerTrackStats | null; - remoteAudio: VoiceEngineV2PerTrackStats | null; - remoteScreenShareAudio: VoiceEngineV2PerTrackStats | null; - perTrackStats: Array; -} - -function trackSummaryToPerTrack(track: VoiceEngineV2StatsTrackSummary | null): VoiceEngineV2PerTrackStats | null { - if (!track) return null; - return { - direction: track.direction === 'recv' ? 'recv' : 'send', - kind: track.kind === 'audio' || track.kind === 'video' ? track.kind : 'unknown', - trackIdentifier: track.trackIdentifier, - codec: track.codec, - bitrateKbps: track.bitrateKbps, - packetsLost: track.packetsLost, - jitterMs: track.jitterMs, - framesPerSecond: track.framesPerSecond, - sourceFramesPerSecond: track.sourceFramesPerSecond, - configuredFramesPerSecond: track.configuredFramesPerSecond, - targetFramesPerSecond: track.targetFramesPerSecond, - effectiveFramesPerSecond: track.effectiveFramesPerSecond, - frameWidth: track.frameWidth, - frameHeight: track.frameHeight, - sourceFrameWidth: track.sourceFrameWidth, - sourceFrameHeight: track.sourceFrameHeight, - targetBitrateKbps: track.targetBitrateKbps, - framesProduced: track.framesProduced, - framesAccepted: track.framesAccepted, - framesDropped: track.framesDropped, - framesCoalesced: track.framesCoalesced, - framesCaptured: track.framesCaptured, - captureFailures: track.captureFailures, - maxQueueAgeMs: track.maxQueueAgeMs, - maxPushLatencyMs: track.maxPushLatencyMs, - adaptiveSendTier: track.adaptiveSendTier, - adaptiveSendReason: track.adaptiveSendReason, - }; -} - -function summaryToOverrides( - summary: VoiceEngineV2StatsSummary, - stats: VoiceEngineV2Stats, -): NativeStatsForNerdsOverrides { - const localAudio = trackSummaryToPerTrack(summary.localAudio); - const localVideo = trackSummaryToPerTrack(summary.localVideo); - const localScreenShare = trackSummaryToPerTrack(summary.localScreenShare); - const localScreenShareAudio = trackSummaryToPerTrack(summary.localScreenShareAudio); - const remoteAudio = trackSummaryToPerTrack(summary.remoteAudio); - const remoteVideo = trackSummaryToPerTrack(summary.remoteVideo); - const remoteScreenShare = trackSummaryToPerTrack(summary.remoteScreenShare); - const remoteScreenShareAudio = trackSummaryToPerTrack(summary.remoteScreenShareAudio); - return { - network: { - ...summary.network, - droppedVideoFrameCallbacks: stats.droppedVideoFrameCallbacks ?? stats.droppedNativeVideoFrames, - }, - localAudio, - localVideo, - localScreenShare, - localScreenShareAudio, - remoteAudio, - remoteVideo, - remoteScreenShare, - remoteScreenShareAudio, - perTrackStats: [ - localAudio, - localVideo, - localScreenShare, - localScreenShareAudio, - remoteAudio, - remoteVideo, - remoteScreenShare, - remoteScreenShareAudio, - ].filter((track): track is VoiceEngineV2PerTrackStats => track !== null), - }; -} - -export function mapNativeStatsToOverrides(stats: VoiceEngineV2Stats): NativeStatsForNerdsOverrides { - return summaryToOverrides(summarizeVoiceEngineV2Stats(stats), stats); -} - -export function mapNativeStatsToPerTrackStats(stats: VoiceEngineV2Stats): Array { - return mapNativeStatsToOverrides(stats).perTrackStats; -} diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/nativeVoiceVolume.test.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/nativeVoiceVolume.test.ts deleted file mode 100644 index 9ee155d06..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/nativeVoiceVolume.test.ts +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {computeNativeParticipantVolume} from '@app/features/voice/engine/native_voice_engine/nativeVoiceVolume'; -import {computeVoiceEngineV2NativeParticipantVolume} from '@fluxer/voice_engine_v2'; -import {describe, expect, it} from 'vitest'; - -describe('computeNativeParticipantVolume', () => { - it('is a compatibility export for the v2 native participant volume policy', () => { - expect(computeNativeParticipantVolume).toBe(computeVoiceEngineV2NativeParticipantVolume); - }); -}); diff --git a/fluxer_app/src/features/voice/engine/native_voice_engine/nativeVoiceVolume.ts b/fluxer_app/src/features/voice/engine/native_voice_engine/nativeVoiceVolume.ts deleted file mode 100644 index ff403ec0b..000000000 --- a/fluxer_app/src/features/voice/engine/native_voice_engine/nativeVoiceVolume.ts +++ /dev/null @@ -1,6 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -export { - computeVoiceEngineV2NativeParticipantVolume as computeNativeParticipantVolume, - type VoiceEngineV2NativeParticipantVolumeInput as NativeParticipantVolumeInput, -} from '@fluxer/voice_engine_v2'; diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.test.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.test.ts new file mode 100644 index 000000000..f9166d3d9 --- /dev/null +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.test.ts @@ -0,0 +1,79 @@ +// @vitest-environment happy-dom +// SPDX-License-Identifier: AGPL-3.0-or-later + +import { + createVoiceConnectionSnapshot, + transitionVoiceConnectionSnapshot, + type VoiceConnectionSnapshot, +} from '@app/features/voice/engine/VoiceConnectionStateMachine'; +import { + getRoomFromMediaEngine, + getVoiceConnectionContextFromMediaEngine, +} from '@app/features/voice/engine/VoiceMediaEngineBridge'; +import {afterEach, describe, expect, it} from 'vitest'; + +type MediaEngineWindow = typeof globalThis & {_mediaEngine?: unknown}; + +function connectedWithoutRoomSnapshot(): VoiceConnectionSnapshot { + const started = transitionVoiceConnectionSnapshot(createVoiceConnectionSnapshot(), { + type: 'connection.start', + guildId: 'guild-1', + channelId: 'channel-1', + }); + return transitionVoiceConnectionSnapshot(started, {type: 'connection.connected'}); +} + +function seedMediaEngineFromSnapshot(snapshot: VoiceConnectionSnapshot, overrides: Record = {}): void { + const {context} = snapshot; + (window as MediaEngineWindow)._mediaEngine = { + guildId: context.guildId, + channelId: context.channelId, + connectionId: context.connectionId, + connected: context.connected, + connecting: context.connecting, + reconnecting: context.reconnecting, + room: context.room ?? undefined, + ...overrides, + }; +} + +describe('voice connection projection while connected without a room', () => { + afterEach(() => { + delete (window as MediaEngineWindow)._mediaEngine; + }); + + it('models a connected connection whose LiveKit room is still null', () => { + const {context} = connectedWithoutRoomSnapshot(); + + expect(context.connected).toBe(true); + expect(context.room).toBeNull(); + }); + + it('projects the LiveKit room when the store reports connected without a room', () => { + seedMediaEngineFromSnapshot(connectedWithoutRoomSnapshot()); + + expect(getVoiceConnectionContextFromMediaEngine()?.connected).toBe(true); + expect(getRoomFromMediaEngine()).toBeNull(); + }); + + it('projects the LiveKit room when only the v2 model reports connected without a room', () => { + seedMediaEngineFromSnapshot(connectedWithoutRoomSnapshot(), { + connected: undefined, + connecting: undefined, + reconnecting: undefined, + room: undefined, + voiceEngineV2Model: { + connection: { + connected: true, + connecting: false, + reconnecting: false, + status: 'connected', + gateway: {selfVoiceState: {guildId: 'guild-1', channelId: 'channel-1'}}, + }, + }, + }); + + expect(getVoiceConnectionContextFromMediaEngine()?.connected).toBe(true); + expect(getRoomFromMediaEngine()).toBeNull(); + }); +}); diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.ts index 069b3aa64..dd8235474 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.ts @@ -494,43 +494,6 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store { ); } - acceptNativeVoiceServerUpdate(raw: VoiceServerUpdateData): boolean { - assertVoiceServerUpdateShape(raw, 'acceptNativeVoiceServerUpdate.raw'); - const decision = selectVoiceConnectionServerUpdateDecision(this.connectionSnapshot, raw); - if (decision.type === 'ignore') { - logger.warn('Native voice engine ignoring VOICE_SERVER_UPDATE', { - reason: decision.reason, - expectedGuildId: decision.expectedGuildId, - incomingGuildId: decision.incomingGuildId, - expectedChannelId: decision.expectedChannelId, - incomingChannelId: decision.incomingChannelId, - attemptId: decision.attemptId, - }); - return false; - } - this.clearVoiceServerTimeout(); - this.update(() => { - this.transitionConnection({ - type: 'voiceServer.accepted', - guildId: decision.guildId, - channelId: decision.resolvedChannelId, - endpoint: decision.endpoint, - connectionId: decision.connectionId, - isChannelMove: decision.isChannelMove, - }); - }); - this.throttle.setInFlightConnect(true); - logger.info('Native voice engine accepted VOICE_SERVER_UPDATE', { - guildId: decision.guildId, - channelId: decision.resolvedChannelId, - endpoint: decision.endpoint, - connectionId: decision.connectionId, - isChannelMove: decision.isChannelMove, - isRegionChange: decision.isRegionChange, - }); - return true; - } - private async handleVoiceServerUpdateAsync( raw: VoiceServerUpdateData, onRoomCreated: (room: Room, attemptId: number, guildId: string | null, channelId: string) => void, diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppDebugLoggingHostAdapter.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppDebugLoggingHostAdapter.ts index 61288406b..c527a874a 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppDebugLoggingHostAdapter.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppDebugLoggingHostAdapter.ts @@ -18,7 +18,6 @@ import {asVoiceTrackSource, VoiceTrackSource} from '@app/features/voice/engine/V import {getNativeAudioCaptureDiagnosticState} from '@app/features/voice/utils/NativeAudioCaptureBridge'; import type {DesktopVoiceDebugEventSinkEntry, ElectronAPI} from '@app/types/electron.d'; import type {VoiceDebugLoggingStatusResponse} from '@fluxer/schema/src/domains/channel/ChannelSchemas'; -import type {VoiceEngineV2BridgeVideoFrame} from '@fluxer/voice_engine_v2/bridge'; import type { LocalTrackPublication, Participant, @@ -43,20 +42,11 @@ const MAX_QUEUED_EVENTS = 1000; const MAX_UPLOAD_BATCH_EVENTS = 200; export const VOICE_ENGINE_V2_APP_DEBUG_EVENT_SINK_MAX_ENTRIES = 1000; export const VOICE_ENGINE_V2_APP_DEBUG_EVENT_SINK_MAX_LINE_CHARS = 262_144; -const MAX_NATIVE_VIDEO_FIRST_FRAME_EVENTS = 64; const SNAPSHOT_INTERVAL_MS = 10000; const MAX_DEVICE_SNAPSHOT_AUDIO_TARGETS = 200; const MAX_DEVICE_SNAPSHOT_PIPEWIRE_GRAPH_RECORDS = 500; const SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC = 'fluxer.rtc.codec-negotiation.v1'; const TEXT_DECODER = new TextDecoder(); -export const VOICE_DEBUG_EXCLUDED_NATIVE_ENGINE_EVENT_TYPES: ReadonlySet = new Set([ - 'activeSpeakers', - 'audioLevel', - 'audioLevels', - 'speakingChanged', - 'stats', -]); - class BoundedRing { private readonly items: Array; private readonly capacity: number; @@ -344,27 +334,6 @@ function summarizeRemoteTrack(track: RemoteTrack): Record { }; } -function summarizeNativeVideoFrame(frame: VoiceEngineV2BridgeVideoFrame): Record { - assertNonNullObject(frame, 'native video frame'); - assertNonNullObject(frame.meta, 'native video frame metadata'); - assertString(frame.meta.participantSid, 'native video frame participantSid'); - assertString(frame.meta.trackSid, 'native video frame trackSid'); - assertFiniteNumber(frame.meta.width, 'native video frame width'); - assertFiniteNumber(frame.meta.height, 'native video frame height'); - assertFiniteNumber(frame.meta.timestampUs, 'native video frame timestampUs'); - return { - participantSid: frame.meta.participantSid, - participantIdentity: frame.meta.participantIdentity, - trackSid: frame.meta.trackSid, - trackName: frame.meta.trackName, - source: frame.meta.source, - width: frame.meta.width, - height: frame.meta.height, - timestampUs: frame.meta.timestampUs, - byteLength: frame.data.byteLength, - }; -} - function getRtpCapabilities(kind: 'audio' | 'video'): Record { const sender = typeof RTCRtpSender !== 'undefined' && typeof RTCRtpSender.getCapabilities === 'function' @@ -635,8 +604,6 @@ export class VoiceEngineV2AppDebugLoggingHostAdapter extends Store { private eventSinkSequence = 0; private eventSinkForwardFailureCount = 0; private lastSnapshotAtMs = 0; - private readonly nativeVideoFirstFrameTrackSids = new Set(); - private nativeVideoFirstFrameOverflowRecorded = false; constructor(options: VoiceEngineV2AppDebugLoggingHostAdapterOptions = {}) { super(); @@ -711,8 +678,6 @@ export class VoiceEngineV2AppDebugLoggingHostAdapter extends Store { this.participantIdentity = options.room?.localParticipant?.identity ?? null; this.collectSnapshot = options.collectSnapshot; }); - this.nativeVideoFirstFrameTrackSids.clear(); - this.nativeVideoFirstFrameOverflowRecorded = false; if (options.room) { this.bindRoom(options.room); } @@ -746,8 +711,6 @@ export class VoiceEngineV2AppDebugLoggingHostAdapter extends Store { this.roomDisposer?.(); this.roomDisposer = null; this.queue.clear(); - this.nativeVideoFirstFrameTrackSids.clear(); - this.nativeVideoFirstFrameOverflowRecorded = false; this.update(() => { this.active = false; this.sessionId = null; @@ -790,43 +753,6 @@ export class VoiceEngineV2AppDebugLoggingHostAdapter extends Store { } } - recordNativeEngineEvent(event: {type: string; payload?: unknown}): void { - assertNonNullObject(event, 'event'); - assertString(event.type, 'event.type'); - if (VOICE_DEBUG_EXCLUDED_NATIVE_ENGINE_EVENT_TYPES.has(event.type)) return; - if (this.channelId === null) return; - this.record(`native.engine.${event.type}`, {payload: event.payload ?? null}); - } - - recordNativeVideoDiagnostic(type: string, data?: Record): void { - assertString(type, 'native video diagnostic type'); - if (this.channelId === null) return; - this.record(`native.video.${type}`, data); - } - - recordNativeVideoFrame(frame: VoiceEngineV2BridgeVideoFrame): void { - if (this.channelId === null) return; - const summary = summarizeNativeVideoFrame(frame); - const trackSid = frame.meta.trackSid; - if (this.nativeVideoFirstFrameTrackSids.has(trackSid)) return; - if (this.nativeVideoFirstFrameTrackSids.size >= MAX_NATIVE_VIDEO_FIRST_FRAME_EVENTS) { - this.recordNativeVideoFirstFrameOverflow(trackSid); - return; - } - this.nativeVideoFirstFrameTrackSids.add(trackSid); - this.recordNativeVideoDiagnostic('frame.first', summary); - } - - private recordNativeVideoFirstFrameOverflow(trackSid: string): void { - assertString(trackSid, 'native video overflow trackSid'); - if (this.nativeVideoFirstFrameOverflowRecorded) return; - this.nativeVideoFirstFrameOverflowRecorded = true; - this.recordNativeVideoDiagnostic('frame.first_overflow', { - trackSid, - cap: MAX_NATIVE_VIDEO_FIRST_FRAME_EVENTS, - }); - } - private buildRoomEventBindings(room: Room): Array { return [ [RoomEvent.Connected, () => this.record('livekit.room.connected', {room: summarizeRoom(room)})], diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeBridge.test.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeBridge.test.ts deleted file mode 100644 index 85f8a8a09..000000000 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeBridge.test.ts +++ /dev/null @@ -1,61 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {beforeEach, describe, expect, it, vi} from 'vitest'; - -const nativeVoiceEngineSelected = vi.fn<() => boolean>(); -const electronApi = vi.fn<() => unknown>(); - -vi.mock('@app/features/voice/engine/native_voice_engine/NativeVoiceEngineSelection', () => ({ - isNativeVoiceEngineSelected: () => nativeVoiceEngineSelected(), -})); - -vi.mock('@app/features/ui/utils/NativeUtils', () => ({ - getElectronAPI: () => electronApi(), -})); - -const { - isVoiceEngineV2AppNativeAudioDeviceBridgeAvailable, - isVoiceEngineV2AppNativeScreenShareAudioBridgeAvailable, - isVoiceEngineV2AppNativeScreenShareBridgeAvailable, - isVoiceEngineV2AppNativeScreenShareEncodingUpdateAvailable, -} = await import('./VoiceEngineV2AppNativeBridge'); - -const FULLY_CAPABLE_BRIDGE = { - getCapabilities: async () => ({ - screenShare: true, - screenShareAudio: true, - screenShareEncodingUpdate: true, - }), - listAudioInputDevices: async () => [], - listAudioOutputDevices: async () => [], -}; - -describe('VoiceEngineV2AppNativeBridge availability', () => { - beforeEach(() => { - electronApi.mockReturnValue({voiceEngine: FULLY_CAPABLE_BRIDGE}); - }); - - it('reports native capabilities when the native voice engine is selected', async () => { - nativeVoiceEngineSelected.mockReturnValue(true); - expect(await isVoiceEngineV2AppNativeScreenShareBridgeAvailable()).toBe(true); - expect(await isVoiceEngineV2AppNativeScreenShareEncodingUpdateAvailable()).toBe(true); - expect(await isVoiceEngineV2AppNativeScreenShareAudioBridgeAvailable()).toBe(true); - expect(isVoiceEngineV2AppNativeAudioDeviceBridgeAvailable()).toBe(true); - }); - - it('reports nothing available when the native voice engine is not selected, even on a fully capable bridge', async () => { - nativeVoiceEngineSelected.mockReturnValue(false); - expect(await isVoiceEngineV2AppNativeScreenShareBridgeAvailable()).toBe(false); - expect(await isVoiceEngineV2AppNativeScreenShareEncodingUpdateAvailable()).toBe(false); - expect(await isVoiceEngineV2AppNativeScreenShareAudioBridgeAvailable()).toBe(false); - expect(isVoiceEngineV2AppNativeAudioDeviceBridgeAvailable()).toBe(false); - }); - - it('does not query the bridge at all when the native voice engine is not selected', async () => { - nativeVoiceEngineSelected.mockReturnValue(false); - const getCapabilities = vi.fn(async () => ({screenShare: true})); - electronApi.mockReturnValue({voiceEngine: {...FULLY_CAPABLE_BRIDGE, getCapabilities}}); - await isVoiceEngineV2AppNativeScreenShareBridgeAvailable(); - expect(getCapabilities).not.toHaveBeenCalled(); - }); -}); diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeBridge.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeBridge.ts deleted file mode 100644 index 63a7bb401..000000000 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeBridge.ts +++ /dev/null @@ -1,52 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {getElectronAPI} from '@app/features/ui/utils/NativeUtils'; -import {isNativeVoiceEngineSelected} from '@app/features/voice/engine/native_voice_engine/NativeVoiceEngineSelection'; -import type {VoiceEngineV2BridgeApi, VoiceEngineV2BridgeCapabilities} from '@fluxer/voice_engine_v2/bridge'; - -function buildMissingBridgeError(context: string): Error { - const error = new Error(`${context}: voice engine v2 native bridge is unavailable`); - error.name = 'VoiceEngineV2AppNativeBridgeUnavailableError'; - return error; -} - -export function getVoiceEngineV2AppNativeBridge(): VoiceEngineV2BridgeApi | null { - return getElectronAPI()?.voiceEngine ?? null; -} - -export function requireVoiceEngineV2AppNativeBridge(context: string): VoiceEngineV2BridgeApi { - const bridge = getVoiceEngineV2AppNativeBridge(); - if (!bridge) throw buildMissingBridgeError(context); - return bridge; -} - -export async function getVoiceEngineV2AppNativeBridgeCapabilities(): Promise { - const bridge = getVoiceEngineV2AppNativeBridge(); - if (!bridge) return null; - return bridge.getCapabilities(); -} - -export async function isVoiceEngineV2AppNativeScreenShareBridgeAvailable(): Promise { - if (!isNativeVoiceEngineSelected()) return false; - const capabilities = await getVoiceEngineV2AppNativeBridgeCapabilities(); - return capabilities?.screenShare === true; -} - -export async function isVoiceEngineV2AppNativeScreenShareEncodingUpdateAvailable(): Promise { - if (!isNativeVoiceEngineSelected()) return false; - const capabilities = await getVoiceEngineV2AppNativeBridgeCapabilities(); - return capabilities?.screenShareEncodingUpdate === true; -} - -export async function isVoiceEngineV2AppNativeScreenShareAudioBridgeAvailable(): Promise { - if (!isNativeVoiceEngineSelected()) return false; - const capabilities = await getVoiceEngineV2AppNativeBridgeCapabilities(); - return capabilities?.screenShareAudio === true; -} - -export function isVoiceEngineV2AppNativeAudioDeviceBridgeAvailable(): boolean { - if (!isNativeVoiceEngineSelected()) return false; - const bridge = getVoiceEngineV2AppNativeBridge(); - if (!bridge) return false; - return typeof bridge.listAudioInputDevices === 'function' && typeof bridge.listAudioOutputDevices === 'function'; -} diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeCameraDeviceMapping.test.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeCameraDeviceMapping.test.ts deleted file mode 100644 index fac253055..000000000 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeCameraDeviceMapping.test.ts +++ /dev/null @@ -1,128 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import type {VoiceEngineV2BridgeCameraDevice} from '@fluxer/voice_engine_v2'; -import {describe, expect, it} from 'vitest'; -import {resolveVoiceEngineV2NativeCameraDeviceId} from './VoiceEngineV2AppNativeCameraDeviceMapping'; - -function browserCamera(deviceId: string, label: string): MediaDeviceInfo { - return {deviceId, label, kind: 'videoinput'} as MediaDeviceInfo; -} - -function nativeCamera( - deviceId: string, - label: string, - description = label, - options: {index?: number; deviceIdAliases?: Array} = {}, -): VoiceEngineV2BridgeCameraDevice { - return {deviceId, label, description, ...options}; -} - -describe('resolveVoiceEngineV2NativeCameraDeviceId', () => { - it('keeps default camera selection on the native default route', () => { - const result = resolveVoiceEngineV2NativeCameraDeviceId({ - requestedDeviceId: 'default', - browserDevices: [browserCamera('browser-studio', 'Studio Display Camera')], - nativeDevices: [nativeCamera('native-studio', 'Studio Display Camera')], - }); - - expect(result).toMatchObject({status: 'default'}); - expect(result.deviceId).toBeUndefined(); - }); - - it('accepts native camera ids directly, including non-numeric ids', () => { - expect( - resolveVoiceEngineV2NativeCameraDeviceId({ - requestedDeviceId: 'native-studio', - browserDevices: [browserCamera('browser-studio', 'Studio Display Camera')], - nativeDevices: [nativeCamera('native-studio', 'Studio Display Camera')], - }), - ).toMatchObject({status: 'direct', deviceId: 'native-studio'}); - }); - - it('maps native camera aliases to the canonical native camera id', () => { - expect( - resolveVoiceEngineV2NativeCameraDeviceId({ - requestedDeviceId: '0', - browserDevices: [], - nativeDevices: [ - nativeCamera('native-studio', 'Studio Display Camera', 'Apple Studio Display Camera', { - index: 0, - deviceIdAliases: ['native-studio', '0'], - }), - ], - }), - ).toMatchObject({status: 'direct', deviceId: 'native-studio'}); - }); - - it('reports unavailable for unresolvable ids when no native devices exist', () => { - const result = resolveVoiceEngineV2NativeCameraDeviceId({ - requestedDeviceId: '1', - browserDevices: [], - nativeDevices: [], - }); - - expect(result).toMatchObject({status: 'unavailable', nativeDeviceCount: 0}); - expect(result.deviceId).toBeUndefined(); - }); - - it('maps browser camera ids to native ids by exact camera label', () => { - expect( - resolveVoiceEngineV2NativeCameraDeviceId({ - requestedDeviceId: 'browser-studio', - browserDevices: [browserCamera('browser-studio', 'Studio Display Camera')], - nativeDevices: [nativeCamera('native-studio', 'Studio Display Camera')], - }), - ).toMatchObject({ - status: 'mapped', - deviceId: 'native-studio', - browserLabel: 'Studio Display Camera', - nativeLabel: 'Studio Display Camera', - }); - }); - - it('maps browser labels that include extra transport metadata', () => { - expect( - resolveVoiceEngineV2NativeCameraDeviceId({ - requestedDeviceId: 'browser-c920', - browserDevices: [browserCamera('browser-c920', 'HD Pro Webcam C920 (046d:082d)')], - nativeDevices: [nativeCamera('native-c920', 'HD Pro Webcam C920')], - }), - ).toMatchObject({status: 'mapped', deviceId: 'native-c920'}); - }); - - it('does not pick a native camera when the best label match is ambiguous', () => { - const result = resolveVoiceEngineV2NativeCameraDeviceId({ - requestedDeviceId: 'browser-virtual', - browserDevices: [browserCamera('browser-virtual', 'OBS Virtual Camera')], - nativeDevices: [ - nativeCamera('native-obs-a', 'OBS Virtual Camera'), - nativeCamera('native-obs-b', 'OBS Virtual Camera'), - ], - }); - - expect(result).toMatchObject({status: 'ambiguous', matchCount: 2}); - expect(result.deviceId).toBeUndefined(); - }); - - it('reports unavailable when native camera enumeration returns no devices', () => { - const result = resolveVoiceEngineV2NativeCameraDeviceId({ - requestedDeviceId: 'browser-studio', - browserDevices: [browserCamera('browser-studio', 'Studio Display Camera')], - nativeDevices: [], - }); - - expect(result).toMatchObject({status: 'unavailable', nativeDeviceCount: 0, matchCount: 0}); - expect(result.deviceId).toBeUndefined(); - }); - - it('returns unmapped when Chromium has not exposed a camera label', () => { - const result = resolveVoiceEngineV2NativeCameraDeviceId({ - requestedDeviceId: 'browser-redacted', - browserDevices: [browserCamera('browser-redacted', '')], - nativeDevices: [nativeCamera('native-studio', 'Studio Display Camera')], - }); - - expect(result).toMatchObject({status: 'unmapped'}); - expect(result.deviceId).toBeUndefined(); - }); -}); diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeCameraDeviceMapping.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeCameraDeviceMapping.ts deleted file mode 100644 index 415f3ce02..000000000 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeCameraDeviceMapping.ts +++ /dev/null @@ -1,174 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; -import type {VoiceEngineV2BridgeCameraDevice} from '@fluxer/voice_engine_v2'; - -const MAX_CAMERA_DEVICE_MAPPING_CANDIDATES = 64; -const MAX_CAMERA_DEVICE_MAPPING_ALIASES = 8; - -export type VoiceEngineV2NativeCameraDeviceResolutionStatus = - | 'default' - | 'direct' - | 'mapped' - | 'unmapped' - | 'ambiguous' - | 'unavailable'; - -export interface VoiceEngineV2NativeCameraDeviceResolution { - status: VoiceEngineV2NativeCameraDeviceResolutionStatus; - deviceId?: string; - requestedDeviceId?: string; - browserLabel?: string; - nativeLabel?: string; - nativeDeviceCount: number; - matchCount: number; -} - -export interface VoiceEngineV2NativeCameraDeviceMappingInput { - requestedDeviceId?: string; - browserDevices: ReadonlyArray; - nativeDevices: ReadonlyArray; -} - -interface NativeCameraMatch { - device: VoiceEngineV2BridgeCameraDevice; - score: number; -} - -function normalizeCameraDeviceText(value: string): string { - assert.equal(typeof value, 'string', 'camera device text must be a string'); - return value - .normalize('NFKD') - .toLowerCase() - .replace(/[\u0300-\u036f]/g, '') - .replace(/[^a-z0-9]+/g, ' ') - .trim(); -} - -function nativeCameraDeviceMatchesId(device: VoiceEngineV2BridgeCameraDevice, deviceId: string): boolean { - assert.ok(device !== null && typeof device === 'object', 'native camera device must be an object'); - assert.ok(deviceId.length > 0, 'native camera device id match requires a device id'); - if (device.deviceId === deviceId) return true; - const aliases = device.deviceIdAliases ?? []; - assert.ok(Array.isArray(aliases), 'native camera device aliases must be an array when present'); - const aliasCount = Math.min(aliases.length, MAX_CAMERA_DEVICE_MAPPING_ALIASES); - for (let aliasIndex = 0; aliasIndex < aliasCount; aliasIndex += 1) { - if (aliases[aliasIndex]?.trim() === deviceId) return true; - } - return false; -} - -function cameraDeviceTextMatchScore(browserText: string, nativeText: string, score: number): number { - assert.equal(typeof browserText, 'string', 'browser camera text must be a string'); - assert.equal(typeof nativeText, 'string', 'native camera text must be a string'); - assert.ok(score > 0, 'camera device match score must be positive'); - if (!browserText || !nativeText) return 0; - if (browserText === nativeText) return score; - if (browserText.includes(nativeText) || nativeText.includes(browserText)) return score - 10; - return 0; -} - -function scoreNativeCameraDevice(browserLabel: string, nativeDevice: VoiceEngineV2BridgeCameraDevice): number { - assert.equal(typeof browserLabel, 'string', 'browser camera label must be a string'); - assert.ok(nativeDevice !== null && typeof nativeDevice === 'object', 'native camera device must be an object'); - const browserText = normalizeCameraDeviceText(browserLabel); - const nativeLabel = normalizeCameraDeviceText(nativeDevice.label); - const nativeDescription = normalizeCameraDeviceText(nativeDevice.description); - const labelScore = cameraDeviceTextMatchScore(browserText, nativeLabel, 100); - const descriptionScore = cameraDeviceTextMatchScore(browserText, nativeDescription, 90); - return Math.max(labelScore, descriptionScore); -} - -function findBrowserCameraDevice(devices: ReadonlyArray, deviceId: string): MediaDeviceInfo | null { - assert.ok(Array.isArray(devices), 'browser camera devices must be an array'); - assert.ok(deviceId.length > 0, 'browser camera device lookup requires a device id'); - const deviceCount = Math.min(devices.length, MAX_CAMERA_DEVICE_MAPPING_CANDIDATES); - for (let deviceIndex = 0; deviceIndex < deviceCount; deviceIndex += 1) { - const device = devices[deviceIndex]; - if (device?.deviceId === deviceId) return device; - } - return null; -} - -function findNativeCameraDevice( - devices: ReadonlyArray, - deviceId: string, -): VoiceEngineV2BridgeCameraDevice | null { - assert.ok(Array.isArray(devices), 'native camera devices must be an array'); - assert.ok(deviceId.length > 0, 'native camera device lookup requires a device id'); - const deviceCount = Math.min(devices.length, MAX_CAMERA_DEVICE_MAPPING_CANDIDATES); - for (let deviceIndex = 0; deviceIndex < deviceCount; deviceIndex += 1) { - const device = devices[deviceIndex]; - if (device && nativeCameraDeviceMatchesId(device, deviceId)) return device; - } - return null; -} - -function selectNativeCameraLabelMatch( - browserLabel: string, - devices: ReadonlyArray, -): {match: NativeCameraMatch | null; ambiguous: boolean; matchCount: number} { - assert.equal(typeof browserLabel, 'string', 'browser camera label must be a string'); - assert.ok(Array.isArray(devices), 'native camera devices must be an array'); - let best: NativeCameraMatch | null = null; - let ambiguous = false; - let matchCount = 0; - const deviceCount = Math.min(devices.length, MAX_CAMERA_DEVICE_MAPPING_CANDIDATES); - for (let deviceIndex = 0; deviceIndex < deviceCount; deviceIndex += 1) { - const device = devices[deviceIndex]; - if (!device?.deviceId) continue; - const score = scoreNativeCameraDevice(browserLabel, device); - if (score <= 0) continue; - matchCount += 1; - if (best === null || score > best.score) { - best = {device, score}; - ambiguous = false; - } else if (score === best.score) { - ambiguous = true; - } - } - return {match: best, ambiguous, matchCount}; -} - -export function resolveVoiceEngineV2NativeCameraDeviceId( - input: VoiceEngineV2NativeCameraDeviceMappingInput, -): VoiceEngineV2NativeCameraDeviceResolution { - assert.ok(input !== null && typeof input === 'object', 'native camera device mapping input is required'); - assert.ok(Array.isArray(input.browserDevices), 'browser camera devices must be an array'); - assert.ok(Array.isArray(input.nativeDevices), 'native camera devices must be an array'); - const requestedDeviceId = input.requestedDeviceId?.trim(); - const nativeDeviceCount = Math.min(input.nativeDevices.length, MAX_CAMERA_DEVICE_MAPPING_CANDIDATES); - if (!requestedDeviceId || requestedDeviceId === 'default') { - return {status: 'default', nativeDeviceCount, matchCount: 0}; - } - const directNativeDevice = findNativeCameraDevice(input.nativeDevices, requestedDeviceId); - if (directNativeDevice) { - return { - status: 'direct', - deviceId: directNativeDevice.deviceId, - requestedDeviceId, - nativeLabel: directNativeDevice.label, - nativeDeviceCount, - matchCount: 1, - }; - } - if (nativeDeviceCount === 0) { - return {status: 'unavailable', requestedDeviceId, nativeDeviceCount, matchCount: 0}; - } - assert.ok(nativeDeviceCount > 0, 'native camera label matching requires native devices'); - const browserDevice = findBrowserCameraDevice(input.browserDevices, requestedDeviceId); - const browserLabel = browserDevice?.label.trim(); - if (!browserLabel) return {status: 'unmapped', requestedDeviceId, nativeDeviceCount, matchCount: 0}; - const {match, ambiguous, matchCount} = selectNativeCameraLabelMatch(browserLabel, input.nativeDevices); - if (match === null) return {status: 'unmapped', requestedDeviceId, browserLabel, nativeDeviceCount, matchCount}; - if (ambiguous) return {status: 'ambiguous', requestedDeviceId, browserLabel, nativeDeviceCount, matchCount}; - return { - status: 'mapped', - deviceId: match.device.deviceId, - requestedDeviceId, - browserLabel, - nativeLabel: match.device.label, - nativeDeviceCount, - matchCount, - }; -} diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeCaptureExecutionAdapter.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeCaptureExecutionAdapter.ts deleted file mode 100644 index cc9147717..000000000 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeCaptureExecutionAdapter.ts +++ /dev/null @@ -1,395 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {Logger} from '@app/features/platform/utils/AppLogger'; -import type { - NativeMediaPort, - VoiceEngineV2NativeAudioTapOptions, - VoiceEngineV2NativeCaptureOptions, - VoiceEngineV2NativeFrameSinkOptions, -} from '@fluxer/voice_engine_v2'; -import { - assertVoiceEngineV2BridgeAudioOptionsInvariants, - assertVoiceEngineV2BridgeVideoOptionsInvariants, -} from '@fluxer/voice_engine_v2/bridge'; - -export type VoiceEngineV2NativeMediaDiagnosticLevel = 'debug' | 'info' | 'warn' | 'error'; - -export interface VoiceEngineV2NativeMediaDiagnosticEvent { - level: VoiceEngineV2NativeMediaDiagnosticLevel; - code: string; - captureId?: string; - sinkId?: string; - tapId?: string; - detail?: unknown; -} - -export interface ElectronNativeMediaStartResult { - captureId: string; - zeroCopyAvailable: boolean; - fallback?: 'cpu-copy' | null; -} - -export interface ElectronNativeMediaUpdateResult { - captureId: string; - zeroCopyAvailable: boolean; - fallback?: 'cpu-copy' | null; -} - -export interface ElectronNativeMediaAttachResult { - sinkId: string; - zeroCopyAvailable: boolean; - fallback?: 'cpu-copy' | null; -} - -export interface ElectronNativeMediaStartOptions extends VoiceEngineV2NativeCaptureOptions { - zeroCopyRequired: true; -} - -export interface ElectronNativeMediaAttachOptions { - trackSid?: string; - zeroCopyRequired: true; -} - -export interface ElectronNativeMediaApi { - start(options: ElectronNativeMediaStartOptions): Promise; - update(options: ElectronNativeMediaStartOptions): Promise; - stop(captureId: string): Promise; - startAudioTap?(options: VoiceEngineV2NativeAudioTapOptions): Promise<{tapId: string}>; - stopAudioTap?(tapId: string): Promise; - attachSink( - captureId: string, - sinkId: string, - options: ElectronNativeMediaAttachOptions, - ): Promise; - detachSink(sinkId: string): Promise; - isKnownCapture?(captureId: string): boolean; -} - -export interface VoiceEngineV2AppNativeCaptureExecutionAdapterOptions { - electronApi?: ElectronNativeMediaApi | null; - getDiagnosticsEmit?: (event: VoiceEngineV2NativeMediaDiagnosticEvent) => void; - logger?: Logger; -} - -const ADAPTER_NAME = 'VoiceEngineV2AppNativeCaptureExecutionAdapter'; -const ZERO_COPY_DIAGNOSTIC_CODE = 'zeroCopyUnavailable'; -const AUDIO_TAP_NOT_SUPPORTED_CODE = 'audioTapNotSupported'; -const ZERO_COPY_REQUIRED_FAILURE_REASON = 'zeroCopyRequired could not be satisfied'; - -function buildOperatingError(method: string, reason: string): Error { - const error = new Error(`${ADAPTER_NAME}.${method}: ${reason}`); - error.name = 'VoiceEngineV2AppNativeMediaOperatingError'; - return error; -} - -function assertNonEmptyString(value: unknown, method: string, field: string): asserts value is string { - if (typeof value !== 'string') { - throw buildOperatingError(method, `${field} is not a string`); - } - if (value.length === 0) { - throw buildOperatingError(method, `${field} must be a non-empty string`); - } -} - -function assertCaptureOptions( - method: string, - options: VoiceEngineV2NativeCaptureOptions | undefined | null, -): asserts options is VoiceEngineV2NativeCaptureOptions { - if (typeof options !== 'object' || options === null) { - throw buildOperatingError(method, 'options is not an object'); - } - assertNonEmptyString(options.captureId, method, 'options.captureId'); - if (typeof options.source !== 'object' || options.source === null) { - throw buildOperatingError(method, 'options.source is not an object'); - } - assertNonEmptyString(options.source.id, method, 'options.source.id'); - if (typeof options.width !== 'number' || !Number.isFinite(options.width) || options.width <= 0) { - throw buildOperatingError(method, 'options.width is not a positive number'); - } - if (typeof options.height !== 'number' || !Number.isFinite(options.height) || options.height <= 0) { - throw buildOperatingError(method, 'options.height is not a positive number'); - } - if (typeof options.frameRate !== 'number' || !Number.isFinite(options.frameRate) || options.frameRate <= 0) { - throw buildOperatingError(method, 'options.frameRate is not a positive number'); - } - if (typeof options.includeCursor !== 'boolean') { - throw buildOperatingError(method, 'options.includeCursor is not a boolean'); - } - if (typeof options.includeAudio !== 'boolean') { - throw buildOperatingError(method, 'options.includeAudio is not a boolean'); - } -} - -function assertAudioTapOptions( - method: string, - options: VoiceEngineV2NativeAudioTapOptions | undefined | null, -): asserts options is VoiceEngineV2NativeAudioTapOptions { - if (typeof options !== 'object' || options === null) { - throw buildOperatingError(method, 'options is not an object'); - } - assertNonEmptyString(options.tapId, method, 'options.tapId'); - switch (options.source) { - case 'system': - case 'application': - case 'window': - break; - default: { - throw buildOperatingError(method, 'options.source is not a known kind'); - } - } - if (typeof options.sampleRate !== 'number' || !Number.isFinite(options.sampleRate) || options.sampleRate <= 0) { - throw buildOperatingError(method, 'options.sampleRate is not a positive number'); - } - if (typeof options.numChannels !== 'number' || !Number.isFinite(options.numChannels) || options.numChannels <= 0) { - throw buildOperatingError(method, 'options.numChannels is not a positive number'); - } -} - -function assertFrameSinkOptions( - method: string, - options: VoiceEngineV2NativeFrameSinkOptions | undefined | null, -): asserts options is VoiceEngineV2NativeFrameSinkOptions { - if (typeof options !== 'object' || options === null) { - throw buildOperatingError(method, 'options is not an object'); - } - assertNonEmptyString(options.sinkId, method, 'options.sinkId'); - assertNonEmptyString(options.captureId, method, 'options.captureId'); - if (options.trackSid !== undefined && (typeof options.trackSid !== 'string' || options.trackSid.length === 0)) { - throw buildOperatingError(method, 'options.trackSid must be a non-empty string when provided'); - } -} - -function isZeroCopyFallback(result: {zeroCopyAvailable: boolean; fallback?: 'cpu-copy' | null}): boolean { - if (result.zeroCopyAvailable === false) return true; - if (result.fallback === 'cpu-copy') return true; - return false; -} - -export class VoiceEngineV2AppNativeCaptureExecutionAdapter implements NativeMediaPort { - private readonly electronApi: ElectronNativeMediaApi | null; - private readonly emitDiagnostic: (event: VoiceEngineV2NativeMediaDiagnosticEvent) => void; - private readonly logger: Logger; - private readonly knownCaptures: Set; - private readonly knownSinks: Map; - private readonly knownTaps: Set; - - constructor(options: VoiceEngineV2AppNativeCaptureExecutionAdapterOptions = {}) { - const electronApi = options.electronApi ?? null; - if (electronApi !== null && typeof electronApi !== 'object') { - throw new Error(`${ADAPTER_NAME}: electronApi must be an object when provided`); - } - this.electronApi = electronApi; - this.emitDiagnostic = options.getDiagnosticsEmit ?? (() => undefined); - this.logger = options.logger ?? new Logger(ADAPTER_NAME); - this.knownCaptures = new Set(); - this.knownSinks = new Map(); - this.knownTaps = new Set(); - } - - async startCapture(options: VoiceEngineV2NativeCaptureOptions): Promise { - assertCaptureOptions('startCapture', options); - assertVoiceEngineV2BridgeVideoOptionsInvariants({width: options.width, height: options.height}); - const api = this.electronApi; - if (api === null) { - throw buildOperatingError('startCapture', 'electronApi is unavailable'); - } - const result = await this.invoke('startCapture', () => api.start({...options, zeroCopyRequired: true})); - if (typeof result !== 'object' || result === null) { - throw buildOperatingError('startCapture', 'electronApi.start returned a non-object'); - } - assertNonEmptyString(result.captureId, 'startCapture', 'result.captureId'); - if (result.captureId !== options.captureId) { - throw buildOperatingError('startCapture', 'result.captureId does not match the requested captureId'); - } - if (isZeroCopyFallback(result)) { - this.emitZeroCopyFallback({captureId: result.captureId}); - await this.stopStartedZeroCopyFallback(api, result.captureId, 'startCapture'); - throw buildOperatingError('startCapture', ZERO_COPY_REQUIRED_FAILURE_REASON); - } - this.knownCaptures.add(result.captureId); - } - - async updateCapture(options: VoiceEngineV2NativeCaptureOptions): Promise { - assertCaptureOptions('updateCapture', options); - assertVoiceEngineV2BridgeVideoOptionsInvariants({width: options.width, height: options.height}); - const api = this.electronApi; - if (api === null) { - throw buildOperatingError('updateCapture', 'electronApi is unavailable'); - } - if (!this.isCaptureKnown(options.captureId)) { - throw buildOperatingError('updateCapture', 'captureId is not known'); - } - const result = await this.invoke('updateCapture', () => api.update({...options, zeroCopyRequired: true})); - if (typeof result !== 'object' || result === null) { - throw buildOperatingError('updateCapture', 'electronApi.update returned a non-object'); - } - assertNonEmptyString(result.captureId, 'updateCapture', 'result.captureId'); - if (result.captureId !== options.captureId) { - throw buildOperatingError('updateCapture', 'result.captureId does not match the requested captureId'); - } - if (isZeroCopyFallback(result)) { - this.emitZeroCopyFallback({captureId: result.captureId}); - await this.stopStartedZeroCopyFallback(api, result.captureId, 'updateCapture'); - this.knownCaptures.delete(result.captureId); - this.releaseSinksForCapture(result.captureId); - throw buildOperatingError('updateCapture', ZERO_COPY_REQUIRED_FAILURE_REASON); - } - } - - async stopCapture(captureId: string): Promise { - assertNonEmptyString(captureId, 'stopCapture', 'captureId'); - const api = this.electronApi; - if (api === null) { - throw buildOperatingError('stopCapture', 'electronApi is unavailable'); - } - if (!this.isCaptureKnown(captureId)) { - throw buildOperatingError('stopCapture', 'captureId is not known'); - } - await this.invoke('stopCapture', () => api.stop(captureId)); - this.knownCaptures.delete(captureId); - this.releaseSinksForCapture(captureId); - } - - async startAudioTap(options: VoiceEngineV2NativeAudioTapOptions): Promise { - assertAudioTapOptions('startAudioTap', options); - assertVoiceEngineV2BridgeAudioOptionsInvariants({ - sampleRate: options.sampleRate, - numChannels: options.numChannels, - }); - const api = this.electronApi; - if (api === null) { - throw buildOperatingError('startAudioTap', 'electronApi is unavailable'); - } - if (typeof api.startAudioTap !== 'function') { - throw buildOperatingError('startAudioTap', AUDIO_TAP_NOT_SUPPORTED_CODE); - } - const result = await this.invoke('startAudioTap', () => api.startAudioTap!(options)); - if (typeof result !== 'object' || result === null) { - throw buildOperatingError('startAudioTap', 'electronApi.startAudioTap returned a non-object'); - } - assertNonEmptyString(result.tapId, 'startAudioTap', 'result.tapId'); - if (result.tapId !== options.tapId) { - throw buildOperatingError('startAudioTap', 'result.tapId does not match the requested tapId'); - } - this.knownTaps.add(result.tapId); - } - - async stopAudioTap(tapId: string): Promise { - assertNonEmptyString(tapId, 'stopAudioTap', 'tapId'); - const api = this.electronApi; - if (api === null) { - throw buildOperatingError('stopAudioTap', 'electronApi is unavailable'); - } - if (typeof api.stopAudioTap !== 'function') { - throw buildOperatingError('stopAudioTap', AUDIO_TAP_NOT_SUPPORTED_CODE); - } - if (!this.knownTaps.has(tapId)) { - throw buildOperatingError('stopAudioTap', 'tapId is not known'); - } - await this.invoke('stopAudioTap', () => api.stopAudioTap!(tapId)); - this.knownTaps.delete(tapId); - } - - async attachFrameSink(options: VoiceEngineV2NativeFrameSinkOptions): Promise { - assertFrameSinkOptions('attachFrameSink', options); - const api = this.electronApi; - if (api === null) { - throw buildOperatingError('attachFrameSink', 'electronApi is unavailable'); - } - if (!this.isCaptureKnown(options.captureId)) { - throw buildOperatingError('attachFrameSink', 'captureId is not known'); - } - const result = await this.invoke('attachFrameSink', () => - api.attachSink(options.captureId, options.sinkId, { - trackSid: options.trackSid, - zeroCopyRequired: true, - }), - ); - if (typeof result !== 'object' || result === null) { - throw buildOperatingError('attachFrameSink', 'electronApi.attachSink returned a non-object'); - } - assertNonEmptyString(result.sinkId, 'attachFrameSink', 'result.sinkId'); - if (result.sinkId !== options.sinkId) { - throw buildOperatingError('attachFrameSink', 'result.sinkId does not match the requested sinkId'); - } - if (isZeroCopyFallback(result)) { - this.emitZeroCopyFallback({captureId: options.captureId, sinkId: result.sinkId}); - await this.detachAttachedZeroCopyFallback(api, result.sinkId, 'attachFrameSink'); - throw buildOperatingError('attachFrameSink', ZERO_COPY_REQUIRED_FAILURE_REASON); - } - this.knownSinks.set(result.sinkId, options.captureId); - } - - async detachFrameSink(sinkId: string): Promise { - assertNonEmptyString(sinkId, 'detachFrameSink', 'sinkId'); - const api = this.electronApi; - if (api === null) { - throw buildOperatingError('detachFrameSink', 'electronApi is unavailable'); - } - if (!this.knownSinks.has(sinkId)) { - throw buildOperatingError('detachFrameSink', 'sinkId is not known'); - } - await this.invoke('detachFrameSink', () => api.detachSink(sinkId)); - this.knownSinks.delete(sinkId); - } - - private isCaptureKnown(captureId: string): boolean { - if (this.knownCaptures.has(captureId)) return true; - const api = this.electronApi; - if (api === null) return false; - if (typeof api.isKnownCapture !== 'function') return false; - return api.isKnownCapture(captureId) === true; - } - - private releaseSinksForCapture(captureId: string): void { - const toRelease: Array = []; - for (const [sinkId, owner] of this.knownSinks) { - if (owner === captureId) toRelease.push(sinkId); - } - for (const sinkId of toRelease) { - this.knownSinks.delete(sinkId); - } - } - - private emitZeroCopyFallback(target: {captureId: string; sinkId?: string}): void { - this.logger.warn('zero-copy unavailable, falling back to CPU copy', target); - this.emitDiagnostic({ - level: 'warn', - code: ZERO_COPY_DIAGNOSTIC_CODE, - captureId: target.captureId, - sinkId: target.sinkId, - }); - } - - private async stopStartedZeroCopyFallback( - api: ElectronNativeMediaApi, - captureId: string, - method: string, - ): Promise { - assertNonEmptyString(captureId, method, 'captureId'); - await this.invoke(`${method}.stopFallbackCapture`, () => api.stop(captureId)).catch((error) => { - this.logger.warn('Failed to stop native capture after zero-copy fallback', {captureId, method, error}); - }); - } - - private async detachAttachedZeroCopyFallback( - api: ElectronNativeMediaApi, - sinkId: string, - method: string, - ): Promise { - assertNonEmptyString(sinkId, method, 'sinkId'); - await this.invoke(`${method}.detachFallbackSink`, () => api.detachSink(sinkId)).catch((error) => { - this.logger.warn('Failed to detach native frame sink after zero-copy fallback', {sinkId, method, error}); - }); - } - - private async invoke(method: string, body: () => Promise | T): Promise { - try { - return await Promise.resolve(body()); - } catch (error) { - this.logger.warn(`${method} delegate threw`, {error}); - if (error instanceof Error) throw error; - throw buildOperatingError(method, typeof error === 'string' ? error : 'delegate failed'); - } - } -} diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeLocalTrackStateSync.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeLocalTrackStateSync.ts deleted file mode 100644 index add60265b..000000000 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeLocalTrackStateSync.ts +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; -import LocalVoiceState from '@app/features/voice/state/LocalVoiceState'; - -export function applyVoiceEngineV2NativeScreenShareAudioState(enabled: boolean): void { - assert.equal(typeof enabled, 'boolean', 'native screen-share audio state must be a boolean'); - const selfStream = LocalVoiceState.getSelfStream(); - assert.equal(typeof selfStream, 'boolean', 'self stream state must be a boolean'); - LocalVoiceState.updateSelfStreamAudio(enabled && selfStream); -} diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeMicrophoneSettings.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeMicrophoneSettings.ts deleted file mode 100644 index 7bd55167d..000000000 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeMicrophoneSettings.ts +++ /dev/null @@ -1,64 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; -import VoiceDevicePermissionState from '@app/features/voice/engine/VoiceDevicePermissionState'; -import type VoiceSettingsStore from '@app/features/voice/state/VoiceSettings'; -import {normaliseAudioBitrateBps} from '@app/features/voice/utils/AudioPublishOptions'; -import {resolveEffectiveDeviceId} from '@app/features/voice/utils/VoiceDeviceManager'; -import { - getActiveInputDeviceLabel, - resolveVoiceProcessingFromStateForDeviceLabel, -} from '@app/features/voice/utils/VoiceProcessingProfile'; -import type {VoiceEngineV2MicrophoneOptions} from '@fluxer/voice_engine_v2'; - -export interface VoiceEngineV2NativeMicrophonePublishOptions extends VoiceEngineV2MicrophoneOptions { - maxBitrateBps?: number; -} - -export function resolveVoiceEngineV2NativeMicrophoneMaxBitrateBps( - channelBitrateBps: number | null | undefined, -): number | undefined { - assert.ok( - channelBitrateBps == null || typeof channelBitrateBps === 'number', - 'native microphone channel bitrate must be a number when provided', - ); - const maxBitrateBps = normaliseAudioBitrateBps(channelBitrateBps); - assert.ok( - maxBitrateBps === undefined || maxBitrateBps > 0, - 'native microphone max bitrate must be positive when resolved', - ); - return maxBitrateBps; -} - -export function resolveVoiceEngineV2NativeMicrophoneDeviceId(store: typeof VoiceSettingsStore): string { - assert.equal(typeof store.getInputDeviceId, 'function', 'native microphone settings store requires getInputDeviceId'); - const storedDeviceId = store.getInputDeviceId(); - assert.equal(typeof storedDeviceId, 'string', 'native microphone stored input device id must be a string'); - const {inputDevices} = VoiceDevicePermissionState.getState(); - if (inputDevices.length === 0) return storedDeviceId; - const effectiveDeviceId = resolveEffectiveDeviceId(storedDeviceId, inputDevices); - assert.ok(effectiveDeviceId !== null, 'native microphone device id must resolve when input devices are present'); - return effectiveDeviceId; -} - -export function resolveVoiceEngineV2NativeMicrophonePublishOptions( - store: typeof VoiceSettingsStore, - options: VoiceEngineV2NativeMicrophonePublishOptions = {}, - channelBitrateBps: number | null = null, -): VoiceEngineV2NativeMicrophonePublishOptions { - assert.ok(store !== null && typeof store === 'object', 'native microphone settings store must be an object'); - assert.equal(typeof store.getInputDeviceId, 'function', 'native microphone settings store requires getInputDeviceId'); - assert.ok(options !== null && typeof options === 'object', 'native microphone options must be an object'); - const activeInputDeviceLabel = getActiveInputDeviceLabel(store); - const profile = resolveVoiceProcessingFromStateForDeviceLabel(store, activeInputDeviceLabel); - const maxBitrateBps = options.maxBitrateBps ?? resolveVoiceEngineV2NativeMicrophoneMaxBitrateBps(channelBitrateBps); - return { - deviceId: options.deviceId ?? resolveVoiceEngineV2NativeMicrophoneDeviceId(store), - echoCancellation: options.echoCancellation ?? profile.echoCancellation, - noiseSuppression: options.noiseSuppression ?? profile.browserNoiseSuppression, - autoGainControl: options.autoGainControl ?? profile.autoGainControl, - deepFilter: options.deepFilter ?? profile.deepFilter, - deepFilterNoiseReductionLevel: options.deepFilterNoiseReductionLevel ?? profile.deepFilterNoiseReductionLevel, - ...(maxBitrateBps !== undefined ? {maxBitrateBps} : {}), - }; -} diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeVoiceConnectionLifecycle.test.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeVoiceConnectionLifecycle.test.ts deleted file mode 100644 index 940296ca7..000000000 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeVoiceConnectionLifecycle.test.ts +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {describe, expect, it} from 'vitest'; -import type {VoiceConnectionState} from './VoiceEngineV2AppConnectionHostAdapter'; -import {VoiceEngineV2AppNativeVoiceConnectionLifecycle} from './VoiceEngineV2AppNativeVoiceConnectionLifecycle'; - -function connectionState(overrides: Partial = {}): VoiceConnectionState { - return { - room: null, - guildId: 'guild-1', - channelId: 'channel-1', - connecting: true, - connected: false, - reconnecting: false, - voiceServerEndpoint: 'wss://voice.example.test', - connectionId: 'conn-1', - ...overrides, - }; -} - -describe('VoiceEngineV2AppNativeVoiceConnectionLifecycle', () => { - it('creates monotonic attempts and ignores stale clear requests', () => { - const lifecycle = new VoiceEngineV2AppNativeVoiceConnectionLifecycle(); - const first = lifecycle.createAttempt({ - guildId: 'guild-1', - channelId: 'channel-1', - connectionId: 'conn-1', - endpoint: 'wss://voice.example.test', - reason: 'server-update', - }); - const second = lifecycle.createAttempt({ - guildId: 'guild-1', - channelId: 'channel-1', - connectionId: 'conn-2', - endpoint: 'wss://voice.example.test', - reason: 'connect-timeout-retry', - }); - - lifecycle.clearActiveAttempt(first); - - expect(first.id).toBe(1); - expect(second.id).toBe(2); - expect(lifecycle.activeAttemptId).toBe(2); - expect(lifecycle.isActiveAttempt(second)).toBe(true); - }); - - it('checks the active attempt against the current connection identity', () => { - const lifecycle = new VoiceEngineV2AppNativeVoiceConnectionLifecycle(); - const attempt = lifecycle.createAttempt({ - guildId: 'guild-1', - channelId: 'channel-1', - connectionId: 'conn-1', - endpoint: 'wss://voice.example.test', - reason: 'server-update', - }); - - expect(lifecycle.isCurrentAttemptForConnection(attempt, connectionState())).toBe(true); - expect(lifecycle.isCurrentAttemptForConnection(attempt, connectionState({connectionId: 'conn-2'}))).toBe(false); - }); - - it('detects duplicate active and ready server updates', () => { - const lifecycle = new VoiceEngineV2AppNativeVoiceConnectionLifecycle(); - lifecycle.createAttempt({ - guildId: 'guild-1', - channelId: 'channel-1', - connectionId: 'conn-1', - endpoint: 'wss://voice.example.test', - reason: 'server-update', - }); - const update = { - guildId: 'guild-1', - channelId: 'channel-1', - connectionId: 'conn-1', - endpoint: 'wss://voice.example.test', - token: 'token-1', - }; - - expect(lifecycle.isDuplicateServerUpdate(update, connectionState(), 'token-1')).toBe(true); - expect( - lifecycle.isDuplicateServerUpdate(update, connectionState({voiceServerEndpoint: 'wss://other'}), 'token-1'), - ).toBe(false); - - lifecycle.clearSession(); - lifecycle.setReadyConnectionId('conn-1'); - expect( - lifecycle.isDuplicateServerUpdate(update, connectionState({connecting: false, connected: true}), 'token-1'), - ).toBe(true); - }); -}); diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeVoiceConnectionLifecycle.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeVoiceConnectionLifecycle.ts deleted file mode 100644 index 26b77d6f4..000000000 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeVoiceConnectionLifecycle.ts +++ /dev/null @@ -1,126 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; -import type {VoiceConnectionState} from './VoiceEngineV2AppConnectionHostAdapter'; - -export type VoiceEngineV2AppNativeVoiceConnectReason = - | 'server-update' - | 'connect-timeout-retry' - | 'transport-reconnect'; - -export interface VoiceEngineV2AppNativeVoiceConnectAttempt { - id: number; - guildId: string | null; - channelId: string; - connectionId: string | null; - endpoint: string; - reason: VoiceEngineV2AppNativeVoiceConnectReason; -} - -export interface VoiceEngineV2AppNativeVoiceServerUpdateIdentity { - guildId: string | null; - channelId: string; - connectionId: string | null; - endpoint: string; - token: string; -} - -export class VoiceEngineV2AppNativeVoiceConnectionLifecycle { - private connectAttemptSerial = 0; - private activeAttempt: VoiceEngineV2AppNativeVoiceConnectAttempt | null = null; - private readyConnectionIdValue: string | null = null; - - get activeAttemptId(): number | null { - return this.activeAttempt?.id ?? null; - } - - get currentAttempt(): VoiceEngineV2AppNativeVoiceConnectAttempt | null { - return this.activeAttempt; - } - - get readyConnectionId(): string | null { - return this.readyConnectionIdValue; - } - - setReadyConnectionId(connectionId: string | null): void { - assert.ok(connectionId === null || typeof connectionId === 'string', 'ready connection id must be string or null'); - this.readyConnectionIdValue = connectionId; - } - - createAttempt( - input: Omit, - ): VoiceEngineV2AppNativeVoiceConnectAttempt { - assert.ok(input !== null && typeof input === 'object', 'native voice attempt input is required'); - assert.equal(typeof input.channelId, 'string', 'native voice attempt channelId must be a string'); - assert.ok(input.channelId.length > 0, 'native voice attempt channelId must not be empty'); - assert.equal(typeof input.endpoint, 'string', 'native voice attempt endpoint must be a string'); - assert.ok(input.endpoint.length > 0, 'native voice attempt endpoint must not be empty'); - this.connectAttemptSerial += 1; - const attempt = {...input, id: this.connectAttemptSerial}; - this.activeAttempt = attempt; - return attempt; - } - - clearActiveAttempt(attempt?: VoiceEngineV2AppNativeVoiceConnectAttempt): void { - if (attempt && this.activeAttempt?.id !== attempt.id) return; - this.activeAttempt = null; - } - - clearSession(): void { - this.clearActiveAttempt(); - this.readyConnectionIdValue = null; - } - - isActiveAttempt(attempt: VoiceEngineV2AppNativeVoiceConnectAttempt): boolean { - assert.ok(attempt !== null && typeof attempt === 'object', 'active attempt check requires an attempt'); - assert.ok(Number.isInteger(attempt.id), 'active attempt id must be an integer'); - return this.activeAttempt?.id === attempt.id; - } - - isCurrentAttemptForConnection( - attempt: VoiceEngineV2AppNativeVoiceConnectAttempt, - current: VoiceConnectionState, - ): boolean { - assert.ok(current !== null && typeof current === 'object', 'current voice connection state is required'); - if (!this.isActiveAttempt(attempt)) return false; - if (current.connectionId !== attempt.connectionId) return false; - if (current.channelId !== attempt.channelId) return false; - return (current.guildId ?? null) === attempt.guildId; - } - - isDuplicateServerUpdate( - update: VoiceEngineV2AppNativeVoiceServerUpdateIdentity, - current: VoiceConnectionState, - previousToken: string | null, - ): boolean { - assert.ok(update !== null && typeof update === 'object', 'server update identity is required'); - assert.ok(current !== null && typeof current === 'object', 'current voice connection state is required'); - if (!update.connectionId) return false; - if (previousToken !== update.token) return false; - if (!voiceConnectionMatchesUpdate(current, update)) return false; - if (!current.connecting && !current.connected && !current.reconnecting) return false; - if (this.readyConnectionIdValue === update.connectionId && current.connected) return true; - return attemptMatchesUpdate(this.activeAttempt, update); - } -} - -function voiceConnectionMatchesUpdate( - current: VoiceConnectionState, - update: VoiceEngineV2AppNativeVoiceServerUpdateIdentity, -): boolean { - if (current.guildId !== update.guildId) return false; - if (current.channelId !== update.channelId) return false; - if (current.connectionId !== update.connectionId) return false; - return current.voiceServerEndpoint === update.endpoint; -} - -function attemptMatchesUpdate( - attempt: VoiceEngineV2AppNativeVoiceConnectAttempt | null, - update: VoiceEngineV2AppNativeVoiceServerUpdateIdentity, -): boolean { - if (!attempt) return false; - if (attempt.guildId !== update.guildId) return false; - if (attempt.channelId !== update.channelId) return false; - if (attempt.connectionId !== update.connectionId) return false; - return attempt.endpoint === update.endpoint; -} diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeVoiceLiveKitMediaAdapter.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeVoiceLiveKitMediaAdapter.ts deleted file mode 100644 index a3d8b925e..000000000 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppNativeVoiceLiveKitMediaAdapter.ts +++ /dev/null @@ -1,140 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {Logger} from '@app/features/platform/utils/AppLogger'; -import type {VoiceEngine} from '@app/features/voice/engine/native_voice_engine/VoiceEngine'; -import type { - LiveKitMediaPort, - VoiceEngineV2CameraEncodingOptions, - VoiceEngineV2CameraOptions, - VoiceEngineV2ConnectOptions, - VoiceEngineV2DataOptions, - VoiceEngineV2DisconnectReason, - VoiceEngineV2MicrophoneOptions, - VoiceEngineV2OutputDeviceOptions, - VoiceEngineV2ScreenAudioOptions, - VoiceEngineV2ScreenEncodingOptions, - VoiceEngineV2ScreenOptions, -} from '@fluxer/voice_engine_v2'; -import type {VoiceEngineV2BridgeOperationResult} from '@fluxer/voice_engine_v2/bridge'; - -const ADAPTER_NAME = 'VoiceEngineV2AppNativeVoiceLiveKitMediaAdapter'; - -export interface VoiceEngineV2AppNativeVoiceMediaCameraDelegate { - publishCamera(options: VoiceEngineV2CameraOptions): Promise; - updateCameraEncoding(options: VoiceEngineV2CameraEncodingOptions): Promise; - unpublishCamera(options?: VoiceEngineV2CameraOptions): Promise; -} - -export interface VoiceEngineV2AppNativeVoiceMediaScreenDelegate { - publishScreen(options: VoiceEngineV2ScreenOptions): Promise; - updateScreenEncoding(options: VoiceEngineV2ScreenEncodingOptions): Promise; - unpublishScreen(): Promise; -} - -export interface VoiceEngineV2AppNativeVoiceLiveKitMediaAdapterOptions { - getEngine: () => VoiceEngine; - camera: VoiceEngineV2AppNativeVoiceMediaCameraDelegate; - screen: VoiceEngineV2AppNativeVoiceMediaScreenDelegate; - logger?: Logger; -} - -function buildOperationError(method: string, result: VoiceEngineV2BridgeOperationResult): Error { - const message = result.ok ? 'operation unexpectedly succeeded' : result.error.message; - const error = new Error(`${ADAPTER_NAME}.${method}: ${message}`); - error.name = 'VoiceEngineV2AppNativeVoiceMediaOperationError'; - return error; -} - -function assertOperationSucceeded(method: string, result: VoiceEngineV2BridgeOperationResult): void { - if (result.ok) return; - throw buildOperationError(method, result); -} - -export class VoiceEngineV2AppNativeVoiceLiveKitMediaAdapter implements LiveKitMediaPort { - private readonly getEngine: () => VoiceEngine; - private readonly camera: VoiceEngineV2AppNativeVoiceMediaCameraDelegate; - private readonly screen: VoiceEngineV2AppNativeVoiceMediaScreenDelegate; - private readonly logger: Logger; - - constructor(options: VoiceEngineV2AppNativeVoiceLiveKitMediaAdapterOptions) { - if (typeof options !== 'object' || options === null) { - throw new Error(`${ADAPTER_NAME}: options is required`); - } - this.getEngine = options.getEngine; - this.camera = options.camera; - this.screen = options.screen; - this.logger = options.logger ?? new Logger(ADAPTER_NAME); - } - - async prewarm(): Promise { - await this.getEngine().prewarm(); - } - - async connect(options: VoiceEngineV2ConnectOptions): Promise { - await this.getEngine().connect(options); - } - - async disconnect(_reason: VoiceEngineV2DisconnectReason): Promise { - await this.getEngine().disconnect(); - } - - async publishMicrophone(options: VoiceEngineV2MicrophoneOptions): Promise { - const result = await this.getEngine().publishMicrophone(options); - assertOperationSucceeded('publishMicrophone', result); - } - - async unpublishMicrophone(): Promise { - const result = await this.getEngine().setMicEnabled(false); - assertOperationSucceeded('unpublishMicrophone', result); - } - - async setMicrophoneEnabled(enabled: boolean): Promise { - const result = await this.getEngine().setMicEnabled(enabled); - assertOperationSucceeded('setMicrophoneEnabled', result); - } - - async publishCamera(options: VoiceEngineV2CameraOptions): Promise { - await this.camera.publishCamera(options); - } - - async updateCameraEncoding(options: VoiceEngineV2CameraEncodingOptions): Promise { - await this.camera.updateCameraEncoding(options); - } - - async unpublishCamera(options?: VoiceEngineV2CameraOptions): Promise { - await this.camera.unpublishCamera(options); - } - - async publishScreen(options: VoiceEngineV2ScreenOptions): Promise { - await this.screen.publishScreen(options); - } - - async updateScreenEncoding(options: VoiceEngineV2ScreenEncodingOptions): Promise { - await this.screen.updateScreenEncoding(options); - } - - async unpublishScreen(): Promise { - await this.screen.unpublishScreen(); - } - - async publishScreenAudio(options: VoiceEngineV2ScreenAudioOptions): Promise { - await this.getEngine().publishScreenShareAudio(options); - } - - async unpublishScreenAudio(): Promise { - await this.getEngine().unpublishScreenShareAudio(); - } - - async setOutputDevice(options: VoiceEngineV2OutputDeviceOptions): Promise { - await this.getEngine().setAudioOutputDevice(options.deviceId); - } - - async publishData(options: VoiceEngineV2DataOptions): Promise { - try { - await this.getEngine().publishData(options); - } catch (error) { - this.logger.warn('Native data publish failed', {error}); - throw error; - } - } -} diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppParticipantAdapter.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppParticipantAdapter.ts index 52ef1acbe..b4e63cdc5 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppParticipantAdapter.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppParticipantAdapter.ts @@ -2,12 +2,7 @@ import assert from 'node:assert/strict'; import type {LivekitParticipantSnapshot} from '@app/features/voice/engine/VoiceParticipantStateMachine'; -import { - asVoiceConnectionQuality, - type VoiceConnectionQuality, - VoiceConnectionQuality as VoiceConnectionQualityValue, - VoiceTrackSource, -} from '@app/features/voice/engine/VoiceTrackSource'; +import {asVoiceConnectionQuality, VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource'; import type { VoiceEngineV2Controller, VoiceEngineV2Event, @@ -19,30 +14,7 @@ import type {Participant, Room, TrackPublication} from 'livekit-client'; import {Track} from 'livekit-client'; import {assertNonNullObject, assertString} from './VoiceEngineV2AppAdapterAssertions'; -export const NATIVE_SPEAKING_HEARTBEAT_TIMEOUT_MS = 3_000; const MAX_DISCARDED_CONNECTION_IDS = 4096; -const MAX_NATIVE_SPEAKING_HEARTBEATS = 4096; - -export interface VoiceEngineV2AppNativeSpeakingSample { - participantSid: string; - identity: string; - source: string; - isLocal: boolean; - speaking: boolean; -} - -export interface VoiceEngineV2AppNativeParticipantFields { - identity: string; - sid: string; - name?: string; - isLocal?: boolean; - metadata?: string; - attributes?: Readonly>; - isMicrophoneEnabled?: boolean; - isCameraEnabled?: boolean; - isScreenShareEnabled?: boolean; - isScreenShareAudioEnabled?: boolean; -} export interface VoiceEngineV2AppParticipantTrackFlags { isMicrophoneEnabled?: boolean; @@ -56,7 +28,6 @@ export interface VoiceEngineV2AppParticipantAdapterOptions { getModel: () => VoiceEngineV2Model; ingest?: (event: VoiceEngineV2Event) => void; getCurrentConnectionId?: () => string | null | undefined; - now?: () => number; } export type VoiceEngineV2AppLivekitParticipantSnapshot = LivekitParticipantSnapshot & @@ -166,77 +137,18 @@ function updateTrackFlags( }; } -function withAudioLevelSpeaking( - participant: VoiceEngineV2AppLivekitParticipantSnapshot, - speaking: boolean, - nowMs: number, -): VoiceEngineV2AppLivekitParticipantSnapshot { - return { - ...participant, - isAudioLevelSpeaking: speaking, - lastSpokeAt: speaking ? nowMs : participant.lastSpokeAt, - }; -} - -function participantAttributesEqual( - next: Readonly>, - previous: Readonly>, -): boolean { - assertNonNullObject(next, 'next'); - assertNonNullObject(previous, 'previous'); - const nextKeys = Object.keys(next); - if (nextKeys.length !== Object.keys(previous).length) return false; - for (const key of nextKeys) { - if (next[key] !== previous[key]) return false; - } - return true; -} - -function nativeFieldsMatchExistingSnapshot( - fields: VoiceEngineV2AppNativeParticipantFields, - existing: VoiceEngineV2AppLivekitParticipantSnapshot, -): boolean { - assertNonNullObject(fields, 'fields'); - assertNonNullObject(existing, 'existing'); - if (existing.joinedAt == null) return false; - if ((fields.name ?? existing.name ?? '') !== existing.name) return false; - if ((fields.sid || existing.sid || '') !== existing.sid) return false; - if ((fields.isLocal ?? existing.isLocal ?? false) !== existing.isLocal) return false; - if ((fields.metadata ?? existing.metadata) !== existing.metadata) return false; - if ((fields.isMicrophoneEnabled ?? existing.isMicrophoneEnabled ?? false) !== existing.isMicrophoneEnabled) { - return false; - } - if ((fields.isCameraEnabled ?? existing.isCameraEnabled ?? false) !== existing.isCameraEnabled) { - return false; - } - if ((fields.isScreenShareEnabled ?? existing.isScreenShareEnabled ?? false) !== existing.isScreenShareEnabled) { - return false; - } - if ( - (fields.isScreenShareAudioEnabled ?? existing.isScreenShareAudioEnabled ?? false) !== - existing.isScreenShareAudioEnabled - ) { - return false; - } - if (fields.attributes === undefined) return true; - return participantAttributesEqual(fields.attributes, existing.attributes ?? {}); -} - export class VoiceEngineV2AppParticipantAdapter { private readonly controller: VoiceEngineV2Controller; private readonly getModel: () => VoiceEngineV2Model; private readonly ingest: (event: VoiceEngineV2Event) => void; private readonly getCurrentConnectionId: () => string | null | undefined; - private readonly now: () => number; private readonly discardedConnectionIds = new Set(); - private readonly nativeSpeakingHeartbeatsMs = new Map(); constructor(options: VoiceEngineV2AppParticipantAdapterOptions) { this.controller = options.controller; this.getModel = options.getModel; this.ingest = options.ingest ?? ((event) => this.controller.dispatch(event)); this.getCurrentConnectionId = options.getCurrentConnectionId ?? (() => null); - this.now = options.now ?? (() => Date.now()); } get participants(): Readonly> { @@ -268,7 +180,6 @@ export class VoiceEngineV2AppParticipantAdapter { const participant = participants[identity]; if (!participant) continue; if (participant.connectionId === connectionId) { - this.nativeSpeakingHeartbeatsMs.delete(participant.identity); this.ingest({type: 'room.participantLeft', participantIdentity: participant.identity}); } } @@ -314,39 +225,6 @@ export class VoiceEngineV2AppParticipantAdapter { } } - upsertParticipantFromNative(fields: VoiceEngineV2AppNativeParticipantFields): void { - assertNonNullObject(fields, 'fields'); - assertString(fields.identity, 'fields.identity'); - if (this.isConnectionDiscarded(extractParticipantConnectionId(fields.identity))) return; - const existing = this.participants[fields.identity]; - if (existing && nativeFieldsMatchExistingSnapshot(fields, existing)) return; - const participant: VoiceEngineV2AppLivekitParticipantSnapshot = { - identity: fields.identity, - name: fields.name ?? existing?.name ?? '', - userId: extractParticipantUserId(fields.identity), - connectionId: extractParticipantConnectionId(fields.identity), - sid: fields.sid || existing?.sid || '', - isLocal: fields.isLocal ?? existing?.isLocal ?? false, - isSpeaking: existing?.isSpeaking ?? false, - isAudioLevelSpeaking: existing?.isAudioLevelSpeaking ?? false, - connectionQuality: existing?.connectionQuality ?? VoiceConnectionQualityValue.Unknown, - metadata: fields.metadata ?? existing?.metadata, - attributes: - fields.attributes === undefined - ? (existing?.attributes ?? Object.freeze({})) - : Object.freeze({...fields.attributes}), - audioTrackSids: existing?.audioTrackSids ?? Object.freeze([]), - videoTrackSids: existing?.videoTrackSids ?? Object.freeze([]), - isMicrophoneEnabled: fields.isMicrophoneEnabled ?? existing?.isMicrophoneEnabled ?? false, - isCameraEnabled: fields.isCameraEnabled ?? existing?.isCameraEnabled ?? false, - isScreenShareEnabled: fields.isScreenShareEnabled ?? existing?.isScreenShareEnabled ?? false, - isScreenShareAudioEnabled: fields.isScreenShareAudioEnabled ?? existing?.isScreenShareAudioEnabled ?? false, - joinedAt: existing?.joinedAt ?? this.now(), - lastSpokeAt: existing?.lastSpokeAt ?? null, - }; - this.upsertSnapshot(participant); - } - patchParticipantTrackFlags(identity: string, flags: VoiceEngineV2AppParticipantTrackFlags): void { assertString(identity, 'identity'); assertNonNullObject(flags, 'flags'); @@ -356,14 +234,6 @@ export class VoiceEngineV2AppParticipantAdapter { this.upsertSnapshot(updateTrackFlags(participant, flags)); } - setConnectionQualityForNative(sid: string, quality: VoiceConnectionQuality): void { - assertString(sid, 'sid'); - const participant = this.getParticipantBySid(sid); - if (!participant) return; - if (participant.connectionQuality === quality) return; - this.upsertSnapshot({...participant, connectionQuality: quality}); - } - updateActiveSpeakersBySid(sids: ReadonlyArray): void { assert.ok(Array.isArray(sids), 'sids must be array'); const speakerSids = new Set(sids); @@ -377,64 +247,8 @@ export class VoiceEngineV2AppParticipantAdapter { } } - applyNativeSpeakingSample(sample: VoiceEngineV2AppNativeSpeakingSample, nowMs: number = this.now()): void { - assertNonNullObject(sample, 'sample'); - assertString(sample.identity, 'sample.identity'); - assert.equal(sample.source, 'microphone', 'native speaking sample must come from a microphone track'); - assert.equal(typeof sample.speaking, 'boolean', 'sample.speaking must be a boolean'); - assert.ok(Number.isFinite(nowMs), 'nowMs must be finite'); - const participant = this.resolveNativeSpeakingParticipant(sample); - if (!participant) return; - if (this.isConnectionDiscarded(participant.connectionId)) return; - if (sample.speaking) { - assert.ok( - this.nativeSpeakingHeartbeatsMs.size <= MAX_NATIVE_SPEAKING_HEARTBEATS, - `nativeSpeakingHeartbeatsMs exceeded cap=${MAX_NATIVE_SPEAKING_HEARTBEATS}`, - ); - this.nativeSpeakingHeartbeatsMs.set(participant.identity, nowMs); - } else { - this.nativeSpeakingHeartbeatsMs.delete(participant.identity); - } - if (participant.isAudioLevelSpeaking === sample.speaking) return; - this.upsertSnapshot(withAudioLevelSpeaking(participant, sample.speaking, nowMs)); - } - - sweepNativeSpeakingHeartbeats(nowMs: number = this.now()): void { - assert.ok(Number.isFinite(nowMs), 'nowMs must be finite'); - assert.ok( - this.nativeSpeakingHeartbeatsMs.size <= MAX_NATIVE_SPEAKING_HEARTBEATS, - `nativeSpeakingHeartbeatsMs exceeded cap=${MAX_NATIVE_SPEAKING_HEARTBEATS}`, - ); - const expired: Array = []; - for (const [identity, heartbeatMs] of this.nativeSpeakingHeartbeatsMs) { - if (nowMs - heartbeatMs >= NATIVE_SPEAKING_HEARTBEAT_TIMEOUT_MS) { - expired.push(identity); - } - } - for (const identity of expired) { - this.nativeSpeakingHeartbeatsMs.delete(identity); - const participant = this.participants[identity]; - if (!participant) continue; - if (!participant.isAudioLevelSpeaking) continue; - this.upsertSnapshot(withAudioLevelSpeaking(participant, false, nowMs)); - } - } - - private resolveNativeSpeakingParticipant( - sample: VoiceEngineV2AppNativeSpeakingSample, - ): VoiceEngineV2AppLivekitParticipantSnapshot | undefined { - assertNonNullObject(sample, 'sample'); - const byIdentity = this.participants[sample.identity]; - if (byIdentity) return byIdentity; - if (sample.participantSid.length > 0) { - return this.getParticipantBySid(sample.participantSid); - } - return undefined; - } - removeParticipant(identity: string): void { assertString(identity, 'identity'); - this.nativeSpeakingHeartbeatsMs.delete(identity); this.ingest({type: 'room.participantLeft', participantIdentity: identity}); } @@ -503,7 +317,6 @@ export class VoiceEngineV2AppParticipantAdapter { } clear(): void { - this.nativeSpeakingHeartbeatsMs.clear(); const participants = this.participants; for (const identity in participants) { const participant = participants[identity]; diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppPermissionAdapter.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppPermissionAdapter.ts index c9ebfaad4..c55f0296c 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppPermissionAdapter.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppPermissionAdapter.ts @@ -34,12 +34,6 @@ const logger = new Logger('VoiceEngineV2AppPermissionAdapter'); export type VoiceEngineV2AppPermissionTrackSource = 'audio' | 'video' | 'screenShare'; -export interface VoiceEngineV2AppNativePermissionEnforcement { - revokeMicrophone: () => Promise; - revokeCamera: () => Promise; - revokeScreenShare: () => Promise; -} - type RemotePublicationAdapter = { isDesired?: boolean; setEnabled?: (enabled: boolean) => void; @@ -49,11 +43,10 @@ type RemotePublicationAdapter = { class VoiceEngineV2AppPermissionAdapter extends Store { private snapshot: VoicePermissionSnapshot = createVoicePermissionSnapshot(); private currentRoom: Room | null = null; - private nativeEnforcement: VoiceEngineV2AppNativePermissionEnforcement | null = null; private permissionDisposer: (() => void) | null = null; private hasEnforceableMediaSession(): boolean { - return this.currentRoom != null || this.nativeEnforcement != null; + return this.currentRoom != null; } private sendPermissionEvent( @@ -131,11 +124,6 @@ class VoiceEngineV2AppPermissionAdapter extends Store { } private revokePermissionForRoom(source: VoiceEngineV2AppPermissionTrackSource, room: Room | null | undefined): void { - const nativeEnforcement = this.nativeEnforcement; - if (nativeEnforcement) { - void this.handleNativePermissionRevoked(source, nativeEnforcement); - return; - } if (!room) { logger.debug('No active room, skipping permission enforcement', {source}); return; @@ -143,31 +131,6 @@ class VoiceEngineV2AppPermissionAdapter extends Store { void this.handlePermissionRevoked(source, room); } - private async handleNativePermissionRevoked( - source: VoiceEngineV2AppPermissionTrackSource, - enforcement: VoiceEngineV2AppNativePermissionEnforcement, - ): Promise { - assert.ok(source === 'audio' || source === 'video' || source === 'screenShare', 'unknown permission track source'); - assert.equal(enforcement, this.nativeEnforcement, 'native permission revocation requires the active enforcement'); - logger.info('Revoking permission via native session', {source}); - try { - switch (source) { - case 'audio': - await enforcement.revokeMicrophone(); - break; - case 'video': - await enforcement.revokeCamera(); - break; - case 'screenShare': - await enforcement.revokeScreenShare(); - break; - } - logger.info('Successfully revoked permission via native session', {source}); - } catch (error) { - logger.error('Failed to revoke permission via native session', {source, error}); - } - } - private buildRemotePublicationInputs(room: Room): { microphonePublications: Array; videoPublicationIds: Array; @@ -203,30 +166,11 @@ class VoiceEngineV2AppPermissionAdapter extends Store { this.sendPermissionEvent({type: 'permission.watch.stop'}); this.update(() => { this.currentRoom = room; - this.nativeEnforcement = null; }); this.beginPermissionWatch(guildId, channelId); logger.info('Started permission watching', {guildId, channelId}); } - syncWithNativePermissionState( - guildId: string | null, - channelId: string, - enforcement: VoiceEngineV2AppNativePermissionEnforcement, - ): void { - assert.ok(channelId.length > 0, 'native permission watch requires a channelId'); - assert.equal(typeof enforcement.revokeMicrophone, 'function', 'native enforcement requires revokeMicrophone'); - assert.equal(typeof enforcement.revokeCamera, 'function', 'native enforcement requires revokeCamera'); - assert.equal(typeof enforcement.revokeScreenShare, 'function', 'native enforcement requires revokeScreenShare'); - this.sendPermissionEvent({type: 'permission.watch.stop'}); - this.update(() => { - this.currentRoom = null; - this.nativeEnforcement = enforcement; - }); - this.beginPermissionWatch(guildId, channelId); - logger.info('Started native permission watching', {guildId, channelId}); - } - private beginPermissionWatch(guildId: string | null, channelId: string): void { assert.equal(this.permissionDisposer, null, 'permission watch must be stopped before starting a new watch'); assert.ok(this.hasEnforceableMediaSession(), 'permission watch requires an enforceable media session'); @@ -272,7 +216,7 @@ class VoiceEngineV2AppPermissionAdapter extends Store { handlePermissionChange(permission: 'speak' | 'stream' | 'video', allowed: boolean): void { const room = this.currentRoom; - if (!room && !this.nativeEnforcement) { + if (!room) { logger.warn('No active media session'); return; } @@ -426,7 +370,6 @@ class VoiceEngineV2AppPermissionAdapter extends Store { this.sendPermissionEvent({type: 'permission.reset'}); this.update(() => { this.currentRoom = null; - this.nativeEnforcement = null; }); logger.debug('Permissions reset to defaults'); } diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppProductionHostPorts.test.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppProductionHostPorts.test.ts index 7d1f95b4e..7bd9eea92 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppProductionHostPorts.test.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppProductionHostPorts.test.ts @@ -2,11 +2,9 @@ import { availableVoiceEngineV2Capabilities, - type LiveKitMediaPort, type NativeMediaPort, type StatsPort, type SubscriptionPort, - type VoiceEngineV2CameraEncodingOptions, type VoiceEngineV2CameraOptions, type VoiceEngineV2Event, type VoiceEngineV2MicrophoneOptions, @@ -31,11 +29,6 @@ type ProductionCall = type NativeMediaCall = | {type: 'startCapture'; captureId: string; zeroCopyRequired: boolean} | {type: 'attachFrameSink'; captureId: string; sinkId: string; zeroCopyRequired: boolean}; -type NativeVoiceMediaCall = - | {type: 'connect'; url: string} - | {type: 'publishCamera'; options: VoiceEngineV2CameraOptions} - | {type: 'updateCameraEncoding'; options: VoiceEngineV2CameraEncodingOptions} - | {type: 'unpublishCamera'; options?: VoiceEngineV2CameraOptions}; function createStatsPort(): StatsPort { return { @@ -77,35 +70,6 @@ function createNativeMediaPort(calls: Array): NativeMediaPort { }; } -function createNativeVoiceMediaPort(calls: Array): LiveKitMediaPort { - return { - async prewarm(): Promise {}, - async connect(options): Promise { - calls.push({type: 'connect', url: options.url}); - }, - async disconnect(): Promise {}, - async publishMicrophone(): Promise {}, - async unpublishMicrophone(): Promise {}, - async setMicrophoneEnabled(): Promise {}, - async publishCamera(options): Promise { - calls.push({type: 'publishCamera', options}); - }, - async updateCameraEncoding(options): Promise { - calls.push({type: 'updateCameraEncoding', options}); - }, - async unpublishCamera(options): Promise { - calls.push({type: 'unpublishCamera', options}); - }, - async publishScreen(): Promise {}, - async updateScreenEncoding(): Promise {}, - async unpublishScreen(): Promise {}, - async publishScreenAudio(): Promise {}, - async unpublishScreenAudio(): Promise {}, - async setOutputDevice(): Promise {}, - async publishData(): Promise {}, - }; -} - function createLogger() { return { trace(): void {}, @@ -247,68 +211,6 @@ describe('VoiceEngineV2AppProductionHostPorts', () => { host.dispose(); }); - it('routes native-selected camera commands through native media without touching JS camera delegates', async () => { - const productionCalls: Array = []; - const nativeVoiceMediaCalls: Array = []; - const host = createVoiceEngineV2AppTestControllerHost({ - ports: createVoiceEngineV2AppProductionHostPorts({ - gateway: { - async writeVoiceState(): Promise {}, - async clearVoiceState(): Promise {}, - }, - connection: { - startConnection(guildId, channelId): boolean { - productionCalls.push({type: 'connect', guildId, channelId}); - return true; - }, - }, - media: { - async enableMicrophone(): Promise {}, - async disableMicrophone(): Promise {}, - async setMicrophoneEnabled(): Promise {}, - async setCameraEnabled(): Promise<'applied'> { - throw new Error('JS camera delegate must not be called in native mode'); - }, - async updateCameraEncoding(): Promise {}, - }, - screenShare: { - async publishControllerScreenViaLiveKitFlows(): Promise {}, - async unpublishControllerScreenViaLiveKitFlows(): Promise {}, - async updateActiveScreenShareSettings(): Promise { - return true; - }, - setScreenShareAudioMuted(): void {}, - }, - getRoom: () => ({}) as Room, - getActiveGuildId: () => 'guild-1', - getActiveChannelId: () => 'channel-1', - stats: createStatsPort(), - subscriptions: createSubscriptionPort(), - audioOutputStore: { - async setOutputDevice(): Promise {}, - }, - nativeVoiceMedia: createNativeVoiceMediaPort(nativeVoiceMediaCalls), - getSelectedMediaMode: () => 'native', - logger: createLogger(), - }), - }); - - host.controller.connect({url: 'wss://voice.example.test', token: 'token'}); - await waitForRuntime(); - host.controller.publishCamera({deviceId: 'native-cam-1'}); - await waitForRuntime(); - host.controller.unpublishCamera({sendUpdate: false}); - await waitForRuntime(); - - expect(nativeVoiceMediaCalls).toEqual([ - {type: 'connect', url: 'wss://voice.example.test'}, - {type: 'publishCamera', options: {deviceId: 'native-cam-1'}}, - {type: 'unpublishCamera', options: {sendUpdate: false}}, - ]); - expect(productionCalls).toEqual([]); - host.dispose(); - }); - it('routes native capture and frame sinks through production native-media port', async () => { const nativeMediaCalls: Array = []; const shadowCalls: Array = []; diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppProductionHostPorts.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppProductionHostPorts.ts index 6116abfc3..d403e6cf2 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppProductionHostPorts.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppProductionHostPorts.ts @@ -3,7 +3,6 @@ import type { DevicePort, GatewayPort, - LiveKitMediaPort, NativeMediaPort, StatsPort, SubscriptionPort, @@ -30,10 +29,6 @@ import { type VoiceEngineV2AppLiveKitMediaDelegate, type VoiceEngineV2AppLiveKitScreenShareDelegate, } from './VoiceEngineV2AppLiveKitExecutionAdapter'; -import { - VoiceEngineV2AppSelectedMediaExecutionAdapter, - type VoiceEngineV2AppSelectedMediaMode, -} from './VoiceEngineV2AppSelectedMediaExecutionAdapter'; import {createVoiceEngineV2AppTimerAdapter, type VoiceEngineV2AppTimerScheduler} from './VoiceEngineV2AppTimerAdapter'; export interface VoiceEngineV2AppProductionHostPortsLogger { @@ -59,8 +54,6 @@ export interface VoiceEngineV2AppProductionHostPortsOptions { clock?: VoiceEngineV2ClockPort; devices?: DevicePort; nativeMedia?: NativeMediaPort; - nativeVoiceMedia?: LiveKitMediaPort; - getSelectedMediaMode?: () => VoiceEngineV2AppSelectedMediaMode; ingestion?: VoiceEngineV2AppIngestionPort; passthrough?: VoiceEngineV2HostPorts; lifecycle?: VoiceEngineV2AppLifecycleAdapter; @@ -84,15 +77,7 @@ export function createVoiceEngineV2AppProductionHostPorts( getActiveGuildId: options.getActiveGuildId, getActiveChannelId: options.getActiveChannelId, }); - const media = - options.nativeVoiceMedia && options.getSelectedMediaMode - ? new VoiceEngineV2AppSelectedMediaExecutionAdapter({ - jsMedia: liveKit, - nativeMedia: options.nativeVoiceMedia, - getMode: options.getSelectedMediaMode, - logger: options.logger, - }) - : liveKit; + const media = liveKit; const ingestion = options.ingestion ?? createVoiceEngineV2AppIngestionPort(); const lifecycle = options.lifecycle ?? diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump.ts index 800d8c354..87d854bee 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump.ts @@ -1,113 +1,5 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import assert from 'node:assert/strict'; -import {getEffectiveAudioState} from '@app/features/voice/engine/VoiceEffectiveAudioState'; -import {requireVoiceEngineV2AppNativeBridge} from '@app/features/voice/engine/v2/VoiceEngineV2AppNativeBridge'; -import type {VoiceEngineV2AppScreenShareExecutionAdapter} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter'; -import type {NativeScreenShareOptions} from '@app/features/voice/engine/voice_screen_share_manager/DisplayMediaCapture'; -import { - type NativeEngineAudioTrackEndMessage, - type NativeEngineAudioTrackFrame, - startNativeEngineAudioTrackFramePump, -} from '@app/features/voice/engine/voice_screen_share_manager/NativeEngineAudioTrackPump'; -import {logger} from '@app/features/voice/engine/voice_screen_share_manager/shared'; -import { - type NativeAudioFramePump, - type NativeAudioFramePumpSource, - startLinuxNativeAudioFramePump, - startNativeAudioFramePump, -} from '@app/features/voice/utils/NativeAudioCaptureBridge'; -import { - MAX_NATIVE_AUDIO_CHANNELS, - MAX_NATIVE_AUDIO_FRAME_SECONDS, - MAX_NATIVE_AUDIO_SAMPLE_RATE, -} from '@app/features/voice/utils/native_audio_capture_bridge/shared'; -import type {VoiceEngineV2BridgeApi} from '@fluxer/voice_engine_v2/bridge'; - -const MAX_SCREEN_SHARE_AUDIO_PUMP_FLOAT_SAMPLES = - MAX_NATIVE_AUDIO_SAMPLE_RATE * MAX_NATIVE_AUDIO_CHANNELS * MAX_NATIVE_AUDIO_FRAME_SECONDS; - -export const AUDIO_PUSH_QUEUE_FRAMES_MAX = 32; -export const AUDIO_PUSH_QUEUE_DROP_LOG_INTERVAL = 50; - -interface PendingAudioPushWork { - run: () => Promise; - drop: () => void; -} - -export class BoundedAudioPushQueue { - private readonly pending: Array; - private pendingHead = 0; - private pendingTail = 0; - private pendingCount = 0; - private draining = false; - private droppedFrameCount = 0; - private readonly onError: (error: unknown) => void; - private readonly onDrop: (droppedFrameCount: number) => void; - - constructor(onError: (error: unknown) => void, onDrop: (droppedFrameCount: number) => void) { - assert.ok(AUDIO_PUSH_QUEUE_FRAMES_MAX > 0, 'audio push queue capacity must be positive'); - assert.equal(typeof onError, 'function'); - assert.equal(typeof onDrop, 'function'); - this.pending = new Array(AUDIO_PUSH_QUEUE_FRAMES_MAX).fill(null); - this.onError = onError; - this.onDrop = onDrop; - } - - get droppedFrames(): number { - return this.droppedFrameCount; - } - - get pendingFrames(): number { - return this.pendingCount; - } - - enqueue(work: () => Promise, drop: () => void = () => undefined): void { - assert.equal(typeof work, 'function'); - assert.equal(typeof drop, 'function'); - if (this.pendingCount >= AUDIO_PUSH_QUEUE_FRAMES_MAX) { - const dropped = this.pending[this.pendingHead]; - this.pending[this.pendingHead] = null; - this.pendingHead = (this.pendingHead + 1) % AUDIO_PUSH_QUEUE_FRAMES_MAX; - this.pendingCount -= 1; - if (dropped) { - try { - dropped.drop(); - } catch (error) { - this.onError(error); - } - } - this.droppedFrameCount += 1; - this.onDrop(this.droppedFrameCount); - } - this.pending[this.pendingTail] = {run: work, drop}; - this.pendingTail = (this.pendingTail + 1) % AUDIO_PUSH_QUEUE_FRAMES_MAX; - this.pendingCount += 1; - assert.ok(this.pendingCount <= AUDIO_PUSH_QUEUE_FRAMES_MAX, 'audio push queue exceeded its cap'); - void this.drain(); - } - - private async drain(): Promise { - if (this.draining) return; - this.draining = true; - while (this.pendingCount > 0) { - const work = this.pending[this.pendingHead]; - this.pending[this.pendingHead] = null; - this.pendingHead = (this.pendingHead + 1) % AUDIO_PUSH_QUEUE_FRAMES_MAX; - this.pendingCount -= 1; - if (!work) continue; - try { - await work.run(); - } catch (error) { - this.onError(error); - } - } - assert.ok(this.pendingCount >= 0, 'audio push queue count underflowed'); - assert.ok(this.pendingCount <= AUDIO_PUSH_QUEUE_FRAMES_MAX, 'audio push queue count exceeded cap'); - this.draining = false; - } -} - export interface ScreenShareAudioPumpDiagnostics { active: boolean; captureId: string | null; @@ -122,7 +14,7 @@ export interface ScreenShareAudioPumpDiagnostics { pendingPushFrames: number; } -function createInitialScreenShareAudioPumpDiagnostics(): ScreenShareAudioPumpDiagnostics { +export function getScreenShareAudioPumpDiagnostics(): ScreenShareAudioPumpDiagnostics { return { active: false, captureId: null, @@ -137,390 +29,3 @@ function createInitialScreenShareAudioPumpDiagnostics(): ScreenShareAudioPumpDia pendingPushFrames: 0, }; } - -let screenShareAudioPumpDiagnostics: ScreenShareAudioPumpDiagnostics = createInitialScreenShareAudioPumpDiagnostics(); -let activeScreenShareAudioPushQueue: BoundedAudioPushQueue | null = null; - -export function getScreenShareAudioPumpDiagnostics(): ScreenShareAudioPumpDiagnostics { - const queue = activeScreenShareAudioPushQueue; - return { - ...screenShareAudioPumpDiagnostics, - droppedPushFrames: queue?.droppedFrames ?? screenShareAudioPumpDiagnostics.droppedPushFrames, - pendingPushFrames: queue?.pendingFrames ?? 0, - }; -} - -interface AudioPumpFrameContext { - readonly engine: VoiceEngineV2BridgeApi; - readonly sourceKind: string; - readonly sourceId: string; - getActivePump: () => NativeAudioFramePump | null; - getPublishedFormatKey: () => string | null; - setPublishedFormatKey: (key: string) => void; - enqueuePush: (work: () => Promise, drop?: () => void) => void; - scheduleStop: (reason: string, context?: Record) => void; -} - -function isValidFloatFrame(frame: NativeEngineAudioTrackFrame): boolean { - return ( - Number.isSafeInteger(frame.sampleRate) && - frame.sampleRate > 0 && - frame.sampleRate <= MAX_NATIVE_AUDIO_SAMPLE_RATE && - Number.isSafeInteger(frame.channels) && - frame.channels > 0 && - frame.channels <= MAX_NATIVE_AUDIO_CHANNELS && - frame.samples.length > 0 && - frame.samples.length <= MAX_SCREEN_SHARE_AUDIO_PUMP_FLOAT_SAMPLES && - frame.samples.length % frame.channels === 0 - ); -} - -interface AudioPumpStartInput { - audioTrack?: NativeScreenShareOptions['audioTrack']; - nativeAudioFramePump?: NativeAudioFramePumpSource; - linuxRule?: NativeScreenShareOptions['nativeAudioLinuxRule']; - sourceKind: string; - sourceId: string; -} - -export class VoiceEngineV2AppScreenShareAudioPump { - private readonly adapter: VoiceEngineV2AppScreenShareExecutionAdapter; - - constructor(adapter: VoiceEngineV2AppScreenShareExecutionAdapter) { - this.adapter = adapter; - } - - async stopAudio(stopLocalTrack = true, unpublishRemote = true): Promise { - assert.equal(typeof stopLocalTrack, 'boolean'); - assert.equal(typeof unpublishRemote, 'boolean'); - const pump = this.adapter.nativeEngineScreenShareAudioPump; - this.adapter.nativeEngineScreenShareAudioPump = null; - assert.equal(this.adapter.nativeEngineScreenShareAudioPump, null, 'audioPump must be cleared'); - screenShareAudioPumpDiagnostics = {...screenShareAudioPumpDiagnostics, active: false}; - activeScreenShareAudioPushQueue = null; - if (pump) { - await pump.cleanup(stopLocalTrack).catch((error) => { - logger.warn('Failed to stop native-engine screen-share audio pump', { - captureId: pump.captureId, - error, - }); - }); - } - if (!unpublishRemote) return; - try { - await requireVoiceEngineV2AppNativeBridge('unpublish native screen-share audio').unpublishScreenAudio(); - } catch (error) { - logger.warn('Failed to unpublish native-engine screen-share audio', {error}); - } - } - - private async reassertMicMute(engine: VoiceEngineV2BridgeApi, reason: string): Promise { - if (!getEffectiveAudioState().effectiveMute) return; - const result = await engine.setMicEnabled(false); - if (!result.ok) { - logger.warn('Failed to reassert native-engine microphone mute during screen-share audio publish', { - reason, - error: result.error, - }); - } - } - - private scheduleStopAfterAudioFailure( - state: {scheduled: boolean}, - sourceKind: string, - sourceId: string, - reason: string, - context: Record, - ): void { - assert.equal(typeof reason, 'string'); - assert.ok(state); - if (state.scheduled) return; - state.scheduled = true; - logger.warn('Native-engine screen-share audio failed after publish; stopping screen share', { - reason, - sourceKind, - sourceId, - ...context, - }); - void this.adapter.stopNativeEngineScreenShareInternal({playSound: true}).catch((error) => { - logger.warn('Failed to stop native-engine screen share after audio failure', { - reason, - sourceKind, - sourceId, - error, - }); - }); - } - - private async pushFrameWork( - ctx: AudioPumpFrameContext, - activePump: NativeAudioFramePump, - frame: NativeEngineAudioTrackFrame, - formatKey: string, - ): Promise { - assert.ok(activePump); - assert.equal(typeof formatKey, 'string'); - if (this.adapter.nativeEngineScreenShareAudioPump !== activePump) return; - if (ctx.getPublishedFormatKey() !== formatKey) { - await this.reassertMicMute(ctx.engine, 'before-format-republish'); - await ctx.engine.publishScreenAudio({sampleRate: frame.sampleRate, numChannels: frame.channels}); - await this.reassertMicMute(ctx.engine, 'after-format-republish'); - ctx.setPublishedFormatKey(formatKey); - screenShareAudioPumpDiagnostics.publishStrategy = 'lazy'; - screenShareAudioPumpDiagnostics.publishedFormatKey = formatKey; - } - const accepted = await ctx.engine.pushScreenAudioFloat({ - sampleRate: frame.sampleRate, - numChannels: frame.channels, - samples: frame.samples, - }); - if (this.adapter.nativeEngineScreenShareAudioPump !== activePump) return; - if (accepted) return; - logger.warn('Native-engine screen-share audio frame was rejected', { - captureId: activePump.captureId, - sampleRate: frame.sampleRate, - channels: frame.channels, - }); - ctx.scheduleStop('float-frame-rejected', { - captureId: activePump.captureId, - sampleRate: frame.sampleRate, - channels: frame.channels, - }); - } - - private makeFrameHandler(ctx: AudioPumpFrameContext): (frame: NativeEngineAudioTrackFrame) => void { - assert.ok(ctx); - assert.equal(typeof ctx.getActivePump, 'function'); - return (frame) => { - const activePump = ctx.getActivePump(); - const release = frame.release ?? (() => undefined); - if (!activePump) { - release(); - return; - } - if (!isValidFloatFrame(frame)) { - release(); - logger.warn('Dropping invalid native-engine screen-share audio frame', { - captureId: activePump.captureId, - sampleRate: frame.sampleRate, - channels: frame.channels, - samples: frame.samples.length, - }); - return; - } - const formatKey = `${frame.sampleRate}:${frame.channels}`; - ctx.enqueuePush(async () => { - try { - await this.pushFrameWork(ctx, activePump, frame, formatKey); - } finally { - release(); - } - }, release); - }; - } - - private makeCaptureEndedHandler(ctx: AudioPumpFrameContext): (message: NativeEngineAudioTrackEndMessage) => void { - assert.ok(ctx); - assert.equal(typeof ctx.scheduleStop, 'function'); - return (message) => { - if (this.adapter.nativeEngineScreenShareAudioPump?.captureId !== message.captureId) return; - logger.info('Native-engine screen-share audio capture ended', { - captureId: message.captureId, - reason: message.reason, - detail: message.detail, - }); - ctx.scheduleStop('capture-ended', { - captureId: message.captureId, - endReason: message.reason, - endDetail: message.detail, - }); - }; - } - - private async startUnderlyingPump( - input: AudioPumpStartInput, - onFrame: (frame: NativeEngineAudioTrackFrame) => void, - onEnded: (message: NativeEngineAudioTrackEndMessage) => void, - ): Promise { - assert.ok(input); - assert.equal(typeof onFrame, 'function'); - const linuxRule = input.linuxRule; - const nativeAudioFramePump = input.nativeAudioFramePump; - const audioTrack = input.audioTrack; - if (nativeAudioFramePump) { - return startNativeAudioFramePump( - nativeAudioFramePump, - (message) => { - onFrame({ - sampleRate: message.sampleRate, - channels: message.channels, - samples: new Float32Array(message.samples, 0, message.samples.byteLength / Float32Array.BYTES_PER_ELEMENT), - }); - }, - onEnded, - ); - } - if (linuxRule) { - return startLinuxNativeAudioFramePump( - linuxRule, - (message) => { - onFrame({ - sampleRate: message.sampleRate, - channels: message.channels, - samples: new Float32Array(message.samples, 0, message.samples.byteLength / Float32Array.BYTES_PER_ELEMENT), - }); - }, - onEnded, - ); - } - assert.ok(audioTrack); - return startNativeEngineAudioTrackFramePump(audioTrack, onFrame, onEnded); - } - - async startAudio(nativeOptions: NativeScreenShareOptions): Promise { - assert.ok(nativeOptions); - return this.startAudioInternal({ - audioTrack: nativeOptions.audioTrack, - nativeAudioFramePump: nativeOptions.nativeAudioFramePump, - linuxRule: nativeOptions.nativeAudioLinuxRule, - sourceKind: nativeOptions.source.kind, - sourceId: nativeOptions.source.id, - }); - } - - async startAudioFromTrack(audioTrack: MediaStreamTrack, sourceId: string): Promise { - assert.ok(audioTrack); - assert.equal(typeof sourceId, 'string'); - assert.ok(sourceId.length > 0, 'device audio sourceId must be non-empty'); - return this.startAudioInternal({audioTrack, sourceKind: 'device', sourceId}); - } - - private async startAudioInternal(input: AudioPumpStartInput): Promise { - assert.ok(input); - const linuxRule = input.linuxRule; - const nativeAudioFramePump = input.nativeAudioFramePump; - const audioTrack = input.audioTrack; - if (!nativeAudioFramePump && !linuxRule && !audioTrack) return false; - await this.stopAudio(); - screenShareAudioPumpDiagnostics = createInitialScreenShareAudioPumpDiagnostics(); - const engine = requireVoiceEngineV2AppNativeBridge('publish native screen-share audio'); - const sourceKind = input.sourceKind; - const sourceId = input.sourceId; - const stopState = {scheduled: false}; - let pump: NativeAudioFramePump | null = null; - let publishedFormatKey: string | null = null; - const pushQueue = new BoundedAudioPushQueue( - (error) => { - logger.warn('Failed to push native-engine screen-share audio frame', { - captureId: pump?.captureId, - error, - }); - ctx.scheduleStop('float-frame-push-failed', {captureId: pump?.captureId, error}); - }, - (droppedFrameCount) => { - if (droppedFrameCount % AUDIO_PUSH_QUEUE_DROP_LOG_INTERVAL !== 1) return; - logger.warn('Native-engine screen-share audio push queue is full; dropping oldest frame', { - captureId: pump?.captureId, - droppedFrameCount, - queueFramesMax: AUDIO_PUSH_QUEUE_FRAMES_MAX, - }); - }, - ); - activeScreenShareAudioPushQueue = pushQueue; - const ctx: AudioPumpFrameContext = { - engine, - sourceKind, - sourceId, - getActivePump: () => pump, - getPublishedFormatKey: () => publishedFormatKey, - setPublishedFormatKey: (key) => { - publishedFormatKey = key; - }, - enqueuePush: (work, drop) => { - pushQueue.enqueue(work, drop); - }, - scheduleStop: (reason, context = {}) => { - this.scheduleStopAfterAudioFailure(stopState, sourceKind, sourceId, reason, context); - }, - }; - const nextPump = await this.startUnderlyingPump( - input, - this.makeFrameHandler(ctx), - this.makeCaptureEndedHandler(ctx), - ); - if (!nextPump) { - activeScreenShareAudioPushQueue = null; - logger.warn( - 'Native-engine screen-share audio pump unavailable; aborting screen share because audio was requested', - {sourceKind, sourceId, hasLinuxRule: Boolean(linuxRule), hasAudioTrack: Boolean(audioTrack)}, - ); - throw new Error('Native-engine screen-share audio pump unavailable'); - } - pump = nextPump; - this.adapter.nativeEngineScreenShareAudioPump = pump; - const usesNativeSink = Boolean(nativeAudioFramePump) || Boolean(linuxRule); - await this.afterPumpStarted(ctx, engine, pump, usesNativeSink); - logger.info('Native-engine screen-share audio pump started', {captureId: pump.captureId}); - return this.adapter.nativeEngineScreenShareAudioPump === pump; - } - - private async afterPumpStarted( - ctx: AudioPumpFrameContext, - engine: VoiceEngineV2BridgeApi, - pump: NativeAudioFramePump, - usesNativeSink: boolean, - ): Promise { - assert.ok(pump); - assert.equal(typeof usesNativeSink, 'boolean'); - screenShareAudioPumpDiagnostics = { - ...screenShareAudioPumpDiagnostics, - active: true, - captureId: pump.captureId, - sampleRate: pump.sampleRate, - channels: pump.channels, - usesNativeSink, - }; - const formatKnown = pump.sampleRate > 0 && pump.channels > 0; - if (!usesNativeSink || !formatKnown) { - screenShareAudioPumpDiagnostics.eagerPublish = 'skipped'; - return; - } - await this.eagerlyPublishScreenAudio(ctx, engine, pump); - } - - private async eagerlyPublishScreenAudio( - ctx: AudioPumpFrameContext, - engine: VoiceEngineV2BridgeApi, - pump: NativeAudioFramePump, - ): Promise { - assert.ok(pump.sampleRate > 0, 'eager publish requires a positive sampleRate'); - assert.ok(pump.channels > 0, 'eager publish requires a positive channel count'); - const formatKey = `${pump.sampleRate}:${pump.channels}`; - try { - await this.reassertMicMute(engine, 'before-eager-publish'); - await engine.publishScreenAudio({sampleRate: pump.sampleRate, numChannels: pump.channels}); - await this.reassertMicMute(engine, 'after-eager-publish'); - } catch (error) { - screenShareAudioPumpDiagnostics.eagerPublish = 'failed'; - screenShareAudioPumpDiagnostics.eagerPublishError = error instanceof Error ? error.message : String(error); - logger.warn('Failed to eagerly publish native-engine screen-share audio; native sink has no track to feed', { - captureId: pump.captureId, - sampleRate: pump.sampleRate, - channels: pump.channels, - error, - }); - return; - } - if (this.adapter.nativeEngineScreenShareAudioPump !== pump) return; - ctx.setPublishedFormatKey(formatKey); - screenShareAudioPumpDiagnostics.publishStrategy = 'eager'; - screenShareAudioPumpDiagnostics.publishedFormatKey = formatKey; - screenShareAudioPumpDiagnostics.eagerPublish = 'succeeded'; - screenShareAudioPumpDiagnostics.eagerPublishError = null; - logger.info('Eagerly published native-engine screen-share audio track for native sink', { - captureId: pump.captureId, - sampleRate: pump.sampleRate, - channels: pump.channels, - }); - } -} diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareCaptureCoordinator.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareCaptureCoordinator.ts deleted file mode 100644 index 38427774f..000000000 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareCaptureCoordinator.ts +++ /dev/null @@ -1,1695 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; -import i18n from '@app/app/I18n'; -import {SoundType} from '@app/features/notification/utils/SoundUtils'; -import * as SoundCommands from '@app/features/ui/commands/SoundCommands'; -import {getElectronAPI} from '@app/features/ui/utils/NativeUtils'; -import AdaptiveScreenShareEngine from '@app/features/voice/engine/AdaptiveScreenShareEngine'; -import type {NativeVoiceEngineLocalTrackPublication} from '@app/features/voice/engine/native_voice_engine/nativeVoiceEngineEventMapper'; -import { - markScreenShareCaptureActive, - markScreenShareCaptureEnded, -} from '@app/features/voice/engine/ScreenShareCaptureDiagnostics'; -import type {VoiceScreenShareSourceType} from '@app/features/voice/engine/VoiceScreenShareStateMachine'; -import { - isVoiceEngineV2AppNativeScreenShareBridgeAvailable, - isVoiceEngineV2AppNativeScreenShareEncodingUpdateAvailable, - requireVoiceEngineV2AppNativeBridge, -} from '@app/features/voice/engine/v2/VoiceEngineV2AppNativeBridge'; -import type {VoiceEngineV2AppScreenShareSetEnabledOptions} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareControllerRouting'; -import type {VoiceEngineV2AppScreenShareExecutionAdapter} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter'; -import { - guardScreenShareEntry, - SCREEN_SHARE_SOURCE_SWITCH_UNSUPPORTED_PLATFORM_WARNING, - SCREEN_SHARE_UNSUPPORTED_PLATFORM_WARNING, -} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareGuards'; -import { - getDeviceScreenSharePublishDimensions, - isScreenShareVideoCodecValue, - isUserCancelledOrPermissionDeniedError, -} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareNativePublishOptions'; -import { - abortScreenShareRestartMigration, - announceScreenShareRestartMigration, - commitScreenShareRestartMigration, - type ScreenShareRestartMigrationSession, -} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareRestartMigration'; -import { - applyScreenShareState, - buildScreenShareFailureTransition, - runScreenShareActivationRitual, -} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareRituals'; -import { - type NativeEngineScreenCapture, - type NativeScreenShareOptions, - startNativeCaptureForEngine, - stopNativeCaptureForEngine, -} from '@app/features/voice/engine/voice_screen_share_manager/DisplayMediaCapture'; -import { - ensureNativeCameraPermissionForDeviceShare, - ensureNativeMicrophonePermissionForDeviceShare, -} from '@app/features/voice/engine/voice_screen_share_manager/NativePermissionGate'; -import { - type DeviceScreenShareCaptureOptions, - logger, -} from '@app/features/voice/engine/voice_screen_share_manager/shared'; -import ActiveScreenShareSource from '@app/features/voice/state/ActiveScreenShareSource'; -import LocalVoiceState from '@app/features/voice/state/LocalVoiceState'; -import VoiceSettings from '@app/features/voice/state/VoiceSettings'; -import {prepareHighFidelityScreenShareAudioTrack} from '@app/features/voice/utils/AudioPublishOptions'; -import {getLastNativeAudioArmFailure} from '@app/features/voice/utils/NativeAudioCaptureBridge'; -import type {NativeScreenCaptureSource} from '@app/types/electron.d'; -import type {VoiceEngineV2ScreenOptions} from '@fluxer/voice_engine_v2'; -import {msg} from '@lingui/core/macro'; -import type {Room, ScreenShareCaptureOptions, TrackPublishOptions} from 'livekit-client'; - -const SCREEN_SHARE_SOURCE_NO_LONGER_AVAILABLE_DESCRIPTOR = msg({ - message: 'The window or display you were sharing is no longer available, so your screen share was stopped.', - comment: 'Body of a modal shown when a native screen share stops because the selected display or window disappeared.', - context: 'screen-share', -}); -const WINDOW_SCREEN_SHARE_SOURCE_DESCRIPTOR = msg({ - message: 'Window', - comment: 'Fallback display name for a window screen-share source when the original window title is unavailable.', - context: 'screen-share', -}); -const SCREEN_SCREEN_SHARE_SOURCE_DESCRIPTOR = msg({ - message: 'Screen', - comment: 'Fallback display name for a full-screen screen-share source when the original display name is unavailable.', - context: 'screen-share', -}); - -interface NativeEngineScreenShareSettingsUpdateRequest { - room: Room | null; - options?: ScreenShareCaptureOptions; - publishOptions?: TrackPublishOptions; -} - -interface NativeEngineScreenShareEncodingState { - width: number; - height: number; - frameRate: number | undefined; - maxBitrateBps: number | undefined; - codec: TrackPublishOptions['videoCodec'] | undefined; -} - -type NativeEngineScreenShareSettingsUpdatePlan = - | { - action: 'noop'; - } - | { - action: 'update-encoding'; - state: NativeEngineScreenShareEncodingState; - } - | { - action: 'restart'; - reason: 'codec-changed' | 'encoding-update-unavailable'; - }; - -export type VoiceEngineV2AppScreenShareCaptureIdGenerator = () => string; - -export interface VoiceEngineV2AppScreenShareCaptureCoordinatorOptions { - captureIdGenerator?: VoiceEngineV2AppScreenShareCaptureIdGenerator; -} - -export interface NativeEngineScreenShareCaptureDimensions { - width: number; - height: number; -} - -export type VoiceEngineV2AppScreenShareActiveCaptureRecord = - | { - readonly kind: 'idle'; - } - | { - readonly kind: 'restore'; - readonly options: NativeScreenShareOptions; - readonly publishOptions: TrackPublishOptions | undefined; - } - | { - readonly kind: 'display'; - readonly captureId: string; - readonly captureDimensions: NativeEngineScreenShareCaptureDimensions; - readonly options: NativeScreenShareOptions; - readonly publishOptions: TrackPublishOptions | undefined; - readonly publishedTrackSid: string | null; - } - | { - readonly kind: 'device'; - readonly publishedTrackSid: string | null; - }; - -interface NativeEngineScreenSharePublicationSidWaiter { - captureId: string; - resolve: (trackSid: string) => void; - reject: (error: Error) => void; - timeoutId: NodeJS.Timeout; -} - -interface NativeEngineScreenShareStartCaptureOptions { - sendUpdate?: boolean; - playSound?: boolean; - allowDuringPending?: boolean; -} - -const EMPTY_ACTIVE_CAPTURE_RECORD: VoiceEngineV2AppScreenShareActiveCaptureRecord = {kind: 'idle'}; -const NATIVE_SCREEN_SHARE_PUBLICATION_SID_WAIT_MS = 5000; -const NATIVE_SCREEN_SHARE_PUBLICATION_SID_WAITERS_MAX = 8; - -interface ParsedDesktopCaptureSourceId { - kind: 'window' | 'screen'; - id: string; - desktopCaptureSourceId: string; -} - -function parseDesktopCaptureSourceId(sourceId: string): ParsedDesktopCaptureSourceId | null { - const parsed = /^(window|screen):([^:]+):(?:0|1)$/.exec(sourceId); - if (!parsed) return null; - const kind = parsed[1] === 'window' ? 'window' : 'screen'; - const id = parsed[2]; - assert.ok(id.length > 0, 'desktop capture source id must include a native id'); - return {kind, id, desktopCaptureSourceId: sourceId}; -} - -function buildNativeOptionsFromControllerScreenPublish(options: VoiceEngineV2ScreenOptions): NativeScreenShareOptions { - assert.ok(options.captureId.length > 0, 'captureId is required'); - const selectedSourceId = ActiveScreenShareSource.getSourceId() ?? options.captureId; - const parsed = parseDesktopCaptureSourceId(selectedSourceId) ?? parseDesktopCaptureSourceId(options.captureId); - const kind = parsed?.kind ?? 'screen'; - const source: NativeScreenCaptureSource = { - kind, - id: parsed?.id ?? options.captureId, - name: kind === 'window' ? 'Window' : 'Screen', - width: options.width, - height: options.height, - }; - return { - source, - captureId: options.captureId, - desktopCaptureSourceId: parsed?.desktopCaptureSourceId, - resolution: { - width: options.width, - height: options.height, - ...(options.maxFramerate !== undefined ? {frameRate: options.maxFramerate} : {}), - }, - }; -} - -function buildTrackPublishOptionsFromControllerScreenPublish( - options: VoiceEngineV2ScreenOptions, -): TrackPublishOptions | undefined { - const publishOptions: TrackPublishOptions = {}; - let hasPublishOptions = false; - if (isScreenShareVideoCodecValue(options.codec)) { - publishOptions.videoCodec = options.codec; - hasPublishOptions = true; - } - if (options.maxBitrateBps !== undefined) { - publishOptions.screenShareEncoding = { - maxBitrate: options.maxBitrateBps, - ...(options.maxFramerate !== undefined ? {maxFramerate: options.maxFramerate} : {}), - }; - hasPublishOptions = true; - } - return hasPublishOptions ? publishOptions : undefined; -} - -export function resolveNativeEngineScreenShareRestartCommitTrackSid(args: { - readonly publishedTrackSid: string | null; - readonly activeCaptureId: string | null; - readonly fallbackCaptureId: string; -}): string { - assert.ok(args !== null && typeof args === 'object', 'commit track sid args must be an object'); - assert.ok(args.fallbackCaptureId.length > 0, 'fallbackCaptureId is required'); - if (args.publishedTrackSid) return args.publishedTrackSid; - throw new Error('Native screen-share restart commit requires a published LiveKit track SID'); -} - -function createCryptoNativeScreenShareCaptureId(): string { - const cryptoPort = globalThis.crypto; - if (!cryptoPort || typeof cryptoPort.randomUUID !== 'function') { - throw new Error('Native screen-share capture ID generation requires crypto.randomUUID'); - } - const id = cryptoPort.randomUUID(); - assert.equal(typeof id, 'string', 'crypto.randomUUID must return a string'); - assert.ok(id.length > 0, 'crypto.randomUUID must return a non-empty capture ID'); - return id; -} - -async function captureDeviceShareAudioInputTrack(audioDeviceId: string | undefined): Promise { - assert.ok(navigator.mediaDevices?.getUserMedia, 'getUserMedia is required for device screen-share audio'); - const useExactDevice = audioDeviceId != null && audioDeviceId !== '' && audioDeviceId !== 'default'; - const stream = await navigator.mediaDevices.getUserMedia({ - audio: { - ...(useExactDevice ? {deviceId: {exact: audioDeviceId}} : {}), - echoCancellation: false, - noiseSuppression: false, - autoGainControl: false, - channelCount: 2, - sampleRate: 48_000, - }, - }); - const audioTracks = stream.getAudioTracks(); - const track = audioTracks[0]; - if (!track) { - for (const other of stream.getTracks()) { - other.stop(); - } - throw new Error('No audio track captured for device screen share'); - } - for (const other of stream.getTracks()) { - if (other !== track) { - other.stop(); - } - } - prepareHighFidelityScreenShareAudioTrack(track); - return track; -} - -export class VoiceEngineV2AppScreenShareCaptureCoordinator { - private readonly adapter: VoiceEngineV2AppScreenShareExecutionAdapter; - private readonly captureIdGenerator: VoiceEngineV2AppScreenShareCaptureIdGenerator; - private queuedSettingsUpdate: NativeEngineScreenShareSettingsUpdateRequest | null = null; - private activeCapture: VoiceEngineV2AppScreenShareActiveCaptureRecord = EMPTY_ACTIVE_CAPTURE_RECORD; - private publicationSidWaiters: Array = []; - private routedStopDepth = 0; - - constructor( - adapter: VoiceEngineV2AppScreenShareExecutionAdapter, - options: VoiceEngineV2AppScreenShareCaptureCoordinatorOptions = {}, - ) { - assert.ok(adapter, 'screen-share adapter is required'); - assert.ok(options !== null && typeof options === 'object', 'coordinator options must be an object'); - this.adapter = adapter; - this.captureIdGenerator = options.captureIdGenerator ?? createCryptoNativeScreenShareCaptureId; - assert.equal(typeof this.captureIdGenerator, 'function', 'captureIdGenerator must be a function'); - } - - private assertDisplayCapture( - reason: string, - ): Extract { - assert.equal(typeof reason, 'string', 'display capture assertion reason must be a string'); - if (this.activeCapture.kind !== 'display') { - throw new Error(`Native screen-share display capture required: ${reason}`); - } - return this.activeCapture; - } - - private assertMutableOptionsState( - reason: string, - ): Extract { - assert.equal(typeof reason, 'string', 'capture options assertion reason must be a string'); - if (this.activeCapture.kind !== 'display' && this.activeCapture.kind !== 'restore') { - throw new Error(`Native screen-share options state required: ${reason}`); - } - return this.activeCapture; - } - - private removePublicationSidWaiter(waiter: NativeEngineScreenSharePublicationSidWaiter): void { - this.publicationSidWaiters = this.publicationSidWaiters.filter((candidate) => candidate !== waiter); - assert.ok( - this.publicationSidWaiters.length <= NATIVE_SCREEN_SHARE_PUBLICATION_SID_WAITERS_MAX, - 'publication SID waiter count must stay bounded', - ); - } - - private resolvePublicationSidWaiters(trackSid: string): void { - assert.ok(trackSid.length > 0, 'trackSid must not be empty'); - const captureId = this.activeCaptureId; - if (!captureId) return; - const remaining: Array = []; - for (const waiter of this.publicationSidWaiters) { - if (waiter.captureId !== captureId) { - remaining.push(waiter); - continue; - } - clearTimeout(waiter.timeoutId); - waiter.resolve(trackSid); - } - this.publicationSidWaiters = remaining; - } - - private rejectPublicationSidWaiters(captureId: string | null, reason: string): void { - assert.equal(typeof reason, 'string', 'publication SID rejection reason must be a string'); - const remaining: Array = []; - for (const waiter of this.publicationSidWaiters) { - if (captureId !== null && waiter.captureId !== captureId) { - remaining.push(waiter); - continue; - } - clearTimeout(waiter.timeoutId); - waiter.reject(new Error(`Native screen-share publication SID unavailable: ${reason}`)); - } - this.publicationSidWaiters = remaining; - } - - private async waitForPublishedTrackSid(captureId: string): Promise { - assert.ok(captureId.length > 0, 'captureId is required'); - const currentTrackSid = this.activeCapturePublishedTrackSid; - if (this.activeCaptureId === captureId && currentTrackSid) return currentTrackSid; - assert.ok( - this.publicationSidWaiters.length < NATIVE_SCREEN_SHARE_PUBLICATION_SID_WAITERS_MAX, - 'publication SID waiter count exceeded', - ); - return new Promise((resolve, reject) => { - const waiter: NativeEngineScreenSharePublicationSidWaiter = { - captureId, - resolve, - reject, - timeoutId: setTimeout(() => { - this.removePublicationSidWaiter(waiter); - reject(new Error('Timed out waiting for native screen-share publication SID')); - }, NATIVE_SCREEN_SHARE_PUBLICATION_SID_WAIT_MS), - }; - this.publicationSidWaiters.push(waiter); - }); - } - - get activeCaptureId(): string | null { - return this.activeCapture.kind === 'display' ? this.activeCapture.captureId : null; - } - - set activeCaptureId(value: string | null) { - if (value !== null) { - throw new Error('Native screen-share capture id cannot be assigned without a display capture record'); - } - if (this.activeCapture.kind !== 'display') return; - this.rejectPublicationSidWaiters(this.activeCapture.captureId, 'capture cleared'); - this.activeCapture = EMPTY_ACTIVE_CAPTURE_RECORD; - } - - get activeCaptureDimensions(): NativeEngineScreenShareCaptureDimensions | null { - return this.activeCapture.kind === 'display' ? this.activeCapture.captureDimensions : null; - } - - set activeCaptureDimensions(value: NativeEngineScreenShareCaptureDimensions | null) { - if (value === null) { - if (this.activeCapture.kind !== 'display') return; - throw new Error('Native screen-share display dimensions cannot be cleared while a display capture is active'); - } - assert.ok(value.width > 0, 'active screen-share width must be positive'); - assert.ok(value.height > 0, 'active screen-share height must be positive'); - const capture = this.assertDisplayCapture('updating display capture dimensions'); - this.activeCapture = {...capture, captureDimensions: value}; - } - - get activeCaptureOptions(): NativeScreenShareOptions | null { - if (this.activeCapture.kind === 'display' || this.activeCapture.kind === 'restore') { - return this.activeCapture.options; - } - return null; - } - - set activeCaptureOptions(value: NativeScreenShareOptions | null) { - if (value === null) { - if (this.activeCapture.kind === 'restore') { - this.activeCapture = EMPTY_ACTIVE_CAPTURE_RECORD; - return; - } - if (this.activeCapture.kind === 'idle') return; - throw new Error('Native screen-share options cannot be cleared while a capture is active'); - } - assert.ok(value.source, 'active screen-share options require a source'); - const capture = this.assertMutableOptionsState('updating active capture options'); - this.activeCapture = {...capture, options: value}; - } - - get activeCapturePublishOptions(): TrackPublishOptions | undefined { - if (this.activeCapture.kind === 'display' || this.activeCapture.kind === 'restore') { - return this.activeCapture.publishOptions; - } - return undefined; - } - - set activeCapturePublishOptions(value: TrackPublishOptions | undefined) { - if (value === undefined && this.activeCapture.kind === 'idle') return; - const capture = this.assertMutableOptionsState('updating active capture publish options'); - this.activeCapture = {...capture, publishOptions: value}; - } - - get activeCapturePublishedTrackSid(): string | null { - if (this.activeCapture.kind === 'display' || this.activeCapture.kind === 'device') { - return this.activeCapture.publishedTrackSid; - } - return null; - } - - set activeCapturePublishedTrackSid(value: string | null) { - assert.ok(value === null || typeof value === 'string', 'published track sid must be a string or null'); - if (value !== null) { - this.recordPublishedTrackSid(value); - return; - } - if (this.activeCapture.kind === 'display' || this.activeCapture.kind === 'device') { - this.activeCapture = {...this.activeCapture, publishedTrackSid: null}; - } - } - - recordPublishedTrackSid(trackSid: string, publication?: NativeVoiceEngineLocalTrackPublication): boolean { - assert.equal(typeof trackSid, 'string', 'published track sid must be a string'); - assert.ok(trackSid.length > 0, 'published track sid must be non-empty'); - assert.ok(publication === undefined || typeof publication === 'object', 'publication must be an object'); - if (this.activeCapture.kind !== 'display' && this.activeCapture.kind !== 'device') { - logger.warn('Dropping screen-share published track sid without an active capture', { - publishedTrackSid: trackSid, - state: this.activeCapture.kind, - }); - return false; - } - if (this.activeCapture.kind === 'display' && publication?.trackName) { - if (publication.trackName !== this.activeCapture.captureId) { - logger.warn('Dropping stale native screen-share publication SID for a different capture', { - publishedTrackSid: trackSid, - trackName: publication.trackName, - activeCaptureId: this.activeCapture.captureId, - }); - return false; - } - } - this.activeCapture = {...this.activeCapture, publishedTrackSid: trackSid}; - this.resolvePublicationSidWaiters(trackSid); - return true; - } - - get deviceCaptureActive(): boolean { - return this.activeCapture.kind === 'device'; - } - - set deviceCaptureActive(value: boolean) { - assert.equal(typeof value, 'boolean', 'device capture active flag must be a boolean'); - if (!value) { - if (this.activeCapture.kind === 'device') { - this.activeCapture = EMPTY_ACTIVE_CAPTURE_RECORD; - } - return; - } - if (this.activeCapture.kind === 'display') { - throw new Error('Native device screen share cannot overlap an active display capture'); - } - this.activeCapture = {kind: 'device', publishedTrackSid: null}; - } - - private beginActiveCapture(args: { - captureId: string; - captureDimensions: NativeEngineScreenShareCaptureDimensions; - options: NativeScreenShareOptions; - publishOptions: TrackPublishOptions | undefined; - }): void { - assert.ok(args.captureId.length > 0, 'beginActiveCapture requires a captureId'); - assert.ok(args.options.source, 'beginActiveCapture requires capture options with a source'); - assert.ok(args.captureDimensions.width > 0, 'beginActiveCapture requires a positive width'); - assert.ok(args.captureDimensions.height > 0, 'beginActiveCapture requires a positive height'); - if (this.activeCapture.kind === 'display') { - throw new Error('Native display screen share cannot begin while a display capture is active'); - } - if (this.activeCapture.kind === 'device') { - throw new Error('Native display screen share cannot overlap an active device capture'); - } - this.activeCapture = { - kind: 'display', - captureId: args.captureId, - captureDimensions: args.captureDimensions, - options: args.options, - publishOptions: args.publishOptions, - publishedTrackSid: null, - }; - } - - private commitActiveCapture(capture: NativeEngineScreenCapture, options: NativeScreenShareOptions): void { - assert.ok(capture.captureId.length > 0, 'commitActiveCapture requires a captureId'); - assert.ok(capture.width > 0, 'commitActiveCapture requires a positive width'); - assert.ok(capture.height > 0, 'commitActiveCapture requires a positive height'); - assert.ok(options.source, 'commitActiveCapture requires capture options with a source'); - const previous = this.assertDisplayCapture('committing display capture'); - assert.equal(previous.captureId, capture.captureId, 'committed capture ID must match the active display capture'); - this.activeCapture = { - ...previous, - captureId: capture.captureId, - captureDimensions: {width: capture.width, height: capture.height}, - options, - }; - } - - private clearActiveCaptureForRelease(preserveRestoreState: boolean): void { - assert.equal(typeof preserveRestoreState, 'boolean', 'preserveRestoreState must be a boolean'); - const previous = this.activeCapture; - this.rejectPublicationSidWaiters(previous.kind === 'display' ? previous.captureId : null, 'capture released'); - if (preserveRestoreState && (previous.kind === 'display' || previous.kind === 'restore')) { - this.activeCapture = { - kind: 'restore', - options: previous.options, - publishOptions: previous.publishOptions, - }; - } else { - this.activeCapture = EMPTY_ACTIVE_CAPTURE_RECORD; - } - assert.equal(this.activeCaptureId, null, 'captureId must be cleared on release'); - assert.equal(this.activeCapturePublishedTrackSid, null, 'published track sid must be cleared on release'); - assert.equal(this.deviceCaptureActive, false, 'device capture flag must be cleared on release'); - } - - async publishControllerScreen(options: VoiceEngineV2ScreenOptions): Promise { - assert.ok(options !== null && typeof options === 'object', 'controller screen options must be an object'); - const nativeOptions = buildNativeOptionsFromControllerScreenPublish(options); - const publishOptions = buildTrackPublishOptionsFromControllerScreenPublish(options); - await this.startDisplayCapture(null, nativeOptions, undefined, publishOptions); - } - - async resolveOptionsFromActiveSource(options?: ScreenShareCaptureOptions): Promise { - assert.ok(options === undefined || typeof options === 'object'); - const sourceId = ActiveScreenShareSource.getSourceId(); - if (!sourceId) return null; - assert.equal(typeof sourceId, 'string'); - const parsed = parseDesktopCaptureSourceId(sourceId); - const kind: NativeScreenCaptureSource['kind'] = parsed - ? parsed.kind - : sourceId.startsWith('window:') - ? 'window' - : 'screen'; - const nativeId = parsed ? parsed.id : sourceId; - const desktopCaptureSourceId = parsed?.desktopCaptureSourceId; - const resolution = options?.resolution ?? undefined; - const source: NativeScreenCaptureSource = { - kind, - id: nativeId, - name: - kind === 'window' - ? i18n._(WINDOW_SCREEN_SHARE_SOURCE_DESCRIPTOR) - : i18n._(SCREEN_SCREEN_SHARE_SOURCE_DESCRIPTOR), - width: resolution?.width ?? 1920, - height: resolution?.height ?? 1080, - }; - const nativeOptions: NativeScreenShareOptions = { - source, - ...(desktopCaptureSourceId ? {desktopCaptureSourceId} : {}), - resolution, - contentHint: options?.contentHint, - }; - return this.withAudioIfRequested( - nativeOptions, - options?.audio === true || this.shouldIncludeAudioFromSettings(source.kind), - ); - } - - createCaptureId(): string { - const id = this.captureIdGenerator(); - assert.equal(typeof id, 'string', 'captureIdGenerator must return a string'); - assert.ok(id.length > 0, 'captureIdGenerator must return a non-empty capture ID'); - return id; - } - - getPublishDimensions(nativeOptions: NativeScreenShareOptions): {width: number; height: number} { - assert.ok(nativeOptions); - assert.ok(nativeOptions.source); - const dims = { - width: nativeOptions.resolution?.width ?? nativeOptions.source.width, - height: nativeOptions.resolution?.height ?? nativeOptions.source.height, - }; - assert.ok(dims.width > 0, 'width must be > 0'); - assert.ok(dims.height > 0, 'height must be > 0'); - return dims; - } - - private getCurrentEncodingState(currentOptions: NativeScreenShareOptions): NativeEngineScreenShareEncodingState { - const previousDimensions = this.activeCaptureDimensions ?? { - width: currentOptions.source.width, - height: currentOptions.source.height, - }; - const publishOptions = this.activeCapturePublishOptions; - return { - width: currentOptions.resolution?.width ?? previousDimensions.width, - height: currentOptions.resolution?.height ?? previousDimensions.height, - frameRate: publishOptions?.screenShareEncoding?.maxFramerate ?? currentOptions.resolution?.frameRate, - maxBitrateBps: publishOptions?.screenShareEncoding?.maxBitrate, - codec: publishOptions?.videoCodec, - }; - } - - private getRequestedEncodingState( - currentOptions: NativeScreenShareOptions, - options?: ScreenShareCaptureOptions, - publishOptions?: TrackPublishOptions, - ): NativeEngineScreenShareEncodingState { - const current = this.getCurrentEncodingState(currentOptions); - const nextResolution = options?.resolution ?? currentOptions.resolution; - return { - width: nextResolution?.width ?? current.width, - height: nextResolution?.height ?? current.height, - frameRate: nextResolution?.frameRate ?? publishOptions?.screenShareEncoding?.maxFramerate ?? current.frameRate, - maxBitrateBps: publishOptions?.screenShareEncoding?.maxBitrate ?? current.maxBitrateBps, - codec: publishOptions?.videoCodec ?? current.codec, - }; - } - - private hasEncodingStateChanged( - current: NativeEngineScreenShareEncodingState, - next: NativeEngineScreenShareEncodingState, - ): boolean { - return ( - Math.round(current.width) !== Math.round(next.width) || - Math.round(current.height) !== Math.round(next.height) || - current.frameRate !== next.frameRate || - current.maxBitrateBps !== next.maxBitrateBps || - current.codec !== next.codec - ); - } - - private async planSettingsUpdate( - currentOptions: NativeScreenShareOptions, - options?: ScreenShareCaptureOptions, - publishOptions?: TrackPublishOptions, - ): Promise { - const current = this.getCurrentEncodingState(currentOptions); - const next = this.getRequestedEncodingState(currentOptions, options, publishOptions); - if (!this.hasEncodingStateChanged(current, next)) return {action: 'noop'}; - if (current.codec !== next.codec) return {action: 'restart', reason: 'codec-changed'}; - const canUpdateEncoding = await isVoiceEngineV2AppNativeScreenShareEncodingUpdateAvailable(); - if (!canUpdateEncoding) return {action: 'restart', reason: 'encoding-update-unavailable'}; - return {action: 'update-encoding', state: next}; - } - - private async updateEncodingInPlaceForSettings( - currentOptions: NativeScreenShareOptions, - state: NativeEngineScreenShareEncodingState, - ): Promise { - const captureId = this.activeCaptureId; - if (!captureId) return false; - assert.ok(state.width > 0, 'in-place screen-share width must be positive'); - assert.ok(state.height > 0, 'in-place screen-share height must be positive'); - await requireVoiceEngineV2AppNativeBridge('update native screen share encoding').updateScreenShareEncoding({ - captureId, - width: state.width, - height: state.height, - frameRate: state.frameRate, - maxBitrateBps: state.maxBitrateBps, - }); - const resolution = { - width: state.width, - height: state.height, - ...(state.frameRate !== undefined ? {frameRate: state.frameRate} : {}), - }; - this.activeCaptureDimensions = {width: state.width, height: state.height}; - this.activeCaptureOptions = { - ...currentOptions, - resolution, - }; - logger.info('Updated native-engine screen share encoding in place', { - captureId, - width: state.width, - height: state.height, - frameRate: state.frameRate, - }); - return true; - } - - private queueSettingsUpdate( - room: Room | null, - options?: ScreenShareCaptureOptions, - publishOptions?: TrackPublishOptions, - ): void { - this.queuedSettingsUpdate = {room, options, publishOptions}; - } - - async applyQueuedSettingsUpdate(): Promise { - const queued = this.queuedSettingsUpdate; - if (!queued) return; - assert.ok(queued, 'queued must be present after early-return'); - this.queuedSettingsUpdate = null; - await this.updateActiveSettings(queued.room, queued.options, queued.publishOptions).catch((error) => { - logger.warn('Failed to apply queued native-engine screen-share settings update', {error}); - }); - } - - shouldIncludeAudioFromSettings(sourceKind: NativeScreenCaptureSource['kind']): boolean { - assert.equal(typeof sourceKind, 'string'); - if (VoiceSettings.getScreenShareAudioSourceMode() === 'none') return false; - if (sourceKind === 'screen') return VoiceSettings.getShareDesktopAudio(); - if (sourceKind === 'game') return VoiceSettings.getShareDesktopAudio(); - if (sourceKind === 'window') return VoiceSettings.getShareAppAudio(); - return false; - } - - private isActiveFluxerOwnedWindow(nativeOptions: NativeScreenShareOptions): boolean { - assert.ok(nativeOptions.source, 'nativeOptions.source is required'); - if (nativeOptions.source.kind !== 'window') return false; - if (!ActiveScreenShareSource.isOwnWindow()) return false; - const activeSourceId = ActiveScreenShareSource.getSourceId(); - if (!activeSourceId) return false; - if (nativeOptions.desktopCaptureSourceId === activeSourceId) return true; - return nativeOptions.source.id === activeSourceId; - } - - private async captureAudioOptions( - nativeOptions: NativeScreenShareOptions, - ): Promise | null> { - if (nativeOptions.nativeAudioLinuxRule) { - return {nativeAudioLinuxRule: nativeOptions.nativeAudioLinuxRule}; - } - if (nativeOptions.nativeAudioFramePump) { - return {nativeAudioFramePump: nativeOptions.nativeAudioFramePump}; - } - const electronApi = getElectronAPI(); - if (!electronApi) return null; - if (nativeOptions.source.kind === 'screen' || nativeOptions.source.kind === 'game') { - if (electronApi.platform === 'darwin' || electronApi.platform === 'win32') { - return {nativeAudioFramePump: {kind: 'system'}}; - } - return null; - } - if (nativeOptions.source.kind !== 'window') return null; - let targetPid = nativeOptions.source.targetPid ?? null; - if (nativeOptions.desktopCaptureSourceId && electronApi.nativeAudio) { - const resolvedPid = await electronApi.nativeAudio - .resolveAudioRootPidForSource(nativeOptions.desktopCaptureSourceId) - .catch((error) => { - logger.warn('Failed to resolve native screen-share audio PID for live settings update', { - desktopSourceId: nativeOptions.desktopCaptureSourceId, - platform: electronApi.platform, - error, - }); - return null; - }); - if (typeof resolvedPid === 'number' && resolvedPid > 0) { - targetPid = resolvedPid; - } - } - if (!targetPid) return null; - return {nativeAudioFramePump: {kind: 'window', targetPid}}; - } - - async withAudioIfRequested( - nativeOptions: NativeScreenShareOptions, - includeAudio: boolean, - ): Promise { - assert.ok(nativeOptions); - assert.equal(typeof includeAudio, 'boolean'); - if (!includeAudio) return nativeOptions; - if (nativeOptions.audioTrack) return nativeOptions; - if (nativeOptions.nativeAudioFramePump) return nativeOptions; - if (nativeOptions.nativeAudioLinuxRule) return nativeOptions; - if (this.isActiveFluxerOwnedWindow(nativeOptions)) { - logger.warn('Native voice engine: Fluxer-owned window audio is excluded from screen share capture', { - sourceId: nativeOptions.desktopCaptureSourceId ?? nativeOptions.source.id, - }); - return nativeOptions; - } - const audioOptions = await this.captureAudioOptions(nativeOptions); - if (!audioOptions) { - logger.warn('Native voice engine: screen-share audio was requested but native audio capture was unavailable', { - sourceKind: nativeOptions.source.kind, - sourceId: nativeOptions.source.id, - reason: getLastNativeAudioArmFailure()?.reason ?? null, - detail: getLastNativeAudioArmFailure()?.detail ?? null, - }); - return null; - } - return {...nativeOptions, ...audioOptions}; - } - - private async updateAudioForSettings( - currentOptions: NativeScreenShareOptions, - includeAudio: boolean, - ): Promise { - if (!includeAudio) { - if (!this.adapter.nativeEngineScreenShareAudioPump) return false; - await this.adapter.audioPump.stopAudio(); - const { - audioTrack: _audioTrack, - nativeAudioFramePump: _nativeAudioFramePump, - nativeAudioLinuxRule: _nativeAudioLinuxRule, - ...nextOptions - } = currentOptions; - this.activeCaptureOptions = nextOptions; - LocalVoiceState.updateSelfStreamAudio(false); - return true; - } - if (this.adapter.nativeEngineScreenShareAudioPump) return false; - const nextOptions = await this.withAudioIfRequested(currentOptions, true); - if (!nextOptions || nextOptions === currentOptions) { - logger.warn('Native-engine screen-share audio could not be enabled for active share', { - sourceKind: currentOptions.source.kind, - sourceId: currentOptions.source.id, - reason: getLastNativeAudioArmFailure()?.reason ?? null, - }); - LocalVoiceState.updateSelfStreamAudio(false); - return false; - } - let audioPublished = false; - try { - audioPublished = await this.adapter.audioPump.startAudio(nextOptions); - } catch (error) { - logger.warn('Native-engine screen-share audio start failed for active share; continuing video-only', { - sourceKind: currentOptions.source.kind, - sourceId: currentOptions.source.id, - error, - }); - LocalVoiceState.updateSelfStreamAudio(false); - return false; - } - if (!audioPublished) { - LocalVoiceState.updateSelfStreamAudio(false); - return false; - } - this.activeCaptureOptions = nextOptions; - LocalVoiceState.updateSelfStreamAudio(true); - return true; - } - - async updateActiveSettings( - room: Room | null, - options?: ScreenShareCaptureOptions, - publishOptions?: TrackPublishOptions, - ): Promise { - assert.ok(room === null || typeof room === 'object'); - assert.ok(options === undefined || typeof options === 'object'); - const currentOptions = this.activeCaptureOptions; - if (!this.activeCaptureId || !currentOptions) return false; - const pendingVerdict = guardScreenShareEntry({ - pending: { - active: this.adapter.isScreenSharePending, - debugMessage: 'Already pending, queued native-engine screen-share settings update', - onBlocked: () => { - this.queueSettingsUpdate(room, options, publishOptions); - }, - }, - }); - if (pendingVerdict === 'share-pending') { - return false; - } - const audioSettingProvided = typeof options?.audio === 'boolean'; - const audioEnabled = audioSettingProvided - ? options.audio === true - : this.adapter.nativeEngineScreenShareAudioPump != null; - let appliedAnySetting = false; - try { - const videoPlan = await this.planSettingsUpdate(currentOptions, options, publishOptions); - if (videoPlan.action === 'update-encoding') { - const inPlaceApplied = await this.updateEncodingInPlaceForSettings(currentOptions, videoPlan.state).catch( - (error) => { - logger.warn('Failed to update native-engine screen share encoding in place; falling back to restart', { - error, - }); - return false; - }, - ); - appliedAnySetting = inPlaceApplied || appliedAnySetting; - if (!inPlaceApplied) { - appliedAnySetting = - (await this.restartForSettings(room, currentOptions, options, publishOptions)) || appliedAnySetting; - } - } else if (videoPlan.action === 'restart') { - logger.info('Restarting native-engine screen share for settings update', {reason: videoPlan.reason}); - appliedAnySetting = - (await this.restartForSettings(room, currentOptions, options, publishOptions)) || appliedAnySetting; - } - const latestOptions = this.activeCaptureOptions ?? currentOptions; - if (audioSettingProvided) { - appliedAnySetting = (await this.updateAudioForSettings(latestOptions, audioEnabled)) || appliedAnySetting; - } - if (options && Object.hasOwn(options, 'contentHint')) { - this.activeCaptureOptions = { - ...(this.activeCaptureOptions ?? latestOptions), - contentHint: options.contentHint, - }; - appliedAnySetting = true; - } - if (publishOptions) { - this.activeCapturePublishOptions = publishOptions; - } - return appliedAnySetting; - } finally { - await this.applyQueuedSettingsUpdate(); - } - } - - private async restoreAfterRestartFailure( - currentOptions: NativeScreenShareOptions, - previousPublishOptions: TrackPublishOptions | undefined, - sourceType: VoiceScreenShareSourceType, - previousCaptureId: string | null, - ): Promise { - assert.ok(currentOptions); - try { - if (!this.activeCaptureId) { - await this.startCapture( - currentOptions, - {sendUpdate: false, playSound: false, allowDuringPending: true}, - previousPublishOptions, - ); - } - this.adapter.transitionScreenShareLifecycleInternal({ - type: 'share.resolve', - active: true, - sourceType, - encoderVerificationScheduled: false, - streamingPriorityHeld: this.adapter.streamingPriorityHeld, - }); - } catch (restoreError) { - logger.error('Failed to restore native-engine screen share after restart failure', { - previousCaptureId, - error: restoreError, - }); - this.adapter.transitionScreenShareLifecycleInternal({ - type: 'share.reject', - active: false, - sourceType, - }); - } - } - - private async buildRestartNativeOptions( - currentOptions: NativeScreenShareOptions, - options: ScreenShareCaptureOptions | undefined, - nextCaptureId: string, - ): Promise { - assert.ok(currentOptions.source, 'currentOptions.source is required'); - assert.ok(nextCaptureId.length > 0, 'nextCaptureId is required'); - const { - audioTrack: _audioTrack, - nativeAudioFramePump: _nativeAudioFramePump, - nativeAudioLinuxRule: _nativeAudioLinuxRule, - ...baseOptions - } = currentOptions; - const nextResolution = options?.resolution ?? currentOptions.resolution; - const nextOptions: NativeScreenShareOptions = { - ...baseOptions, - captureId: nextCaptureId, - ...(nextResolution ? {resolution: nextResolution} : {}), - }; - if (options && Object.hasOwn(options, 'contentHint')) { - nextOptions.contentHint = options.contentHint; - } - const includeAudio = - this.adapter.nativeEngineScreenShareAudioPump != null || - currentOptions.audioTrack != null || - currentOptions.nativeAudioFramePump != null || - currentOptions.nativeAudioLinuxRule != null; - return this.withAudioIfRequested(nextOptions, includeAudio); - } - - private async restartForSettings( - room: Room | null, - currentOptions: NativeScreenShareOptions, - options?: ScreenShareCaptureOptions, - publishOptions?: TrackPublishOptions, - ): Promise { - assert.ok(room === null || typeof room === 'object'); - const sourceType = this.adapter.getNativeScreenShareSourceType(currentOptions); - const previousPublishOptions = this.activeCapturePublishOptions; - const nextResolution = options?.resolution ?? currentOptions.resolution; - const nextPublishOptions = publishOptions ?? previousPublishOptions; - const previousCaptureId = this.activeCaptureId; - const nextCaptureId = this.createCaptureId(); - const nextNativeOptions = await this.buildRestartNativeOptions(currentOptions, options, nextCaptureId); - if (!nextNativeOptions) throw new Error('Native screen-share audio unavailable during settings republish'); - const migration = await announceScreenShareRestartMigration({ - room, - generation: ++this.adapter.screenShareMigrationGeneration, - previousPublishOptions, - nextPublishOptions, - candidateTrackSid: nextCaptureId, - reason: 'native-engine-screen-share-settings-republish', - }); - this.adapter.transitionScreenShareLifecycleInternal({ - type: 'share.replace', - sourceType, - codecRepublishInFlight: true, - }); - try { - await this.adapter.releaseNativeEngineScreenShareResourcesInternal({ - reason: 'native-engine-screen-share-settings-republish', - preserveRestoreState: true, - releaseStreamingPriority: false, - unpublishRemote: true, - preserveStreamStateOnUnpublish: true, - }); - await this.startCapture( - nextNativeOptions, - {sendUpdate: false, playSound: false, allowDuringPending: true}, - nextPublishOptions, - ); - const publishedTrackSid = migration ? await this.waitForPublishedTrackSid(nextCaptureId) : null; - if (migration) { - await commitScreenShareRestartMigration( - room, - migration, - resolveNativeEngineScreenShareRestartCommitTrackSid({ - publishedTrackSid, - activeCaptureId: this.activeCaptureId, - fallbackCaptureId: nextCaptureId, - }), - ); - } - logger.info('Restarted native-engine screen share for settings change', { - previousCaptureId, - captureId: this.activeCaptureId, - resolution: nextResolution, - frameRate: nextResolution?.frameRate, - maxBitrateBps: nextPublishOptions?.screenShareEncoding?.maxBitrate, - }); - return true; - } catch (error) { - await this.abortRestartMigration(room, migration); - logger.warn('Failed to restart native-engine screen share for settings change; attempting restore', { - previousCaptureId, - error, - }); - await this.restoreAfterRestartFailure(currentOptions, previousPublishOptions, sourceType, previousCaptureId); - return false; - } - } - - private async abortRestartMigration( - room: Room | null, - migration: ScreenShareRestartMigrationSession | null, - ): Promise { - await abortScreenShareRestartMigration(room, migration, 'native-engine-screen-share-settings-republish-failed'); - } - - clearRestoreState(): void { - if (this.activeCapture.kind === 'restore') { - this.activeCapture = EMPTY_ACTIVE_CAPTURE_RECORD; - } - assert.equal(this.activeCaptureOptions, null, 'options must be cleared'); - assert.equal(this.activeCapturePublishOptions, undefined, 'publishOptions must be cleared'); - assert.equal(this.activeCapturePublishedTrackSid, null, 'published track sid must be cleared'); - } - - private cleanupEndListener(): void { - this.adapter.nativeEngineScreenShareEndDisposer?.(); - this.adapter.nativeEngineScreenShareEndDisposer = null; - const lastCaptureId = this.adapter.nativeEngineScreenShareLifecycleBoundCaptureId; - if (lastCaptureId) { - this.adapter.sourceLifecycleBridge?.unbind(lastCaptureId); - this.adapter.nativeEngineScreenShareLifecycleBoundCaptureId = null; - } - } - - private bindEndListener(captureId: string): void { - this.cleanupEndListener(); - assert.equal(typeof captureId, 'string', 'captureId must be a string'); - assert.ok(captureId.length > 0, 'captureId must not be empty'); - const sourceId = this.activeCaptureOptions?.source.id; - if (sourceId && this.adapter.sourceLifecycleBridge) { - const bound = this.adapter.sourceLifecycleBridge.bind({captureId, sourceId}); - if (bound) { - this.adapter.nativeEngineScreenShareLifecycleBoundCaptureId = captureId; - } - } - const unsubscribe = getElectronAPI()?.nativeScreenCapture?.onEnd?.((message) => { - if (message.captureId !== captureId || message.captureId !== this.activeCaptureId) return; - if (message.reason !== 'source-vanished') return; - logger.warn('Native-engine screen-share source vanished', {captureId, detail: message.detail}); - this.adapter.showScreenShareEndedModalInternal(i18n._(SCREEN_SHARE_SOURCE_NO_LONGER_AVAILABLE_DESCRIPTOR)); - void this.stopCapture({ - sendUpdate: true, - playSound: true, - reason: 'native-engine-screen-share-source-vanished', - }).catch((error) => { - logger.warn('Failed to stop native-engine screen share after source vanished', {captureId, error}); - }); - }); - this.adapter.nativeEngineScreenShareEndDisposer = unsubscribe ?? null; - } - - async releaseResources({ - reason, - preserveRestoreState = false, - releaseStreamingPriority = true, - unpublishRemote = true, - preserveStreamStateOnUnpublish = false, - }: { - reason: string; - preserveRestoreState?: boolean; - releaseStreamingPriority?: boolean; - unpublishRemote?: boolean; - preserveStreamStateOnUnpublish?: boolean; - }): Promise { - assert.equal(typeof reason, 'string'); - assert.equal(typeof preserveRestoreState, 'boolean'); - const captureId = this.activeCaptureId; - const deviceScreenShareActive = this.deviceCaptureActive; - const nativeScreenPublicationActive = - captureId !== null || deviceScreenShareActive || this.activeCapturePublishedTrackSid !== null; - const nativeScreenAudioPublicationActive = this.adapter.nativeEngineScreenShareAudioPump !== null; - this.clearActiveCaptureForRelease(preserveRestoreState); - this.cleanupEndListener(); - this.adapter.clearScreenShareKeepAliveSinkInternal(); - this.adapter.cancelEncoderVerificationInternal(); - AdaptiveScreenShareEngine.stop(); - if (releaseStreamingPriority) { - this.adapter.setStreamingPriorityInternal(false); - } - await this.adapter.audioPump.stopAudio(true, unpublishRemote && nativeScreenAudioPublicationActive); - this.adapter.previewTracking.clearPreview(); - if (unpublishRemote && nativeScreenPublicationActive) { - if (preserveStreamStateOnUnpublish) { - applyScreenShareState(this.adapter, true, false); - this.adapter.syncLocalStreamWatchStateInternal(true); - } - if (await isVoiceEngineV2AppNativeScreenShareBridgeAvailable()) { - try { - await requireVoiceEngineV2AppNativeBridge('unpublish native screen share').unpublishScreen(); - } catch (error) { - logger.warn('Native-engine screen-share unpublish failed', {error, reason}); - } - } else { - logger.warn('Native-engine screen-share unpublish skipped because bridge is unavailable', {reason}); - } - } - - if (captureId) { - await stopNativeCaptureForEngine(captureId); - markScreenShareCaptureEnded(reason); - } else if (deviceScreenShareActive) { - markScreenShareCaptureEnded(reason); - } - } - - private async publishAndStartNativeCapture( - nativeOptions: NativeScreenShareOptions, - publishOptions: TrackPublishOptions | undefined, - effectivePublishOptions: TrackPublishOptions | undefined, - ): Promise { - assert.ok(nativeOptions); - assert.ok(nativeOptions.source); - const captureId = nativeOptions.captureId ?? this.createCaptureId(); - const publishDimensions = this.getPublishDimensions(nativeOptions); - this.beginActiveCapture({ - captureId, - captureDimensions: publishDimensions, - options: nativeOptions, - publishOptions, - }); - const adaptiveScreenShareQuality = VoiceSettings.getAdaptiveScreenShareQuality(); - const maxFramerate = - effectivePublishOptions?.screenShareEncoding?.maxFramerate ?? nativeOptions.resolution?.frameRate; - await requireVoiceEngineV2AppNativeBridge('publish native screen share').publishScreen({ - captureId, - width: publishDimensions.width, - height: publishDimensions.height, - codec: effectivePublishOptions?.videoCodec, - maxBitrateBps: effectivePublishOptions?.screenShareEncoding?.maxBitrate, - maxFramerate, - adaptiveSend: adaptiveScreenShareQuality, - pacing: 'source', - trackName: captureId, - }); - const capture = await startNativeCaptureForEngine({...nativeOptions, captureId}); - this.commitActiveCapture(capture, nativeOptions); - this.bindEndListener(capture.captureId); - this.adapter.previewTracking.registerCapturePreview(capture); - return capture; - } - - private async publishCaptureAudio( - nativeOptions: NativeScreenShareOptions, - capture: NativeEngineScreenCapture, - ): Promise { - assert.ok(nativeOptions); - assert.ok(capture); - const audioPublished = await this.adapter.audioPump.startAudio(nativeOptions); - if (audioPublished) return true; - if (!nativeOptions.audioTrack && !nativeOptions.nativeAudioLinuxRule) return false; - logger.warn('Native-engine screen-share audio was requested but not published; continuing video-only', { - captureId: capture.captureId, - hasAudioTrack: nativeOptions.audioTrack != null, - hasNativeAudioFramePump: nativeOptions.nativeAudioFramePump != null, - hasLinuxRule: nativeOptions.nativeAudioLinuxRule != null, - }); - this.stopUnpublishedAudioTrack(nativeOptions, capture.captureId, 'audio-not-published'); - return false; - } - - private stopUnpublishedAudioTrack( - nativeOptions: NativeScreenShareOptions, - captureId: string | null, - reason: string, - ): void { - assert.ok(nativeOptions); - assert.equal(typeof reason, 'string'); - if (this.adapter.nativeEngineScreenShareAudioPump) return; - if (!nativeOptions.audioTrack) return; - try { - nativeOptions.audioTrack.stop(); - } catch (error) { - logger.debug('Failed to stop unpublished native-engine screen-share audio track', { - captureId, - reason, - error, - }); - } - } - - private async finalizeCaptureSuccess( - nativeOptions: NativeScreenShareOptions, - capture: NativeEngineScreenCapture, - sourceType: VoiceScreenShareSourceType, - sendUpdate: boolean, - playSound: boolean, - audioPublished: boolean, - ): Promise { - assert.ok(capture); - assert.equal(typeof sendUpdate, 'boolean'); - await runScreenShareActivationRitual({ - adapter: this.adapter, - room: null, - participant: null, - active: true, - steps: { - acquireStreamingPriority: true, - enforcePublicationCap: false, - applyState: () => applyScreenShareState(this.adapter, true, sendUpdate, sendUpdate), - applyStatePosition: 'before-pipeline', - publishPipeline: null, - deactivateCleanup: null, - updateLocalParticipant: false, - audioSync: {kind: 'self-stream-before-watch', published: audioPublished}, - syncPersistedAudioPreferenceWhenActive: false, - playSound, - buildResolveTransition: () => ({ - type: 'share.resolve', - active: true, - sourceType, - encoderVerificationScheduled: false, - streamingPriorityHeld: this.adapter.streamingPriorityHeld, - }), - }, - }); - logger.info('Started native-engine screen share', { - sourceKind: nativeOptions.source.kind, - captureId: capture.captureId, - }); - } - - private async handleStartCaptureFailure( - nativeOptions: NativeScreenShareOptions, - capture: NativeEngineScreenCapture | null, - sourceType: VoiceScreenShareSourceType, - sendUpdate: boolean, - error: unknown, - ): Promise { - assert.ok(nativeOptions); - logger.warn('Failed to start native-engine screen share', {error}); - if (capture && !this.adapter.nativeEngineScreenSharePreviewTrackSid) { - await capture.previewBridge.cleanup(false).catch((cleanupError) => { - logger.warn('Failed to clean up native-engine screen-share preview after start failure', { - captureId: capture?.captureId, - error: cleanupError, - }); - }); - } - this.stopUnpublishedAudioTrack(nativeOptions, capture?.captureId ?? null, 'start-capture-failed'); - await this.stopCaptureDirect({sendUpdate, playSound: false}).catch((stopError) => { - logger.warn('Failed to stop native-engine screen share after start failure', {error: stopError}); - }); - this.adapter.transitionScreenShareLifecycleInternal({ - type: 'share.resolve', - active: false, - sourceType, - encoderVerificationScheduled: false, - streamingPriorityHeld: this.adapter.streamingPriorityHeld, - }); - } - - async startCapture( - nativeOptions: NativeScreenShareOptions, - options?: NativeEngineScreenShareStartCaptureOptions, - publishOptions?: TrackPublishOptions, - ): Promise { - assert.ok(nativeOptions); - assert.ok(nativeOptions.source); - const {sendUpdate = true, playSound = true, allowDuringPending = false} = options || {}; - assert.equal(typeof sendUpdate, 'boolean'); - assert.equal(typeof allowDuringPending, 'boolean'); - if (!allowDuringPending) { - const pendingVerdict = guardScreenShareEntry({ - pending: { - active: this.adapter.isScreenSharePending, - debugMessage: 'Already pending, ignoring native-engine screen share request', - }, - }); - if (pendingVerdict === 'share-pending') { - return; - } - } - const sourceType = this.adapter.getNativeScreenShareSourceType(nativeOptions); - this.adapter.transitionScreenShareLifecycleInternal({type: 'share.start', sourceType}); - let capture: NativeEngineScreenCapture | null = null; - try { - const effectivePublishOptions = await this.adapter.getEffectivePublishOptionsInternal(true, publishOptions); - capture = await this.publishAndStartNativeCapture(nativeOptions, publishOptions, effectivePublishOptions); - const audioPublished = await this.publishCaptureAudio(nativeOptions, capture); - await this.finalizeCaptureSuccess(nativeOptions, capture, sourceType, sendUpdate, playSound, audioPublished); - await this.applyQueuedSettingsUpdate(); - } catch (error) { - await this.handleStartCaptureFailure(nativeOptions, capture, sourceType, sendUpdate, error); - throw error; - } - } - - private async publishDeviceCaptureEngine( - options: DeviceScreenShareCaptureOptions | undefined, - effectivePublishOptions: TrackPublishOptions | undefined, - publishDimensions: {width: number; height: number; frameRate: number}, - ): Promise { - assert.ok(publishDimensions); - assert.ok(publishDimensions.frameRate > 0); - const videoDeviceId = options?.videoDeviceId; - await ensureNativeCameraPermissionForDeviceShare('start'); - await requireVoiceEngineV2AppNativeBridge('publish native device screen share').publishDeviceScreenShare({ - ...(videoDeviceId && videoDeviceId !== 'default' ? {deviceId: videoDeviceId} : {}), - width: publishDimensions.width, - height: publishDimensions.height, - frameRate: publishDimensions.frameRate, - ...(isScreenShareVideoCodecValue(effectivePublishOptions?.videoCodec) - ? {codec: effectivePublishOptions.videoCodec} - : {}), - maxBitrateBps: effectivePublishOptions?.screenShareEncoding?.maxBitrate, - maxFramerate: effectivePublishOptions?.screenShareEncoding?.maxFramerate ?? publishDimensions.frameRate, - }); - } - - private async publishDeviceCaptureAudio(options: DeviceScreenShareCaptureOptions | undefined): Promise { - const audioDeviceId = options?.audioDeviceId; - if (audioDeviceId === undefined) { - return false; - } - await ensureNativeMicrophonePermissionForDeviceShare('start'); - let audioTrack: MediaStreamTrack | null = null; - try { - audioTrack = await captureDeviceShareAudioInputTrack(audioDeviceId); - const published = await this.adapter.audioPump.startAudioFromTrack(audioTrack, audioDeviceId || 'default'); - if (!published) { - audioTrack.stop(); - } - return published; - } catch (error) { - logger.warn('Native-engine device screen-share audio was requested but not published; continuing video-only', { - audioDeviceId, - error, - }); - if (audioTrack && this.adapter.nativeEngineScreenShareAudioPump === null) { - audioTrack.stop(); - } - return false; - } - } - - private async finalizeDeviceCaptureSuccess( - options: DeviceScreenShareCaptureOptions | undefined, - effectivePublishOptions: TrackPublishOptions | undefined, - publishDimensions: {width: number; height: number; frameRate: number}, - sendUpdate: boolean, - playSound: boolean, - audioPublished: boolean, - ): Promise { - assert.equal(typeof sendUpdate, 'boolean'); - assert.equal(typeof audioPublished, 'boolean'); - assert.ok(publishDimensions); - this.deviceCaptureActive = true; - this.adapter.previewTracking.registerDevicePreview(options, publishDimensions); - markScreenShareCaptureActive({method: 'device-media', device: {videoDeviceId: options?.videoDeviceId}}); - await runScreenShareActivationRitual({ - adapter: this.adapter, - room: null, - participant: null, - active: true, - steps: { - acquireStreamingPriority: true, - enforcePublicationCap: false, - applyState: () => applyScreenShareState(this.adapter, true, sendUpdate, sendUpdate), - applyStatePosition: 'before-pipeline', - publishPipeline: null, - deactivateCleanup: null, - updateLocalParticipant: false, - audioSync: {kind: 'self-stream-before-watch', published: audioPublished}, - syncPersistedAudioPreferenceWhenActive: false, - playSound, - buildResolveTransition: () => ({ - type: 'share.resolve', - active: true, - sourceType: 'device', - encoderVerificationScheduled: false, - streamingPriorityHeld: this.adapter.streamingPriorityHeld, - }), - }, - }); - logger.info('Started native-engine device screen share', { - videoDeviceId: options?.videoDeviceId, - previewVideoDeviceId: options?.previewVideoDeviceId, - width: publishDimensions.width, - height: publishDimensions.height, - frameRate: publishDimensions.frameRate, - codec: effectivePublishOptions?.videoCodec, - maxBitrateBps: effectivePublishOptions?.screenShareEncoding?.maxBitrate, - maxFramerate: effectivePublishOptions?.screenShareEncoding?.maxFramerate, - }); - } - - private async assertNativeBridgeAvailable(context: string): Promise { - assert.equal(typeof context, 'string'); - assert.ok(context.length > 0, 'native bridge availability context is required'); - if (await isVoiceEngineV2AppNativeScreenShareBridgeAvailable()) return; - throw new Error(`Voice engine v2 native bridge is required for ${context}`); - } - - async startDeviceCapture( - options?: DeviceScreenShareCaptureOptions, - startOptions?: {sendUpdate?: boolean; playSound?: boolean}, - publishOptions?: TrackPublishOptions, - ): Promise { - const {sendUpdate = true, playSound = true} = startOptions || {}; - assert.equal(typeof sendUpdate, 'boolean'); - assert.equal(typeof playSound, 'boolean'); - const pendingVerdict = guardScreenShareEntry({ - pending: { - active: this.adapter.isScreenSharePending, - debugMessage: 'Already pending, ignoring native-engine device screen share request', - }, - }); - if (pendingVerdict === 'share-pending') { - return; - } - await this.assertNativeBridgeAvailable('device screen share capture'); - this.adapter.transitionScreenShareLifecycleInternal({type: 'share.start', sourceType: 'device'}); - try { - const effectivePublishOptions = await this.adapter.getEffectivePublishOptionsInternal(true, publishOptions); - const publishDimensions = getDeviceScreenSharePublishDimensions(options); - await this.publishDeviceCaptureEngine(options, effectivePublishOptions, publishDimensions); - const audioPublished = await this.publishDeviceCaptureAudio(options); - await this.finalizeDeviceCaptureSuccess( - options, - effectivePublishOptions, - publishDimensions, - sendUpdate, - playSound, - audioPublished, - ); - } catch (error) { - logger.warn('Failed to start native-engine device screen share', {error, videoDeviceId: options?.videoDeviceId}); - await this.stopCaptureDirect({ - sendUpdate, - playSound: false, - reason: 'native-engine-device-screen-share-failed', - }).catch((stopError) => { - logger.warn('Failed to stop native-engine device screen share after start failure', {error: stopError}); - }); - this.adapter.transitionScreenShareLifecycleInternal({ - type: 'share.reject', - active: LocalVoiceState.getSelfStream(), - sourceType: LocalVoiceState.getSelfStream() ? 'device' : null, - }); - throw error; - } - } - - async stopCapture(options?: {sendUpdate?: boolean; playSound?: boolean; reason?: string}): Promise { - const {sendUpdate = true, playSound = true, reason = 'native-engine-screen-share-stopped'} = options || {}; - assert.equal(typeof sendUpdate, 'boolean'); - assert.equal(typeof reason, 'string'); - if (this.routedStopDepth > 0) { - await this.stopCaptureDirect({sendUpdate, playSound, reason}); - return; - } - if (!this.adapter.controllerRouting.isStopRoutable()) { - await this.stopCaptureDirect({sendUpdate, playSound, reason}); - return; - } - this.routedStopDepth += 1; - try { - const routedOptions: VoiceEngineV2AppScreenShareSetEnabledOptions = {sendUpdate, playSound, reason}; - await this.adapter.setScreenShareEnabled(null, false, routedOptions); - } finally { - this.routedStopDepth -= 1; - assert.ok(this.routedStopDepth >= 0, 'routed stop depth must not go negative'); - } - } - - async stopCaptureDirect(options?: {sendUpdate?: boolean; playSound?: boolean; reason?: string}): Promise { - const {sendUpdate = true, playSound = true, reason = 'native-engine-screen-share-stopped'} = options || {}; - assert.equal(typeof sendUpdate, 'boolean'); - assert.equal(typeof reason, 'string'); - await this.releaseResources({reason}); - applyScreenShareState(this.adapter, false, sendUpdate, sendUpdate); - LocalVoiceState.updateSelfStreamAudio(false); - this.adapter.syncLocalStreamWatchStateInternal(false); - if (playSound) { - SoundCommands.playSound(SoundType.ScreenShareStop); - } - } - - async startDisplayCapture( - _room: Room | null, - nativeOptions: NativeScreenShareOptions, - options?: { - sendUpdate?: boolean; - playSound?: boolean; - }, - publishOptions?: TrackPublishOptions, - ): Promise { - assert.ok(nativeOptions); - assert.ok(nativeOptions.source); - if (guardScreenShareEntry({platformUnsupportedWarning: SCREEN_SHARE_UNSUPPORTED_PLATFORM_WARNING}) !== 'proceed') { - return; - } - await this.assertNativeBridgeAvailable('desktop screen share capture'); - const nextOptions = await this.withAudioIfRequested( - nativeOptions, - this.shouldIncludeAudioFromSettings(nativeOptions.source.kind), - ); - if (!nextOptions) throw new Error('Native screen-share audio unavailable'); - await this.startCapture(nextOptions, options, publishOptions); - } - - private async releaseForSourceSwitch(): Promise { - if (this.activeCaptureId) { - await this.releaseResources({ - reason: 'native-engine-screen-share-source-replaced', - releaseStreamingPriority: false, - preserveStreamStateOnUnpublish: true, - }); - return true; - } - if (LocalVoiceState.getSelfStream()) { - await this.stopCaptureDirect({ - sendUpdate: false, - playSound: false, - reason: 'native-engine-screen-share-source-replaced', - }); - return true; - } - logger.warn('No active native-engine screen share to replace'); - return false; - } - - private handleReplaceDisplayCaptureFailure(nativeOptions: NativeScreenShareOptions, error: unknown): void { - assert.ok(nativeOptions); - const isCancel = isUserCancelledOrPermissionDeniedError(error); - if (isCancel) { - logger.debug('User cancelled or denied native screen share source switch', {name: (error as Error).name}); - } else { - logger.error('Failed to replace active native display screen share source', { - error, - sourceKind: nativeOptions.source.kind, - }); - } - const active = LocalVoiceState.getSelfStream(); - const sourceType = active ? this.adapter.getNativeScreenShareSourceType(nativeOptions) : null; - this.adapter.transitionScreenShareLifecycleInternal( - buildScreenShareFailureTransition({cancelled: isCancel, active, sourceType}), - ); - } - - async replaceActiveDisplayCapture( - _room: Room | null, - nativeOptions: NativeScreenShareOptions, - options?: ScreenShareCaptureOptions, - publishOptions?: TrackPublishOptions, - ): Promise { - assert.ok(nativeOptions); - assert.ok(nativeOptions.source); - const entryVerdict = guardScreenShareEntry({ - platformUnsupportedWarning: SCREEN_SHARE_SOURCE_SWITCH_UNSUPPORTED_PLATFORM_WARNING, - pending: { - active: this.adapter.isScreenSharePending, - debugMessage: 'Already pending, ignoring native screen share source switch', - }, - }); - if (entryVerdict !== 'proceed') { - return false; - } - await this.assertNativeBridgeAvailable('desktop screen share source switching'); - try { - const proceed = await this.releaseForSourceSwitch(); - if (!proceed) return false; - const nextOptions = await this.withAudioIfRequested( - { - ...nativeOptions, - resolution: nativeOptions.resolution ?? options?.resolution, - contentHint: nativeOptions.contentHint ?? options?.contentHint, - }, - this.shouldIncludeAudioFromSettings(nativeOptions.source.kind), - ); - if (!nextOptions) throw new Error('Native screen-share audio unavailable'); - await this.startCapture(nextOptions, {sendUpdate: true, playSound: false}, publishOptions); - return true; - } catch (error) { - this.handleReplaceDisplayCaptureFailure(nativeOptions, error); - return false; - } - } - - async replaceActiveDeviceCapture( - options?: DeviceScreenShareCaptureOptions, - publishOptions?: TrackPublishOptions, - ): Promise { - assert.ok(options === undefined || typeof options === 'object'); - const entryVerdict = guardScreenShareEntry({ - platformUnsupportedWarning: SCREEN_SHARE_SOURCE_SWITCH_UNSUPPORTED_PLATFORM_WARNING, - pending: { - active: this.adapter.isScreenSharePending, - debugMessage: 'Already pending, ignoring native device screen share source switch', - }, - }); - if (entryVerdict !== 'proceed') { - return false; - } - await this.assertNativeBridgeAvailable('device screen share source switching'); - if (!LocalVoiceState.getSelfStream()) { - logger.warn('No active native-engine screen share to replace'); - return false; - } - try { - await this.stopCaptureDirect({ - sendUpdate: false, - playSound: false, - reason: 'native-engine-device-screen-share-source-replaced', - }); - await this.startDeviceCapture(options, {sendUpdate: true, playSound: false}, publishOptions); - return LocalVoiceState.getSelfStream(); - } catch (error) { - logger.error('Failed to replace active native device screen share source', {error}); - this.adapter.transitionScreenShareLifecycleInternal({ - type: 'share.reject', - active: LocalVoiceState.getSelfStream(), - sourceType: LocalVoiceState.getSelfStream() ? 'device' : null, - }); - return false; - } - } - - async replaceActiveDisplayFromActiveSource( - options?: ScreenShareCaptureOptions, - publishOptions?: TrackPublishOptions, - ): Promise { - assert.ok(options === undefined || typeof options === 'object'); - const nativeOptions = await this.resolveOptionsFromActiveSource(options); - if (!nativeOptions) { - logger.warn('Native voice engine: no desktop source selected for screen-share source switch'); - return false; - } - return this.replaceActiveDisplayCapture(null, nativeOptions, options, publishOptions); - } -} diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareCodecMigration.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareCodecMigration.ts index e0c21df69..ca7e90b8c 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareCodecMigration.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareCodecMigration.ts @@ -24,10 +24,6 @@ import { type VideoCodec, } from 'livekit-client'; -function isVideoCodecValue(value: unknown): value is VideoCodec { - return value === 'av1' || value === 'h265' || value === 'h264' || value === 'vp9' || value === 'vp8'; -} - interface MigrationContext { readonly room: Room; readonly participant: LocalParticipant; @@ -115,9 +111,6 @@ export class VoiceEngineV2AppScreenShareCodecMigration { logger.warn('Screen share updates are not supported on native'); return false; } - if (this.adapter.captureCoordinator.activeCaptureId != null) { - return this.adapter.captureCoordinator.updateActiveSettings(room, options, publishOptions); - } const participant = room?.localParticipant; if (!participant || !participant.isScreenShareEnabled) return false; const screenSharePublication = participant.getTrackPublication(Track.Source.ScreenShare); @@ -527,45 +520,6 @@ export class VoiceEngineV2AppScreenShareCodecMigration { return nextPublishOptions; } - private async renegotiateNativeActiveCodec( - room: Room | null, - codec: VideoCodec, - reason: NegotiationReason, - options: {force?: boolean}, - ): Promise { - if (!this.adapter.captureCoordinator.activeCaptureId) return false; - if (this.adapter.isScreenSharePending) { - this.adapter.queuePendingCodecRepublishRequestInternal(codec, reason, options); - return false; - } - const previousOptions = this.adapter.captureCoordinator.activeCapturePublishOptions ?? {}; - const currentCodec = isVideoCodecValue(previousOptions.videoCodec) ? previousOptions.videoCodec : undefined; - const decision = selectScreenShareCodecRepublishDecision({ - currentCodec, - nextCodec: codec, - reason, - force: options.force, - allowLiveRepublish: options.force === true, - }); - if (decision.action === 'noop') return false; - if (decision.action === 'defer') { - this.adapter.deferActiveCodecRepublishRequestInternal(codec, reason, options); - logger.info('Deferring negotiated native screen share codec change until the next share start', { - currentCodec, - codec, - reason, - }); - return false; - } - const nextPublishOptions = this.buildCodecRenegotiationPublishOptions( - previousOptions, - codec, - currentCodec, - options.force === true, - ); - return this.adapter.captureCoordinator.updateActiveSettings(room, undefined, nextPublishOptions); - } - async renegotiateActiveCodec( room: Room | null, codec: VideoCodec, @@ -575,9 +529,6 @@ export class VoiceEngineV2AppScreenShareCodecMigration { assert.equal(typeof codec, 'string'); assert.ok(reason !== undefined, 'reason required'); if (Platform.OS !== 'web') return false; - if (this.adapter.captureCoordinator.activeCaptureId != null) { - return this.renegotiateNativeActiveCodec(room, codec, reason, options); - } const participant = room?.localParticipant; if (!room) return false; if (!participant) return false; diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareControllerRouting.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareControllerRouting.ts index 71ed0e544..96467b9a8 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareControllerRouting.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareControllerRouting.ts @@ -80,12 +80,21 @@ interface PendingScreenShareStopRequest { readonly room: Room | null; readonly sendUpdate: boolean; readonly playSound: boolean; - readonly reason: string | null; operationIds: ReadonlyArray; verbSettled: boolean; failure: unknown; } +function createScreenShareCaptureId(): string { + const cryptoPort = globalThis.crypto; + if (!cryptoPort || typeof cryptoPort.randomUUID !== 'function') { + throw new Error('Screen-share capture ID generation requires crypto.randomUUID'); + } + const captureId = cryptoPort.randomUUID(); + assert.ok(captureId.length > 0, 'crypto.randomUUID must return a non-empty capture ID'); + return captureId; +} + function buildScreenSharePublishInactiveError(): Error { const error = new Error('Screen-share publish settled without an active share'); error.name = SCREEN_SHARE_PUBLISH_INACTIVE_ERROR_NAME; @@ -226,7 +235,7 @@ export class VoiceEngineV2AppScreenShareControllerRouting { } this.ensurePublishRequestCapacity(); const request: PendingScreenSharePublishRequest = { - captureId: this.adapter.captureCoordinator.createCaptureId(), + captureId: createScreenShareCaptureId(), room, options, publishOptions, @@ -296,7 +305,6 @@ export class VoiceEngineV2AppScreenShareControllerRouting { room, sendUpdate: options?.sendUpdate ?? true, playSound: options?.playSound ?? true, - reason: options?.reason ?? null, operationIds: [], verbSettled: false, failure: null, @@ -448,43 +456,4 @@ export class VoiceEngineV2AppScreenShareControllerRouting { throw error; } } - - async publishViaNativeCapture(options: VoiceEngineV2ScreenOptions): Promise { - assert.ok(options.captureId.length > 0, 'native screen publish requires a captureId'); - const request = this.takePublishRequest(options.captureId); - if (request === null) { - await this.adapter.captureCoordinator.publishControllerScreen(options); - return; - } - try { - await this.adapter.executeNativeControllerScreenSharePublish( - request.captureId, - request.options, - request.publishOptions, - ); - } catch (error) { - request.failure = error; - throw error; - } - if (!LocalVoiceState.getSelfStream()) { - request.settledInactive = true; - throw buildScreenSharePublishInactiveError(); - } - } - - async unpublishViaNativeCapture(): Promise { - const request = this.takeStopRequest(this.executingScreenOperationId()); - const fallbackReason = - request !== null ? 'native-engine-screen-share-disabled' : 'voice-engine-v2-controller-unpublish'; - try { - await this.adapter.captureCoordinator.stopCaptureDirect({ - sendUpdate: request?.sendUpdate ?? true, - playSound: request?.playSound ?? true, - reason: request?.reason ?? fallbackReason, - }); - } catch (error) { - if (request !== null) request.failure = error; - throw error; - } - } } diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx index e5f6dbc2d..ddefb42bf 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter.tsx @@ -9,8 +9,6 @@ import * as SoundCommands from '@app/features/ui/commands/SoundCommands'; import {getElectronAPI} from '@app/features/ui/utils/NativeUtils'; import {getStreamKey} from '@app/features/voice/components/StreamKeys'; import AdaptiveScreenShareEngine from '@app/features/voice/engine/AdaptiveScreenShareEngine'; -import type {NativeVoiceEngineLocalTrackPublication} from '@app/features/voice/engine/native_voice_engine/nativeVoiceEngineEventMapper'; -import {markScreenShareCaptureEnded} from '@app/features/voice/engine/ScreenShareCaptureDiagnostics'; import type {NegotiationReason} from '@app/features/voice/engine/ScreenShareCodecNegotiation'; import ScreenShareCodecNegotiation from '@app/features/voice/engine/ScreenShareCodecNegotiation'; import {Store} from '@app/features/voice/engine/Store'; @@ -31,31 +29,19 @@ import { getLocalScreenShareVideoPublications, unpublishLocalMediaPublications, } from '@app/features/voice/engine/VoiceTrackPublicationUtils'; -import {isVoiceEngineV2AppNativeScreenShareBridgeAvailable} from '@app/features/voice/engine/v2/VoiceEngineV2AppNativeBridge'; -import {VoiceEngineV2AppScreenShareAudioPump} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump'; -import {VoiceEngineV2AppScreenShareCaptureCoordinator} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareCaptureCoordinator'; import {VoiceEngineV2AppScreenShareCodecMigration} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareCodecMigration'; import { selectVoiceEngineV2AppScreenShareSetEnabledOptions, type VoiceEngineV2AppScreenShareControllerGateway, VoiceEngineV2AppScreenShareControllerRouting, - type VoiceEngineV2AppScreenShareSetEnabledSelection, } from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareControllerRouting'; import {VoiceEngineV2AppScreenShareLiveKitFlows} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareLiveKitFlows'; -import {VoiceEngineV2AppScreenSharePreviewTracking} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenSharePreviewTracking'; -import { - type NativeScreenShareReconnectSnapshot, - VoiceEngineV2AppScreenShareReconnect, -} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareReconnect'; import { applyVoiceEngineV2AppScreenShareState, type VoiceScreenShareStateOptions, } from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareStateSync'; import {VoiceEngineV2AppScreenShareTrackPlumbing} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareTrackPlumbing'; -import {routeVoiceEngineV2AppSelectedMedia} from '@app/features/voice/engine/v2/VoiceEngineV2AppSelectedMediaMode'; import type {VoiceEngineV2AppSourceLifecycleBridge} from '@app/features/voice/engine/v2/VoiceEngineV2AppSourceLifecycleBridge'; -import type {NativeScreenShareOptions} from '@app/features/voice/engine/voice_screen_share_manager/DisplayMediaCapture'; -import type {NativeEngineAudioTrackFrame} from '@app/features/voice/engine/voice_screen_share_manager/NativeEngineAudioTrackPump'; import { captureScreenSharePublicationCleanup, type DeviceScreenShareCaptureOptions, @@ -85,7 +71,6 @@ import { captureNativeAudioTrackForLinuxRouting, commitNativeAudioBridgeReplacement, disarmNativeAudio, - type NativeAudioFramePump, } from '@app/features/voice/utils/NativeAudioCaptureBridge'; import type {NativeAudioStartOptions} from '@app/types/electron.d'; import type {VoiceEngineV2ScreenOptions} from '@fluxer/voice_engine_v2'; @@ -102,7 +87,6 @@ import { type VideoCodec, } from 'livekit-client'; -export type {NativeScreenShareReconnectSnapshot} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareReconnect'; export type {DeviceScreenShareCaptureOptions} from '@app/features/voice/engine/voice_screen_share_manager/shared'; const SCREEN_SHARE_ENDED_MODAL_KEY = 'voice-screen-share-ended'; @@ -126,19 +110,6 @@ export interface ScreenShareReconnectSnapshot { contentSource: ScreenShareContentSource; } -type ScreenShareSetEnabledSelection = VoiceEngineV2AppScreenShareSetEnabledSelection; - -interface ScreenShareStartControls { - sendUpdate: boolean; - playSound: boolean; -} - -function buildNativeScreenSharePublicationError(message: string): Error { - const error = new Error(message); - error.name = 'VoiceEngineV2AppNativeScreenSharePublicationError'; - return error; -} - const selectScreenShareSetEnabledOptions = selectVoiceEngineV2AppScreenShareSetEnabledOptions; class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { @@ -146,20 +117,11 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { private readonly trackPlumbing: VoiceEngineV2AppScreenShareTrackPlumbing; private activeScreenShareEndListener: (() => void) | null = null; private endedScreenShareStopInFlight: Promise | null = null; - nativeEngineScreenSharePreviewTrackSid: string | null = null; - nativeEngineScreenSharePreviewStartToken = 0; - nativeEngineScreenShareAudioPump: NativeAudioFramePump | null = null; encoderVerificationTimer: NodeJS.Timeout | null = null; screenShareMigrationGeneration = 0; - nativeEngineScreenShareEndDisposer: (() => void) | null = null; - nativeEngineScreenShareLifecycleBoundCaptureId: string | null = null; sourceLifecycleBridge: VoiceEngineV2AppSourceLifecycleBridge | null = null; - readonly audioPump: VoiceEngineV2AppScreenShareAudioPump; - readonly previewTracking: VoiceEngineV2AppScreenSharePreviewTracking; - readonly captureCoordinator: VoiceEngineV2AppScreenShareCaptureCoordinator; readonly codecMigration: VoiceEngineV2AppScreenShareCodecMigration; - readonly reconnect: VoiceEngineV2AppScreenShareReconnect; readonly liveKitFlows: VoiceEngineV2AppScreenShareLiveKitFlows; readonly controllerRouting: VoiceEngineV2AppScreenShareControllerRouting; @@ -169,11 +131,7 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { this.trackPlumbing = new VoiceEngineV2AppScreenShareTrackPlumbing({ getActiveContentSource: () => this.getActiveScreenShareContentSourceInternal(), }); - this.audioPump = new VoiceEngineV2AppScreenShareAudioPump(this); - this.previewTracking = new VoiceEngineV2AppScreenSharePreviewTracking(this); - this.captureCoordinator = new VoiceEngineV2AppScreenShareCaptureCoordinator(this); this.codecMigration = new VoiceEngineV2AppScreenShareCodecMigration(this); - this.reconnect = new VoiceEngineV2AppScreenShareReconnect(this); this.liveKitFlows = new VoiceEngineV2AppScreenShareLiveKitFlows(this); this.controllerRouting = new VoiceEngineV2AppScreenShareControllerRouting(this); } @@ -194,55 +152,11 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { return this.isScreenSharePending; } - syncNativeEngineScreenSharePublishedTrackSidInternal( - enabled: boolean, - trackSid: string | undefined, - publication?: NativeVoiceEngineLocalTrackPublication, - ): void { - assert.equal(typeof enabled, 'boolean'); - if (enabled && trackSid) { - this.captureCoordinator.recordPublishedTrackSid(trackSid, publication); - } - if (!enabled && (!trackSid || this.captureCoordinator.activeCapturePublishedTrackSid === trackSid)) { - this.captureCoordinator.activeCapturePublishedTrackSid = null; - } - } - - adoptNativeEngineScreenShareOptionsInternal(nextOptions: NativeScreenShareOptions): void { - assert.ok(nextOptions, 'nextOptions is required'); - assert.ok(nextOptions.source, 'nextOptions.source is required'); - this.captureCoordinator.activeCaptureOptions = nextOptions; - } - setSourceLifecycleBridge(bridge: VoiceEngineV2AppSourceLifecycleBridge | null): void { this.sourceLifecycleBridge = bridge; } - hasActiveScreenShareResources(): boolean { - return Boolean( - this.captureCoordinator.activeCaptureId || - this.captureCoordinator.deviceCaptureActive || - this.nativeEngineScreenShareAudioPump || - this.nativeEngineScreenSharePreviewTrackSid, - ); - } - stopNativeScreenShareForTerminalUnload(): void { - const captureId = this.captureCoordinator.activeCaptureId; - this.captureCoordinator.activeCaptureId = null; - this.captureCoordinator.activeCapturePublishedTrackSid = null; - if (captureId) { - try { - void getElectronAPI() - ?.nativeScreenCapture?.stop(captureId) - ?.catch((error) => { - logger.warn('Failed to stop native screen capture during terminal unload', {captureId, error}); - }); - } catch (error) { - logger.warn('Failed to stop native screen capture during terminal unload', {captureId, error}); - } - markScreenShareCaptureEnded('terminal-unload'); - } if (this.streamingPriorityHeld) { this.transitionScreenShareLifecycleInternal({type: 'share.streamingPriority.set', active: false}); try { @@ -346,10 +260,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { return 'display'; } - getNativeScreenShareSourceType(nativeOptions: NativeScreenShareOptions): VoiceScreenShareSourceType { - return nativeOptions.source.kind === 'window' ? 'native-app' : 'native-display'; - } - getScreenShareSourceTypeForContentSourceInternal( contentSource: ScreenShareContentSource, ): VoiceScreenShareSourceType { @@ -632,14 +542,7 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { snapshot: ScreenShareReconnectSnapshot, publishOptions?: TrackPublishOptions, ): Promise { - return routeVoiceEngineV2AppSelectedMedia({ - js: () => this.liveKitFlows.restoreReconnect(room, snapshot, publishOptions), - native: async () => { - throw buildNativeScreenSharePublicationError( - 'Native voice engine selected; refusing browser screen-share reconnect publication', - ); - }, - }); + return this.liveKitFlows.restoreReconnect(room, snapshot, publishOptions); } async cleanupLingeringScreenShareTracks( @@ -808,21 +711,17 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { ): Promise { assert.equal(typeof enabled, 'boolean'); const selection = selectScreenShareSetEnabledOptions(options); - await routeVoiceEngineV2AppSelectedMedia({ - js: () => - this.liveKitFlows.setEnabled( - room, - enabled, - { - ...selection.captureOptions, - sendUpdate: selection.sendUpdate, - playSound: selection.playSound, - restartIfEnabled: selection.restartIfEnabled, - }, - publishOptions, - ), - native: () => this.setNativeSelectedScreenShareEnabled(enabled, selection, publishOptions), - }); + await this.liveKitFlows.setEnabled( + room, + enabled, + { + ...selection.captureOptions, + sendUpdate: selection.sendUpdate, + playSound: selection.playSound, + restartIfEnabled: selection.restartIfEnabled, + }, + publishOptions, + ); } async startDeviceScreenShare( @@ -830,14 +729,7 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { options?: DeviceScreenShareCaptureOptions, publishOptions?: TrackPublishOptions, ): Promise { - await routeVoiceEngineV2AppSelectedMedia({ - js: () => this.liveKitFlows.startDeviceScreenShare(room, options, publishOptions), - native: async () => { - const {sendUpdate = true, playSound = true} = options || {}; - await this.assertNativeScreenShareBridgeAvailable('device screen-share capture'); - await this.captureCoordinator.startDeviceCapture(options, {sendUpdate, playSound}, publishOptions); - }, - }); + await this.liveKitFlows.startDeviceScreenShare(room, options, publishOptions); } async replaceActiveDisplayScreenShare( @@ -845,13 +737,7 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { options?: ScreenShareCaptureOptions, publishOptions?: TrackPublishOptions, ): Promise { - return routeVoiceEngineV2AppSelectedMedia({ - js: () => this.liveKitFlows.replaceActiveDisplayShare(room, options, publishOptions), - native: async () => { - await this.assertNativeScreenShareBridgeAvailable('desktop screen-share source switch'); - return this.captureCoordinator.replaceActiveDisplayFromActiveSource(options, publishOptions); - }, - }); + return this.liveKitFlows.replaceActiveDisplayShare(room, options, publishOptions); } async replaceActiveDeviceScreenShare( @@ -859,77 +745,7 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { options?: DeviceScreenShareCaptureOptions, publishOptions?: TrackPublishOptions, ): Promise { - return routeVoiceEngineV2AppSelectedMedia({ - js: () => this.liveKitFlows.replaceActiveDeviceShare(room, options, publishOptions), - native: async () => { - await this.assertNativeScreenShareBridgeAvailable('device screen-share source switch'); - return this.captureCoordinator.replaceActiveDeviceCapture(options, publishOptions); - }, - }); - } - - private async assertNativeScreenShareBridgeAvailable(context: string): Promise { - assert.equal(typeof context, 'string'); - if (await isVoiceEngineV2AppNativeScreenShareBridgeAvailable()) return; - logger.error('Native voice engine selected without native screen-share bridge support', {context}); - throw buildNativeScreenSharePublicationError( - `Native voice engine selected but native screen-share bridge is unavailable for ${context}`, - ); - } - - private async setNativeSelectedScreenShareEnabled( - enabled: boolean, - selection: ScreenShareSetEnabledSelection, - publishOptions?: TrackPublishOptions, - ): Promise { - assert.equal(typeof enabled, 'boolean'); - assert.ok(selection.captureOptions !== null && typeof selection.captureOptions === 'object'); - assert.equal(typeof selection.restartIfEnabled, 'boolean'); - if (!enabled) { - await this.captureCoordinator.stopCapture({ - sendUpdate: selection.sendUpdate, - playSound: selection.playSound, - reason: selection.reason ?? 'native-engine-screen-share-disabled', - }); - return; - } - await this.startNativeSelectedScreenShare(this.captureCoordinator.createCaptureId(), selection, publishOptions); - } - - private async startNativeSelectedScreenShare( - captureId: string, - selection: ScreenShareSetEnabledSelection, - publishOptions?: TrackPublishOptions, - ): Promise { - assert.ok(captureId.length > 0, 'native screen-share enable requires a captureId'); - assert.ok(selection.captureOptions !== null && typeof selection.captureOptions === 'object'); - const controls: ScreenShareStartControls = { - sendUpdate: selection.sendUpdate, - playSound: selection.playSound, - }; - await this.assertNativeScreenShareBridgeAvailable('desktop screen-share capture'); - const nativeOptions = await this.captureCoordinator.resolveOptionsFromActiveSource(selection.captureOptions); - if (!nativeOptions) { - logger.warn('Native voice engine selected without a desktop source for screen-share enable'); - throw buildNativeScreenSharePublicationError( - 'Native voice engine selected but no desktop source is selected for screen-share capture', - ); - } - await this.captureCoordinator.startCapture({...nativeOptions, captureId}, controls, publishOptions); - } - - async executeNativeControllerScreenSharePublish( - captureId: string, - options?: ScreenShareCaptureOptions & { - sendUpdate?: boolean; - playSound?: boolean; - restartIfEnabled?: boolean; - reason?: string; - }, - publishOptions?: TrackPublishOptions, - ): Promise { - assert.ok(captureId.length > 0, 'native controller screen-share publish requires a captureId'); - await this.startNativeSelectedScreenShare(captureId, selectScreenShareSetEnabledOptions(options), publishOptions); + return this.liveKitFlows.replaceActiveDeviceShare(room, options, publishOptions); } async applyPendingScreenShareRequestsForRoom(room: Room | null): Promise { @@ -938,65 +754,10 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { await this.applyPendingScreenShareRequestsInternal(room, participant); } - async releaseNativeEngineScreenShareResourcesInternal(args: { - reason: string; - preserveRestoreState?: boolean; - releaseStreamingPriority?: boolean; - unpublishRemote?: boolean; - preserveStreamStateOnUnpublish?: boolean; - }): Promise { - await this.captureCoordinator.releaseResources(args); - } - - async startNativeEngineScreenShareInternal( - nativeOptions: NativeScreenShareOptions, - options?: {sendUpdate?: boolean; playSound?: boolean}, - publishOptions?: TrackPublishOptions, - ): Promise { - await this.captureCoordinator.startCapture(nativeOptions, options, publishOptions); - } - - async stopNativeEngineScreenShareInternal(options?: { - sendUpdate?: boolean; - playSound?: boolean; - reason?: string; - }): Promise { - await this.captureCoordinator.stopCapture(options); - } - stopMediaTrackInternal(track: MediaStreamTrack | undefined): void { stopMediaTrack(track); } - prepareNativeScreenShareReconnect(): NativeScreenShareReconnectSnapshot | null { - return this.reconnect.prepare(); - } - - async releaseNativeScreenShareForReconnect(snapshot: NativeScreenShareReconnectSnapshot): Promise { - await this.reconnect.release(snapshot); - } - - async restoreNativeScreenShareReconnect(snapshot: NativeScreenShareReconnectSnapshot): Promise { - return this.reconnect.restore(snapshot); - } - - async startNativeDeviceScreenShare( - options?: DeviceScreenShareCaptureOptions, - startOptions?: {sendUpdate?: boolean; playSound?: boolean}, - publishOptions?: TrackPublishOptions, - ): Promise { - await this.captureCoordinator.startDeviceCapture(options, startOptions, publishOptions); - } - - async startNativeDisplayScreenShare( - room: Room | null, - nativeOptions: NativeScreenShareOptions, - options?: {sendUpdate?: boolean; playSound?: boolean}, - publishOptions?: TrackPublishOptions, - ): Promise { - await this.captureCoordinator.startDisplayCapture(room, nativeOptions, options, publishOptions); - } - async publishControllerScreenViaLiveKitFlows(room: Room | null, options: VoiceEngineV2ScreenOptions): Promise { if (typeof options.captureId !== 'string' || options.captureId.length === 0) { throw new Error('Controller screen-share publish requires a captureId'); @@ -1008,40 +769,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { await this.controllerRouting.unpublishViaLiveKitFlows(room); } - async publishControllerScreenViaNativeCapture(options: VoiceEngineV2ScreenOptions): Promise { - if (typeof options.captureId !== 'string' || options.captureId.length === 0) { - throw new Error('Controller screen-share publish requires a captureId'); - } - await this.controllerRouting.publishViaNativeCapture(options); - } - - async unpublishControllerScreenViaNativeCapture(): Promise { - await this.controllerRouting.unpublishViaNativeCapture(); - } - - async replaceActiveNativeDisplayScreenShare( - room: Room | null, - nativeOptions: NativeScreenShareOptions, - options?: ScreenShareCaptureOptions, - publishOptions?: TrackPublishOptions, - ): Promise { - return this.captureCoordinator.replaceActiveDisplayCapture(room, nativeOptions, options, publishOptions); - } - - async replaceActiveNativeDisplayScreenShareFromActiveSource( - options?: ScreenShareCaptureOptions, - publishOptions?: TrackPublishOptions, - ): Promise { - return this.captureCoordinator.replaceActiveDisplayFromActiveSource(options, publishOptions); - } - - async replaceActiveNativeDeviceScreenShare( - options?: DeviceScreenShareCaptureOptions, - publishOptions?: TrackPublishOptions, - ): Promise { - return this.captureCoordinator.replaceActiveDeviceCapture(options, publishOptions); - } - async updateActiveScreenShareSettings( room: Room | null, options?: ScreenShareCaptureOptions, @@ -1061,24 +788,15 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { setScreenShareAudioMuted(room: Room | null, muted: boolean): void { assert.equal(typeof muted, 'boolean'); - void routeVoiceEngineV2AppSelectedMedia({ - js: async () => { - const participant = room?.localParticipant; - if (!participant) return; - const publication = participant.getTrackPublication(Track.Source.ScreenShareAudio); - if (!publication) return; - const operation = muted ? publication.mute() : publication.unmute(); - operation.catch((error) => { - logger.warn('Failed to apply immediate screen share audio mute', {error, muted}); - }); - this.syncLocalScreenShareAudioStateInternal(participant, !muted); - }, - native: async () => { - await this.captureCoordinator.updateActiveSettings(room, {audio: !muted}).catch((error) => { - logger.warn('Failed to apply native-engine screen share audio mute', {error, muted}); - }); - }, + const participant = room?.localParticipant; + if (!participant) return; + const publication = participant.getTrackPublication(Track.Source.ScreenShareAudio); + if (!publication) return; + const operation = muted ? publication.mute() : publication.unmute(); + operation.catch((error) => { + logger.warn('Failed to apply immediate screen share audio mute', {error, muted}); }); + this.syncLocalScreenShareAudioStateInternal(participant, !muted); } async toggleScreenShareFromKeybind(room: Room | null): Promise { @@ -1087,13 +805,6 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { } resetStreamTracking(): void { - void this.captureCoordinator - .releaseResources({ - reason: 'native-engine-screen-share-reset', - }) - .catch((error) => { - logger.warn('Failed to release native-engine screen-share resources during reset', {error}); - }); this.clearScreenShareKeepAliveSinkInternal(); this.setStreamingPriorityInternal(false); this.transitionScreenShareLifecycleInternal({type: 'share.reset'}); @@ -1108,5 +819,3 @@ class VoiceEngineV2AppScreenShareExecutionAdapter extends Store { export {VoiceEngineV2AppScreenShareExecutionAdapter}; export default new VoiceEngineV2AppScreenShareExecutionAdapter(); - -export type {NativeEngineAudioTrackFrame}; diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenSharePreviewTracking.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenSharePreviewTracking.ts deleted file mode 100644 index 18fdc3083..000000000 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenSharePreviewTracking.ts +++ /dev/null @@ -1,252 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; -import Users from '@app/features/user/state/Users'; -import NativeVideoTileManager from '@app/features/voice/engine/native_voice_engine/NativeVideoTileManager'; -import { - getVoiceConnectionContextFromMediaEngine, - getVoiceEngineV2SnapshotFromMediaEngine, -} from '@app/features/voice/engine/VoiceMediaEngineBridge'; -import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource'; -import voiceEngineV2AppConnectionHostAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter'; -import type {VoiceEngineV2AppScreenShareExecutionAdapter} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter'; -import {selectVoiceEngineV2AppParticipants} from '@app/features/voice/engine/v2/VoiceEngineV2AppSelectors'; -import type {NativeEngineScreenCapture} from '@app/features/voice/engine/voice_screen_share_manager/DisplayMediaCapture'; -import { - type DeviceScreenShareCaptureOptions, - logger, -} from '@app/features/voice/engine/voice_screen_share_manager/shared'; -import {buildVoiceParticipantIdentity} from '@app/features/voice/utils/VoiceParticipantIdentity'; -import {createLocalVideoTrack, type LocalVideoTrack} from 'livekit-client'; - -const NATIVE_SCREEN_SHARE_PREVIEW_RETRY_DELAY_MS = 250; -const NATIVE_SCREEN_SHARE_PREVIEW_MAX_ATTEMPTS = 8; -const NATIVE_SCREEN_SHARE_PREVIEW_READY_TIMEOUT_MS = 2000; - -interface NativeScreenSharePreviewParticipant { - identity: string; - sid: string; -} - -type NativeScreenSharePreviewParticipantCandidate = NativeScreenSharePreviewParticipant & { - isLocal?: boolean; -}; - -function delay(ms: number): Promise { - assert.ok(Number.isFinite(ms) && ms >= 0, 'ms must be non-negative finite'); - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -export class VoiceEngineV2AppScreenSharePreviewTracking { - private readonly adapter: VoiceEngineV2AppScreenShareExecutionAdapter; - - constructor(adapter: VoiceEngineV2AppScreenShareExecutionAdapter) { - this.adapter = adapter; - } - - registerCapturePreview(capture: NativeEngineScreenCapture): void { - assert.ok(capture, 'capture required'); - assert.equal(typeof capture.captureId, 'string', 'capture.captureId must be string'); - this.clearPreview(); - const token = this.adapter.nativeEngineScreenSharePreviewStartToken; - void this.attachCapturePreview(capture, token); - } - - registerDevicePreview( - options: DeviceScreenShareCaptureOptions | undefined, - dimensions: {width: number; height: number; frameRate: number}, - ): void { - assert.ok(dimensions, 'dimensions required'); - assert.ok(dimensions.width > 0, 'dimensions.width must be > 0'); - this.clearPreview(); - const token = this.adapter.nativeEngineScreenSharePreviewStartToken; - void this.attachDevicePreview(options, dimensions, token); - } - - clearPreview(): void { - this.adapter.nativeEngineScreenSharePreviewStartToken++; - const trackSid = this.adapter.nativeEngineScreenSharePreviewTrackSid; - this.adapter.nativeEngineScreenSharePreviewTrackSid = null; - assert.equal(this.adapter.nativeEngineScreenSharePreviewTrackSid, null, 'trackSid must be null after clearPreview'); - if (trackSid) { - NativeVideoTileManager.unregisterTrack(trackSid); - } - } - - private async cleanupUnattachedPreview(capture: NativeEngineScreenCapture, reason: string): Promise { - await capture.previewBridge.cleanup(false).catch((error) => { - logger.warn('Failed to clean up unattached native-engine screen-share preview', { - captureId: capture.captureId, - reason, - error, - }); - }); - } - - private resolveLocalParticipant(): NativeScreenSharePreviewParticipant | null { - const snapshot = getVoiceEngineV2SnapshotFromMediaEngine(); - const participants: ReadonlyArray = snapshot - ? selectVoiceEngineV2AppParticipants(snapshot).participants - : []; - const connectionId = getVoiceConnectionContextFromMediaEngine()?.connectionId ?? null; - const currentUserId = Users.getCurrentUser()?.id; - const fallbackIdentity = - currentUserId && connectionId ? buildVoiceParticipantIdentity(currentUserId, connectionId) : null; - const localParticipant = - participants.find((participant) => participant.isLocal && participant.identity) ?? - (fallbackIdentity ? participants.find((participant) => participant.identity === fallbackIdentity) : undefined); - if (localParticipant?.identity) { - return { - identity: localParticipant.identity, - sid: localParticipant.sid || localParticipant.identity, - }; - } - if (!fallbackIdentity) return null; - return { - identity: fallbackIdentity, - sid: fallbackIdentity, - }; - } - - private waitForLocalParticipant(token: number): Promise { - assert.ok(Number.isInteger(token), 'token must be an integer'); - assert.ok(token >= 0, 'token must be non-negative'); - const immediate = this.resolveLocalParticipant(); - if (immediate) { - return Promise.resolve(immediate); - } - return new Promise((resolve) => { - let settled = false; - let unsubscribe: (() => void) | null = null; - let timeoutId: NodeJS.Timeout | null = null; - const settle = (participant: NativeScreenSharePreviewParticipant | null): void => { - if (settled) return; - settled = true; - unsubscribe?.(); - if (timeoutId != null) { - clearTimeout(timeoutId); - } - resolve(participant); - }; - const check = (): void => { - if (token !== this.adapter.nativeEngineScreenSharePreviewStartToken) { - settle(null); - return; - } - const participant = this.resolveLocalParticipant(); - if (participant) { - settle(participant); - } - }; - unsubscribe = voiceEngineV2AppConnectionHostAdapter.subscribe(check); - timeoutId = setTimeout( - () => settle(this.resolveLocalParticipant()), - NATIVE_SCREEN_SHARE_PREVIEW_READY_TIMEOUT_MS, - ); - check(); - }); - } - - private async attachCapturePreview(capture: NativeEngineScreenCapture, token: number): Promise { - const localParticipant = await this.waitForLocalParticipant(token); - if (token !== this.adapter.nativeEngineScreenSharePreviewStartToken) { - await this.cleanupUnattachedPreview(capture, 'stale-preview-registration'); - return; - } - if (!localParticipant?.identity) { - await this.cleanupUnattachedPreview(capture, 'missing-local-participant'); - logger.warn('Cannot attach native-engine screen-share preview without local participant', { - captureId: capture.captureId, - }); - return; - } - const trackSid = `native-local-screen:${capture.captureId}`; - this.adapter.nativeEngineScreenSharePreviewTrackSid = trackSid; - NativeVideoTileManager.registerLocalPreviewTrack({ - participantSid: localParticipant.sid, - participantIdentity: localParticipant.identity, - trackSid, - source: VoiceTrackSource.ScreenShare, - width: capture.width, - height: capture.height, - stream: new MediaStream([capture.previewBridge.track]), - cleanup: () => capture.previewBridge.cleanup(false), - }); - } - - private async attachDevicePreview( - options: DeviceScreenShareCaptureOptions | undefined, - dimensions: {width: number; height: number; frameRate: number}, - token: number, - ): Promise { - const localParticipant = await this.waitForLocalParticipant(token); - if (token !== this.adapter.nativeEngineScreenSharePreviewStartToken) return; - if (!localParticipant?.identity) { - logger.warn('Cannot attach native-engine device screen-share preview without local participant'); - return; - } - const track = await this.createDevicePreviewTrack(options, dimensions, token); - if (!track) return; - try { - if (token !== this.adapter.nativeEngineScreenSharePreviewStartToken) { - track.stop(); - return; - } - const settings = track.mediaStreamTrack.getSettings(); - const trackSid = `native-local-device-screen:${localParticipant.identity}`; - this.adapter.nativeEngineScreenSharePreviewTrackSid = trackSid; - NativeVideoTileManager.registerLocalPreviewTrack({ - participantSid: localParticipant.sid, - participantIdentity: localParticipant.identity, - trackSid, - source: VoiceTrackSource.ScreenShare, - width: settings.width ?? dimensions.width, - height: settings.height ?? dimensions.height, - stream: new MediaStream([track.mediaStreamTrack]), - cleanup: async () => { - track?.stop(); - }, - }); - } catch (error) { - track.stop(); - logger.warn('Failed to register native-engine device screen-share local preview', { - error, - videoDeviceId: options?.previewVideoDeviceId, - }); - } - } - - private async createDevicePreviewTrack( - options: DeviceScreenShareCaptureOptions | undefined, - dimensions: {width: number; height: number; frameRate: number}, - token: number, - ): Promise { - let lastError: unknown; - for (let attempt = 1; attempt <= NATIVE_SCREEN_SHARE_PREVIEW_MAX_ATTEMPTS; attempt++) { - if (token !== this.adapter.nativeEngineScreenSharePreviewStartToken) return null; - try { - return await createLocalVideoTrack({ - deviceId: - options?.previewVideoDeviceId && options.previewVideoDeviceId !== 'default' - ? options.previewVideoDeviceId - : undefined, - resolution: { - width: dimensions.width, - height: dimensions.height, - frameRate: dimensions.frameRate, - }, - }); - } catch (error) { - lastError = error; - if (attempt >= NATIVE_SCREEN_SHARE_PREVIEW_MAX_ATTEMPTS) break; - await delay(NATIVE_SCREEN_SHARE_PREVIEW_RETRY_DELAY_MS); - } - } - logger.warn('Failed to create native-engine device screen-share local preview', { - error: lastError, - videoDeviceId: options?.previewVideoDeviceId, - attempts: NATIVE_SCREEN_SHARE_PREVIEW_MAX_ATTEMPTS, - }); - return null; - } -} diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareReconnect.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareReconnect.ts deleted file mode 100644 index b303859b3..000000000 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppScreenShareReconnect.ts +++ /dev/null @@ -1,96 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; -import type {VoiceScreenShareSourceType} from '@app/features/voice/engine/VoiceScreenShareStateMachine'; -import type {VoiceEngineV2AppScreenShareExecutionAdapter} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareExecutionAdapter'; -import type {NativeScreenShareOptions} from '@app/features/voice/engine/voice_screen_share_manager/DisplayMediaCapture'; -import {logger} from '@app/features/voice/engine/voice_screen_share_manager/shared'; -import LocalVoiceState from '@app/features/voice/state/LocalVoiceState'; -import type {TrackPublishOptions} from 'livekit-client'; - -export interface NativeScreenShareReconnectSnapshot { - nativeOptions: NativeScreenShareOptions; - publishOptions?: TrackPublishOptions; - captureId: string | null; - width: number; - height: number; - sourceType: VoiceScreenShareSourceType; -} - -export class VoiceEngineV2AppScreenShareReconnect { - private readonly adapter: VoiceEngineV2AppScreenShareExecutionAdapter; - - constructor(adapter: VoiceEngineV2AppScreenShareExecutionAdapter) { - this.adapter = adapter; - } - - prepare(): NativeScreenShareReconnectSnapshot | null { - const captureCoordinator = this.adapter.captureCoordinator; - const nativeOptions = captureCoordinator.activeCaptureOptions; - if (!nativeOptions) return null; - assert.ok(nativeOptions.source, 'nativeOptions.source required'); - const dimensions = captureCoordinator.activeCaptureDimensions; - const snapshot: NativeScreenShareReconnectSnapshot = { - nativeOptions, - captureId: captureCoordinator.activeCaptureId, - width: dimensions?.width ?? nativeOptions.source.width, - height: dimensions?.height ?? nativeOptions.source.height, - sourceType: this.adapter.getNativeScreenShareSourceType(nativeOptions), - }; - if (captureCoordinator.activeCapturePublishOptions) { - snapshot.publishOptions = captureCoordinator.activeCapturePublishOptions; - } - assert.ok(snapshot.width > 0, 'snapshot.width must be > 0'); - return snapshot; - } - - async release(snapshot: NativeScreenShareReconnectSnapshot): Promise { - assert.ok(snapshot, 'snapshot required'); - assert.ok(snapshot.nativeOptions, 'snapshot.nativeOptions required'); - const captureId = this.adapter.captureCoordinator.activeCaptureId; - if (snapshot.captureId && captureId && snapshot.captureId !== captureId) { - logger.debug('Skipping stale native-engine screen-share reconnect release', { - snapshotCaptureId: snapshot.captureId, - currentCaptureId: captureId, - }); - return; - } - await this.adapter.releaseNativeEngineScreenShareResourcesInternal({ - reason: 'native-engine-screen-share-reconnect', - preserveRestoreState: true, - unpublishRemote: false, - }); - this.adapter.applyScreenShareStateInternal(true, {reason: 'server', sendUpdate: false}); - this.adapter.syncLocalStreamWatchStateInternal(true); - this.adapter.transitionScreenShareLifecycleInternal({ - type: 'share.resolve', - active: true, - sourceType: snapshot.sourceType, - encoderVerificationScheduled: false, - streamingPriorityHeld: this.adapter.streamingPriorityHeld, - }); - } - - async restore(snapshot: NativeScreenShareReconnectSnapshot): Promise { - assert.ok(snapshot, 'snapshot required'); - assert.ok(snapshot.nativeOptions, 'snapshot.nativeOptions required'); - if (!LocalVoiceState.getSelfStream()) { - logger.debug('Skipping native-engine screen-share reconnect restore: local stream no longer requested'); - return false; - } - if (this.adapter.captureCoordinator.activeCaptureId) { - await this.release(snapshot); - } - try { - await this.adapter.startNativeEngineScreenShareInternal( - snapshot.nativeOptions, - {sendUpdate: false, playSound: false}, - snapshot.publishOptions, - ); - return LocalVoiceState.getSelfStream(); - } catch (error) { - logger.warn('Failed to restore native-engine screen share after reconnect', {error}); - return false; - } - } -} diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppSelectedMediaExecutionAdapter.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppSelectedMediaExecutionAdapter.ts deleted file mode 100644 index 278201c95..000000000 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppSelectedMediaExecutionAdapter.ts +++ /dev/null @@ -1,122 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {Logger} from '@app/features/platform/utils/AppLogger'; -import type { - LiveKitMediaPort, - VoiceEngineV2CameraEncodingOptions, - VoiceEngineV2CameraOptions, - VoiceEngineV2ConnectOptions, - VoiceEngineV2DataOptions, - VoiceEngineV2DisconnectReason, - VoiceEngineV2MicrophoneOptions, - VoiceEngineV2OutputDeviceOptions, - VoiceEngineV2ScreenAudioOptions, - VoiceEngineV2ScreenEncodingOptions, - VoiceEngineV2ScreenOptions, -} from '@fluxer/voice_engine_v2'; - -const ADAPTER_NAME = 'VoiceEngineV2AppSelectedMediaExecutionAdapter'; - -export type VoiceEngineV2AppSelectedMediaMode = 'js' | 'native'; - -export interface VoiceEngineV2AppSelectedMediaExecutionAdapterLogger { - info(...args: Array): void; -} - -export interface VoiceEngineV2AppSelectedMediaExecutionAdapterOptions { - jsMedia: LiveKitMediaPort; - nativeMedia: LiveKitMediaPort; - getMode: () => VoiceEngineV2AppSelectedMediaMode; - logger?: VoiceEngineV2AppSelectedMediaExecutionAdapterLogger; -} - -export class VoiceEngineV2AppSelectedMediaExecutionAdapter implements LiveKitMediaPort { - private readonly jsMedia: LiveKitMediaPort; - private readonly nativeMedia: LiveKitMediaPort; - private readonly getMode: () => VoiceEngineV2AppSelectedMediaMode; - private readonly logger: VoiceEngineV2AppSelectedMediaExecutionAdapterLogger; - private lastMode: VoiceEngineV2AppSelectedMediaMode | null = null; - - constructor(options: VoiceEngineV2AppSelectedMediaExecutionAdapterOptions) { - if (typeof options !== 'object' || options === null) { - throw new Error(`${ADAPTER_NAME}: options is required`); - } - this.jsMedia = options.jsMedia; - this.nativeMedia = options.nativeMedia; - this.getMode = options.getMode; - this.logger = options.logger ?? new Logger(ADAPTER_NAME); - } - - async prewarm(): Promise { - await this.selectedMedia('prewarm').prewarm(); - } - - async connect(options: VoiceEngineV2ConnectOptions): Promise { - await this.selectedMedia('connect').connect(options); - } - - async disconnect(reason: VoiceEngineV2DisconnectReason): Promise { - await this.selectedMedia('disconnect').disconnect(reason); - } - - async publishMicrophone(options: VoiceEngineV2MicrophoneOptions): Promise { - await this.selectedMedia('publishMicrophone').publishMicrophone(options); - } - - async unpublishMicrophone(): Promise { - await this.selectedMedia('unpublishMicrophone').unpublishMicrophone(); - } - - async setMicrophoneEnabled(enabled: boolean): Promise { - await this.selectedMedia('setMicrophoneEnabled').setMicrophoneEnabled(enabled); - } - - async publishCamera(options: VoiceEngineV2CameraOptions): Promise { - await this.selectedMedia('publishCamera').publishCamera(options); - } - - async updateCameraEncoding(options: VoiceEngineV2CameraEncodingOptions): Promise { - await this.selectedMedia('updateCameraEncoding').updateCameraEncoding(options); - } - - async unpublishCamera(options?: VoiceEngineV2CameraOptions): Promise { - await this.selectedMedia('unpublishCamera').unpublishCamera(options); - } - - async publishScreen(options: VoiceEngineV2ScreenOptions): Promise { - await this.selectedMedia('publishScreen').publishScreen(options); - } - - async updateScreenEncoding(options: VoiceEngineV2ScreenEncodingOptions): Promise { - await this.selectedMedia('updateScreenEncoding').updateScreenEncoding(options); - } - - async unpublishScreen(): Promise { - await this.selectedMedia('unpublishScreen').unpublishScreen(); - } - - async publishScreenAudio(options: VoiceEngineV2ScreenAudioOptions): Promise { - await this.selectedMedia('publishScreenAudio').publishScreenAudio(options); - } - - async unpublishScreenAudio(): Promise { - await this.selectedMedia('unpublishScreenAudio').unpublishScreenAudio(); - } - - async setOutputDevice(options: VoiceEngineV2OutputDeviceOptions): Promise { - await this.selectedMedia('setOutputDevice').setOutputDevice(options); - } - - async publishData(options: VoiceEngineV2DataOptions): Promise { - await this.selectedMedia('publishData').publishData(options); - } - - private selectedMedia(method: string): LiveKitMediaPort { - const mode = this.getMode(); - if (mode !== this.lastMode) { - this.lastMode = mode; - this.logger.info('Selected voice media port changed', {mode, method}); - } - return mode === 'native' ? this.nativeMedia : this.jsMedia; - } -} diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppSelectedMediaMode.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppSelectedMediaMode.ts deleted file mode 100644 index c0297126b..000000000 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppSelectedMediaMode.ts +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; -import {isNativeVoiceEngineSelected} from '@app/features/voice/engine/native_voice_engine/NativeVoiceEngineSelection'; -import type {VoiceEngineV2AppSelectedMediaMode} from '@app/features/voice/engine/v2/VoiceEngineV2AppSelectedMediaExecutionAdapter'; - -export interface VoiceEngineV2AppSelectedMediaFlows { - readonly js: () => Promise; - readonly native: () => Promise; -} - -export function resolveVoiceEngineV2AppSelectedMediaMode(): VoiceEngineV2AppSelectedMediaMode { - const mode = isNativeVoiceEngineSelected() ? 'native' : 'js'; - assert.ok(mode === 'native' || mode === 'js', 'selected media mode must be native or js'); - return mode; -} - -export async function routeVoiceEngineV2AppSelectedMedia(flows: VoiceEngineV2AppSelectedMediaFlows): Promise { - assert.equal(typeof flows.js, 'function', 'selected media routing requires a js flow'); - assert.equal(typeof flows.native, 'function', 'selected media routing requires a native flow'); - if (resolveVoiceEngineV2AppSelectedMediaMode() === 'native') { - return flows.native(); - } - return flows.js(); -} diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppSubscriptionAdapter.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppSubscriptionAdapter.ts index 087e5696e..e6d89f926 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppSubscriptionAdapter.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppSubscriptionAdapter.ts @@ -1,19 +1,15 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import {Logger} from '@app/features/platform/utils/AppLogger'; -import type {VoiceEngine} from '@app/features/voice/engine/native_voice_engine/VoiceEngine'; import {ScreenShareSubscriptionManager} from '@app/features/voice/engine/ScreenShareSubscriptionManager'; import {Store} from '@app/features/voice/engine/Store'; import {VideoSubscriptionManager} from '@app/features/voice/engine/VideoSubscriptionManager'; import type { - VoiceMediaGraphRemoteSubscriptionCommand, - VoiceMediaGraphRemoteTrackSubscriptionController, VoiceMediaGraphSubscriptionContext, VoiceMediaGraphVideoQuality, } from '@app/features/voice/engine/VoiceMediaGraph'; import {voiceMediaGraphStore} from '@app/features/voice/engine/VoiceMediaGraphStore'; -import {VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource'; -import {ScreenShareWatchErrorCode, ScreenShareWatchFailures} from '@app/features/voice/state/ScreenShareWatchFailures'; +import {isScreenShareAudioPublicationLike, VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource'; import type { VoiceEngineV2ParticipantVolumeOptions, VoiceEngineV2RemoteTrackSubscriptionOptions, @@ -43,40 +39,10 @@ function screenShareContextForQuality(quality: VideoQualityLevel | undefined): V return quality === 'high' ? 'focused' : 'carousel'; } -function nativeSubscriptionErrorMessage(error: unknown): string { - if (error instanceof Error) return error.message; - return String(error); -} - -function isMissingNativeSubscriptionTarget(error: unknown): boolean { - const message = nativeSubscriptionErrorMessage(error); - if (message.includes('remote track subscription participant not found')) return true; - return message.includes('remote track subscription publication not found'); -} - -function isExpectedNativeCameraPublicationRace( - options: VoiceMediaGraphRemoteSubscriptionCommand, - error: unknown, -): boolean { - if (!options.subscribed) return false; - if (options.source !== VoiceTrackSource.Camera) return false; - return isMissingNativeSubscriptionTarget(error); -} - -function isExpectedNativeScreenShareAudioPublicationRace( - options: VoiceMediaGraphRemoteSubscriptionCommand, - error: unknown, -): boolean { - if (!options.subscribed) return false; - if (options.source !== VoiceTrackSource.ScreenShareAudio) return false; - return isMissingNativeSubscriptionTarget(error); -} - class VoiceEngineV2AppSubscriptionAdapter extends Store { private room: Room | null = null; private videoManager = new VideoSubscriptionManager(); private screenShareManager = new ScreenShareSubscriptionManager(); - private nativeController: VoiceMediaGraphRemoteTrackSubscriptionController | null = null; constructor() { super(); @@ -90,9 +56,6 @@ class VoiceEngineV2AppSubscriptionAdapter extends Store { if (this.room) { this.cleanup(); } - if (room) { - this.setNativeEngine(null); - } this.update(() => { this.room = room; }); @@ -105,77 +68,17 @@ class VoiceEngineV2AppSubscriptionAdapter extends Store { } } - setNativeEngine(engine: Pick | null): void { - this.update(() => { - this.nativeController = engine ? this.createNativeController(engine) : null; - }); - this.videoManager.setNativeController(this.nativeController); - this.screenShareManager.setNativeController(this.nativeController); - } - - private createNativeController( - engine: Pick, - ): VoiceMediaGraphRemoteTrackSubscriptionController { - return { - setRemoteTrackSubscription: (options) => { - void engine.setRemoteTrackSubscription(options).then( - () => this.reportNativeSubscriptionApplied(options), - (error: unknown) => this.reportNativeSubscriptionFailed(options, error), - ); - }, - }; - } - - private reportNativeSubscriptionApplied(options: VoiceMediaGraphRemoteSubscriptionCommand): void { - voiceMediaGraphStore.transition({ - type: 'subscription.actualChanged', - participantIdentity: options.participantIdentity, - source: options.source, - at: voiceMediaGraphStore.nowMs(), - subscribed: options.subscribed, - ...(options.enabled !== undefined ? {enabled: options.enabled} : {}), - ...(options.quality !== undefined ? {quality: options.quality} : {}), - }); - } - - private reportNativeSubscriptionFailed(options: VoiceMediaGraphRemoteSubscriptionCommand, error: unknown): void { - const logData = { - participantIdentity: options.participantIdentity, - source: options.source, - subscribed: options.subscribed, - error, - }; - if (isExpectedNativeCameraPublicationRace(options, error)) { - logger.debug('Native remote camera subscription target missing; will retry after publish', logData); - } else if (isExpectedNativeScreenShareAudioPublicationRace(options, error)) { - logger.debug('Native remote screen-share audio subscription target missing; will retry after publish', logData); - return; - } else { - logger.warn('Native remote track subscription update failed', logData); - } - voiceMediaGraphStore.transition({ - type: 'subscription.commandFailed', - participantIdentity: options.participantIdentity, - source: options.source, - at: voiceMediaGraphStore.nowMs(), - code: ScreenShareWatchErrorCode.NativeSubscriptionCommandFailed, - reason: 'native-subscription-command-failed', - }); - if (options.subscribed && options.source === VoiceTrackSource.ScreenShare) { - ScreenShareWatchFailures.reportFailure({ - participantIdentity: options.participantIdentity, - source: options.source, - code: ScreenShareWatchErrorCode.NativeSubscriptionCommandFailed, - reason: 'native-subscription-command-failed', - error, - }); - } - } - async setParticipantVolume(options: VoiceEngineV2ParticipantVolumeOptions): Promise { const participant = this.room?.remoteParticipants.get(options.participantIdentity); if (!participant) return; participant.audioTrackPublications.forEach((publication) => { + if (isScreenShareAudioPublicationLike(publication)) { + logger.debug('Skipping screen share audio publication in participant volume update', { + participantIdentity: options.participantIdentity, + trackSid: publication.trackSid, + }); + return; + } const track = publication.audioTrack; if (track && 'setVolume' in track && typeof track.setVolume === 'function') { track.setVolume(options.volume); @@ -227,7 +130,6 @@ class VoiceEngineV2AppSubscriptionAdapter extends Store { logger.debug('Cleaning up all subscriptions'); this.videoManager.cleanup(); this.screenShareManager.cleanup(); - this.setNativeEngine(null); this.update(() => { this.room = null; }); diff --git a/fluxer_app/src/features/voice/engine/voice_screen_share_manager/DisplayMediaCapture.ts b/fluxer_app/src/features/voice/engine/voice_screen_share_manager/DisplayMediaCapture.ts index eba312a3b..6e8dfd130 100644 --- a/fluxer_app/src/features/voice/engine/voice_screen_share_manager/DisplayMediaCapture.ts +++ b/fluxer_app/src/features/voice/engine/voice_screen_share_manager/DisplayMediaCapture.ts @@ -1,34 +1,12 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import DeveloperOptions from '@app/features/devtools/state/DeveloperOptions'; -import {getElectronAPI} from '@app/features/ui/utils/NativeUtils'; -import {isNativeVoiceEngineSelected} from '@app/features/voice/engine/native_voice_engine/NativeVoiceEngineSelection'; -import { - markScreenShareCaptureActive, - markScreenShareCaptureEnded, - updateScreenShareDisplayMediaSettings, -} from '@app/features/voice/engine/ScreenShareCaptureDiagnostics'; +import {updateScreenShareDisplayMediaSettings} from '@app/features/voice/engine/ScreenShareCaptureDiagnostics'; import { type CapturedScreenShareTracks, stopMediaTrack, stopUnselectedStreamTracks, } from '@app/features/voice/engine/voice_screen_share_manager/shared'; import ActiveScreenShareSource from '@app/features/voice/state/ActiveScreenShareSource'; -import type {NativeAudioFramePumpSource} from '@app/features/voice/utils/NativeAudioCaptureBridge'; -import {createScreenChromiumPreviewBridge} from '@app/features/voice/utils/native_screen_capture_bridge/createChromiumPreviewBridge'; -import { - getNativeScreenCaptureApi, - markNativeScreenShareTrack, - type NativeScreenBridgeHandle, - normalizeNativeScreenCaptureResolution, -} from '@app/features/voice/utils/native_screen_capture_bridge/shared'; -import type { - NativeAudioStartOptions, - NativeScreenCaptureAvailability, - NativeScreenCaptureSource, - NativeScreenCaptureStartOptions, - NativeScreenCaptureStartResult, -} from '@app/types/electron.d'; import type {ScreenShareCaptureOptions} from 'livekit-client'; type DisplayMediaVideoConstraints = MediaTrackConstraints & { @@ -43,17 +21,6 @@ type DisplayMediaTrackSettings = MediaTrackSettings & { cursor?: 'always' | 'motion' | 'never'; displaySurface?: 'browser' | 'monitor' | 'window'; }; -interface NativeCaptureAvailabilityReader { - getAvailability?: () => Promise; -} - -async function readNativeCaptureAvailability( - api: NativeCaptureAvailabilityReader, -): Promise { - if (typeof api.getAvailability !== 'function') return null; - return Promise.resolve(api.getAvailability()).catch(() => null); -} - function resolveDisplayMediaCursorCapture( displaySurface: DisplayMediaVideoConstraints['displaySurface'], ): 'always' | 'motion' | 'never' { @@ -67,29 +34,6 @@ function getRequestedDisplayMediaVideoConstraints( return options.video as DisplayMediaVideoConstraints; } -function getNativePreviewBridgeOptions( - source: NativeScreenCaptureSource, - startResult: NativeScreenCaptureStartResult, - resolution?: ScreenShareCaptureOptions['resolution'], -): { - maxWidth: number; - maxHeight: number; - maxFrameRate: number; - pauseWhenUnfocused: boolean; - previewPlatform?: NodeJS.Platform; -} { - const sourceWidth = Math.max(1, startResult.width || resolution?.width || source.width); - const sourceHeight = Math.max(1, startResult.height || resolution?.height || source.height); - const sourceFrameRate = Math.max(1, startResult.frameRate || resolution?.frameRate || 60); - return { - maxWidth: Math.round(sourceWidth), - maxHeight: Math.round(sourceHeight), - maxFrameRate: Math.round(sourceFrameRate), - pauseWhenUnfocused: false, - previewPlatform: getElectronAPI()?.platform as NodeJS.Platform | undefined, - }; -} - export function resolveCapturedDisplayMediaCursorCapture( track: Pick, options?: ScreenShareCaptureOptions, @@ -178,96 +122,3 @@ export async function createDisplayScreenShareTracks( audioTrack, }; } - -export interface NativeScreenShareOptions { - source: NativeScreenCaptureSource; - desktopCaptureSourceId?: string; - captureId?: string; - resolution?: ScreenShareCaptureOptions['resolution']; - contentHint?: ScreenShareCaptureOptions['contentHint']; - showCursorClicks?: boolean; - captureRect?: NativeScreenCaptureStartOptions['captureRect']; - audioTrack?: MediaStreamTrack; - nativeAudioFramePump?: NativeAudioFramePumpSource; - nativeAudioLinuxRule?: NonNullable; -} - -export interface NativeEngineScreenCapture { - captureId: string; - width: number; - height: number; - previewBridge: NativeScreenBridgeHandle; -} - -export async function isNativeScreenCaptureAvailable(): Promise { - if (!isNativeVoiceEngineSelected()) return false; - const electronApi = getElectronAPI(); - const api = electronApi?.nativeScreenCapture ?? getNativeScreenCaptureApi(); - if (!api) return false; - try { - const availability = await api.getAvailability(); - return availability.available; - } catch { - return false; - } -} - -export async function startNativeCaptureForEngine( - options: NativeScreenShareOptions, -): Promise { - const api = getNativeScreenCaptureApi(); - if (!api) { - throw new Error('Native screen capture API unavailable'); - } - const {source, resolution} = options; - const nativeResolution = normalizeNativeScreenCaptureResolution(resolution); - const availability = await readNativeCaptureAvailability(api); - const startResult = await api.start({ - sourceId: source.id, - sourceKind: source.kind, - width: nativeResolution?.width, - height: nativeResolution?.height, - frameRate: resolution?.frameRate, - injectionMethod: source.kind === 'game' ? DeveloperOptions.gameCaptureInjectionMethod : undefined, - captureId: options.captureId, - colorRange: 'full', - colorSpace: 'rec709', - showCursorClicks: options.showCursorClicks, - captureRect: options.captureRect, - nativeFrameSinkRequired: true, - }); - markScreenShareCaptureActive({ - method: 'native-voice-engine-screen-capture', - sourceId: source.id, - sourceKind: source.kind, - captureId: startResult.captureId, - nativeAvailability: availability, - }); - let previewBridge: NativeScreenBridgeHandle; - try { - if (!options.desktopCaptureSourceId) { - throw new Error('Chromium desktop capture source id unavailable for native screen preview'); - } - previewBridge = await createScreenChromiumPreviewBridge( - options.desktopCaptureSourceId, - getNativePreviewBridgeOptions(source, startResult, resolution), - ); - markNativeScreenShareTrack(previewBridge.track); - } catch (error) { - await api.stop(startResult.captureId).catch(() => undefined); - markScreenShareCaptureEnded('native-voice-engine-screen-preview-bridge-failed'); - throw error; - } - return { - captureId: startResult.captureId, - width: startResult.width || source.width, - height: startResult.height || source.height, - previewBridge, - }; -} - -export async function stopNativeCaptureForEngine(captureId: string): Promise { - const api = getNativeScreenCaptureApi(); - if (!api) return; - await api.stop(captureId).catch(() => undefined); -} diff --git a/fluxer_app/src/features/voice/state/ScreenSharePiPController.ts b/fluxer_app/src/features/voice/state/ScreenSharePiPController.ts index 1588c85bc..0317158c5 100644 --- a/fluxer_app/src/features/voice/state/ScreenSharePiPController.ts +++ b/fluxer_app/src/features/voice/state/ScreenSharePiPController.ts @@ -9,7 +9,6 @@ import PiP from '@app/features/ui/state/PiP'; import Users from '@app/features/user/state/Users'; import {getStreamKey} from '@app/features/voice/components/StreamKeys'; import MediaEngine from '@app/features/voice/engine/MediaEngineFacade'; -import {isVoiceEngineV2NativeProjectionActiveFromMediaEngine} from '@app/features/voice/engine/VoiceMediaEngineBridge'; import {selectVoiceMediaGraphViewerStreamKeys} from '@app/features/voice/engine/VoiceMediaGraph'; import {voiceMediaGraphStore} from '@app/features/voice/engine/VoiceMediaGraphStore'; import LocalVoiceState from '@app/features/voice/state/LocalVoiceState'; @@ -100,30 +99,6 @@ function pickRemoteScreenShareFromSnapshots(channelId: string): ScreenSharePiPSc return null; } -function pickRemoteScreenShareFromVoiceStates(channelId: string): ScreenSharePiPScreenShare | null { - const connectionVoiceStates = MediaEngine.connectionVoiceStates; - const localConnectionId = MediaEngine.connectionId; - for (const connectionKey in connectionVoiceStates) { - const voiceState = connectionVoiceStates[connectionKey]; - if (!voiceState) continue; - if (!voiceState.self_stream) continue; - if (voiceState.channel_id !== channelId) continue; - const connectionId = voiceState.connection_id ?? null; - if (!connectionId || connectionId === localConnectionId) continue; - const userId = voiceState.user_id ?? null; - if (!userId) continue; - return buildScreenShare( - { - participantIdentity: buildVoiceParticipantIdentity(userId, connectionId), - userId, - connectionId, - }, - 'voice-state', - ); - } - return null; -} - function pickLocalSelfShare(channelId: string): ScreenSharePiPScreenShare | null { if (!LocalVoiceState.getSelfStream()) return null; const localConnectionId = MediaEngine.connectionId; @@ -152,9 +127,6 @@ function detectActiveScreenShare(channelId: string | null): ScreenSharePiPScreen } const fromSnapshots = pickRemoteScreenShareFromSnapshots(channelId); if (fromSnapshots) return fromSnapshots; - if (isVoiceEngineV2NativeProjectionActiveFromMediaEngine()) { - return pickRemoteScreenShareFromVoiceStates(channelId); - } return null; } diff --git a/fluxer_app/src/features/voice/state/ScreenShareWatchFailures.ts b/fluxer_app/src/features/voice/state/ScreenShareWatchFailures.ts index c50bfb021..4efdd81d1 100644 --- a/fluxer_app/src/features/voice/state/ScreenShareWatchFailures.ts +++ b/fluxer_app/src/features/voice/state/ScreenShareWatchFailures.ts @@ -19,7 +19,6 @@ export const ScreenShareWatchErrorCode = { SubscriptionEmitTrackUpdateFailed: -2104, ObserverAttachFailed: -2105, ObserverDetachFailed: -2106, - NativeSubscriptionCommandFailed: -2201, RemoteTrackSubscriptionFailed: -2202, PublicationMissingTimeout: -2301, SubscriptionAttachTimeout: -2302, @@ -37,7 +36,6 @@ export type ScreenShareWatchFailureReason = | 'subscription-emit-track-update-failed' | 'observer-attach-failed' | 'observer-detach-failed' - | 'native-subscription-command-failed' | 'remote-track-subscription-failed' | 'publication-missing-timeout' | 'subscription-attach-timeout' diff --git a/fluxer_app/src/features/voice/utils/ScreenShareStartFlow.ts b/fluxer_app/src/features/voice/utils/ScreenShareStartFlow.ts index 55507e13e..a82cb7793 100644 --- a/fluxer_app/src/features/voice/utils/ScreenShareStartFlow.ts +++ b/fluxer_app/src/features/voice/utils/ScreenShareStartFlow.ts @@ -1,6 +1,5 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import i18n from '@app/app/I18n'; import {LimitResolver} from '@app/features/app/utils/LimitResolverAdapter'; import {isLimitToggleEnabled} from '@app/features/app/utils/LimitUtils'; import {Logger} from '@app/features/platform/utils/AppLogger'; @@ -8,9 +7,6 @@ import {getElectronAPI, supportsDesktopScreenShareAudioCapture} from '@app/featu import * as VoiceSettingsCommands from '@app/features/voice/commands/VoiceSettingsCommands'; import MediaEngine from '@app/features/voice/engine/MediaEngineFacade'; import ScreenShareCodecNegotiation from '@app/features/voice/engine/ScreenShareCodecNegotiation'; -import {isVoiceEngineV2AppNativeScreenShareAudioBridgeAvailable} from '@app/features/voice/engine/v2/VoiceEngineV2AppNativeBridge'; -import {resolveVoiceEngineV2AppSelectedMediaMode} from '@app/features/voice/engine/v2/VoiceEngineV2AppSelectedMediaMode'; -import {isNativeScreenCaptureAvailable} from '@app/features/voice/engine/voice_screen_share_manager/DisplayMediaCapture'; import ActiveScreenShareSource from '@app/features/voice/state/ActiveScreenShareSource'; import {clearDesktopSourceIntent, setDesktopSourceIntent} from '@app/features/voice/state/DesktopSourceIntent'; import LocalVoiceState from '@app/features/voice/state/LocalVoiceState'; @@ -28,11 +24,9 @@ import { armNativeAudioForLinuxRouting, armNativeAudioForNextCapture, armNativeSystemAudioForNextCapture, - captureNativeAudioTrackForLinuxRouting, disarmNativeAudio, disarmPendingNativeAudio, getLastNativeAudioArmFailure, - type NativeAudioFramePumpSource, } from '@app/features/voice/utils/NativeAudioCaptureBridge'; import type {ScreenShareAudioCaptureDebugInfo} from '@app/features/voice/utils/ScreenShareAudioCaptureError'; import { @@ -52,28 +46,11 @@ import { ScreenSharePortalUnavailableError, } from '@app/features/voice/utils/ScreenSharePortalUnavailableError'; import {executeScreenShareOperation} from '@app/features/voice/utils/ScreenShareUtils'; -import type {NativeAudioStartOptions, NativeScreenCaptureSource, VirtmicNode} from '@app/types/electron.d'; -import {msg} from '@lingui/core/macro'; -import type {ScreenShareCaptureOptions, TrackPublishOptions, VideoCodec} from 'livekit-client'; +import type {NativeAudioStartOptions, VirtmicNode} from '@app/types/electron.d'; +import type {ScreenShareCaptureOptions, VideoCodec} from 'livekit-client'; const logger = new Logger('ScreenShareStartFlow'); -async function hasVoiceEngineV2NativeScreenShareAudioBridge(): Promise { - try { - return await isVoiceEngineV2AppNativeScreenShareAudioBridgeAvailable(); - } catch (error) { - logger.warn('Failed to resolve voice engine v2 native screen-share audio bridge capability', {error}); - return false; - } -} - -const GAME_WINDOW_DXGI_FALLBACK_DESCRIPTOR = msg({ - message: 'Game window (DXGI fallback)', - comment: - 'Fallback display name for a window screen-share source when browser display capture fails and native DXGI capture is attempted.', - context: 'screen-share', -}); - type LinuxNativeAudioRule = NonNullable; interface LinuxAudioLinkOptions { @@ -148,11 +125,6 @@ export async function reconfigureActiveLinuxScreenShareAudioLink(): Promise { - logger.warn('Failed to stop native-engine screen-share audio for source mode none', {error}); - }); - } return true; } const nativeRule = buildLinuxNativeAudioRule(sourceMode, userIncludeSources, userExcludeSources); @@ -275,10 +247,6 @@ function degradeAudioToVideoOnly( removeAudioFromCaptureOptions(captureOptions); } -function logRequiredNativeAudioUnavailable(debugInfo: ScreenShareAudioCaptureDebugInfo): void { - logger.warn('Native screen-share audio unavailable; aborting screen share because audio was requested', debugInfo); -} - function cleanupNativeAudioAfterCaptureDidNotStart(mode: 'start' | 'switch'): void { if (mode === 'switch') { disarmPendingNativeAudio(); @@ -383,7 +351,6 @@ async function runConfiguredDisplayScreenShare( } await MediaEngine.setScreenShareEnabled(false, {sendUpdate: false, playSound: false}); } - const nativeScreenShareAudioBridgeAvailable = await hasVoiceEngineV2NativeScreenShareAudioBridge(); let nativeAudioArmed = false; const isOwnWindowShare = options?.isOwnWindow === true && sourceId?.startsWith('window:'); if (isOwnWindowShare && requestedAudio) { @@ -408,13 +375,6 @@ async function runConfiguredDisplayScreenShare( requestedDesktopAudio && (electronApi.platform === 'darwin' || electronApi.platform === 'win32') && sourceId?.startsWith('screen:'); - const requestedNativeWindowAudio = - requestedAudio && - !isOwnWindowShare && - (electronApi.platform === 'darwin' || electronApi.platform === 'win32') && - sourceId?.startsWith('window:') === true; - const nativeEngineWillCaptureAudioDirectly = - nativeScreenShareAudioBridgeAvailable && (requestedNativeDesktopAudio || requestedNativeWindowAudio); const requestedNativePickerAudioOnLinux = requestedAudio && electronApi.platform === 'linux' && useWaylandPortal; const linuxDesktopAudioSourceMode = electronApi.platform === 'linux' && (requestedDesktopAudio || requestedNativePickerAudioOnLinux) @@ -438,7 +398,7 @@ async function runConfiguredDisplayScreenShare( }), ); } - } else if (requestedNativeDesktopAudio && !nativeEngineWillCaptureAudioDirectly) { + } else if (requestedNativeDesktopAudio) { try { nativeAudioArmed = await armNativeSystemAudioForNextCapture(); } catch (error) { @@ -490,27 +450,25 @@ async function runConfiguredDisplayScreenShare( sourceId?.startsWith('window:') && (electronApi.platform === 'darwin' || electronApi.platform === 'win32') ) { - if (!nativeEngineWillCaptureAudioDirectly) { - try { - nativeAudioArmed = await armNativeAudioForNextCapture(sourceId); - } catch (error) { - logger.warn('Failed to arm native per-window audio capture', { - sourceId, - error, - }); - } - if (!nativeAudioArmed) { - const debugInfo = buildAudioCaptureFailureDebug({ - sourceId, - reason: getLastNativeAudioArmFailure()?.reason ?? 'native-window-audio-route-unavailable', - }); - logger.warn('Per-window audio unavailable; aborting screen share because audio was requested', { - sourceId, - platform: electronApi.platform, - reason: debugInfo.reason, - }); - degradeAudioToVideoOnly(captureOptions, debugInfo); - } + try { + nativeAudioArmed = await armNativeAudioForNextCapture(sourceId); + } catch (error) { + logger.warn('Failed to arm native per-window audio capture', { + sourceId, + error, + }); + } + if (!nativeAudioArmed) { + const debugInfo = buildAudioCaptureFailureDebug({ + sourceId, + reason: getLastNativeAudioArmFailure()?.reason ?? 'native-window-audio-route-unavailable', + }); + logger.warn('Per-window audio unavailable; aborting screen share because audio was requested', { + sourceId, + platform: electronApi.platform, + reason: debugInfo.reason, + }); + degradeAudioToVideoOnly(captureOptions, debugInfo); } } if ( @@ -531,11 +489,7 @@ async function runConfiguredDisplayScreenShare( }), ); } - if ( - requestedAudio && - !nativeEngineWillCaptureAudioDirectly && - (nativeAudioArmed || electronApi.platform === 'linux') - ) { + if (requestedAudio && (nativeAudioArmed || electronApi.platform === 'linux')) { removeAudioFromCaptureOptions(captureOptions); } try { @@ -604,28 +558,6 @@ async function runConfiguredDisplayScreenShare( if (!capturedAfterError) { ActiveScreenShareSource.clear(); } - if ( - !capturedAfterError && - error instanceof Error && - error.name === 'NotReadableError' && - sourceId && - electronApi.platform === 'win32' && - mode !== 'switch' - ) { - const fallbackResult = await attemptDxgiFallbackCapture( - sourceId, - options?.sourceDimensions, - captureOptions, - publishOptions, - options?.isOwnWindow === true, - ).catch((fallbackError) => { - logger.warn('DXGI fallback capture also failed', {fallbackError}); - return false; - }); - if (fallbackResult) { - return true; - } - } if (useWaylandPortal && !capturedAfterError && mode !== 'switch') { logger.warn('Wayland screen share portal capture failed to start', { sourceId, @@ -650,46 +582,6 @@ async function runConfiguredDisplayScreenShare( return mode === 'switch' ? operationSucceeded : didScreenShareStart(); } -async function attemptDxgiFallbackCapture( - sourceId: string, - sourceDimensions: {width: number; height: number} | undefined, - _captureOptions: ScreenShareCaptureOptions, - publishOptions: TrackPublishOptions | undefined, - isOwnWindow: boolean, -): Promise { - if (sourceId.startsWith('screen:')) { - logger.debug('Skipping DXGI fallback for monitor share; monitor capture must not route through game capture', { - sourceId, - }); - return false; - } - const available = await isNativeScreenCaptureAvailable(); - if (!available) { - logger.debug('DXGI fallback unavailable: native screen capture not supported'); - return false; - } - const source: NativeScreenCaptureSource = { - kind: 'window', - id: sourceId, - name: i18n._(GAME_WINDOW_DXGI_FALLBACK_DESCRIPTOR), - width: sourceDimensions?.width ?? 1920, - height: sourceDimensions?.height ?? 1080, - }; - logger.info('Attempting DXGI fallback for NotReadableError', {sourceId}); - const nativeShareOptions = { - source, - resolution: _captureOptions.resolution ?? undefined, - contentHint: _captureOptions.contentHint, - }; - await MediaEngine.startNativeDisplayScreenShare(nativeShareOptions, undefined, publishOptions); - const succeeded = didScreenShareStart(); - if (succeeded) { - ActiveScreenShareSource.setSourceId(sourceId, {isOwnWindow}); - logger.info('DXGI fallback capture succeeded', {sourceId}); - } - return succeeded; -} - export async function startConfiguredDisplayScreenShare( sourceId?: string | null, options?: ConfiguredDisplayScreenShareOptions, @@ -759,181 +651,3 @@ export async function switchConfiguredDeviceScreenShare(videoDeviceId: string): return false; } } - -export interface ConfiguredNativeDisplayScreenShareOptions { - desktopSourceId?: string | null; - isOwnWindow?: boolean; - videoCodec?: VideoCodec; -} - -async function runConfiguredNativeDisplayScreenShare( - source: NativeScreenCaptureSource, - mode: 'start' | 'switch', - options: ConfiguredNativeDisplayScreenShareOptions = {}, -): Promise { - const electronApi = getElectronAPI(); - const displayShareEnvironment = await getDisplayShareEnvironment(); - const desktopSourceId = options.desktopSourceId ?? null; - const { - captureOptions, - publishOptions, - includeAudio: requestedAudio, - } = getConfiguredScreenShareOptions( - 'display', - displayShareEnvironment, - source.width && source.height ? {width: source.width, height: source.height} : undefined, - desktopSourceId, - source.kind === 'window' ? 'window' : 'monitor', - options.videoCodec, - ); - const linuxAudioSourceMode = electronApi?.platform === 'linux' ? VoiceSettings.getScreenShareAudioSourceMode() : null; - const requestedNativeAudio = requestedAudio && linuxAudioSourceMode !== 'none'; - const nativeScreenShareAudioBridgeAvailable = await hasVoiceEngineV2NativeScreenShareAudioBridge(); - disarmPendingNativeAudio(); - let audioTrack: MediaStreamTrack | null = null; - let nativeAudioFramePump: NativeAudioFramePumpSource | null = null; - let nativeAudioLinuxRule: LinuxNativeAudioRule | null = null; - if (requestedNativeAudio && electronApi) { - if (source.kind === 'screen' || source.kind === 'game') { - if (electronApi.platform === 'darwin' || electronApi.platform === 'win32') { - nativeAudioFramePump = {kind: 'system'}; - } else if (electronApi.platform === 'linux') { - const sourceMode = linuxAudioSourceMode ?? 'system'; - const userIncludeSources = VoiceSettings.getScreenShareAudioIncludeSources().map((entry) => ({...entry})); - const userExcludeSources = VoiceSettings.getScreenShareAudioExcludeSources().map((entry) => ({...entry})); - const linuxRule = buildLinuxNativeAudioRule(sourceMode, userIncludeSources, userExcludeSources); - try { - if (nativeScreenShareAudioBridgeAvailable) { - nativeAudioLinuxRule = linuxRule; - } else { - audioTrack = await captureNativeAudioTrackForLinuxRouting(linuxRule); - } - } catch (error) { - logger.warn('Failed to capture native Linux audio track for screen share', {error}); - } - } - } else if (source.kind === 'window') { - if (options.isOwnWindow) { - logger.warn('Fluxer-owned native window audio is excluded from screen share capture', { - sourceId: desktopSourceId, - platform: electronApi.platform, - }); - } else if ( - electronApi.platform === 'darwin' || - electronApi.platform === 'win32' || - electronApi.platform === 'linux' - ) { - const resolvedPid = - desktopSourceId && electronApi.nativeAudio - ? await electronApi.nativeAudio.resolveAudioRootPidForSource(desktopSourceId).catch((error) => { - logger.warn('Failed to resolve native window audio PID from desktop source', { - desktopSourceId, - platform: electronApi.platform, - error, - }); - return null; - }) - : null; - const targetPid = - typeof resolvedPid === 'number' && resolvedPid > 0 - ? resolvedPid - : typeof source.targetPid === 'number' && source.targetPid > 0 - ? source.targetPid - : null; - if (targetPid) { - nativeAudioFramePump = {kind: 'window', targetPid}; - } - } - } - if (requestedNativeAudio && !audioTrack && !nativeAudioFramePump && !nativeAudioLinuxRule) { - logRequiredNativeAudioUnavailable({ - platform: electronApi.platform, - sourceId: desktopSourceId, - sourceMode: linuxAudioSourceMode ?? (source.kind === 'window' ? 'specific' : 'system'), - sourceKind: source.kind, - reason: getLastNativeAudioArmFailure()?.reason ?? null, - }); - ActiveScreenShareSource.clear(); - return false; - } - } - removeAudioFromCaptureOptions(captureOptions); - const resolution = captureOptions.resolution ?? undefined; - const nativeShareOptions = { - source, - ...(desktopSourceId ? {desktopCaptureSourceId: desktopSourceId} : {}), - resolution, - contentHint: captureOptions.contentHint, - ...(audioTrack ? {audioTrack} : {}), - ...(nativeAudioFramePump ? {nativeAudioFramePump} : {}), - ...(nativeAudioLinuxRule ? {nativeAudioLinuxRule} : {}), - }; - try { - let succeeded: boolean; - if (mode === 'switch') { - succeeded = await MediaEngine.replaceActiveNativeDisplayScreenShare( - nativeShareOptions, - captureOptions, - publishOptions, - ); - } else { - await MediaEngine.startNativeDisplayScreenShare(nativeShareOptions, undefined, publishOptions); - succeeded = didScreenShareStart(); - } - if (!succeeded && audioTrack) { - try { - audioTrack.stop(); - } catch (stopError) { - logger.warn('Failed to stop native screen-share audio track after unsuccessful start', { - sourceKind: source.kind, - error: stopError, - }); - } - } - if (succeeded && desktopSourceId) { - ActiveScreenShareSource.setSourceId(desktopSourceId, {isOwnWindow: options.isOwnWindow === true}); - } else if (!succeeded) { - ActiveScreenShareSource.clear(); - } - return succeeded; - } catch (error) { - if (audioTrack) { - try { - audioTrack.stop(); - } catch (stopError) { - logger.warn('Failed to stop native screen-share audio track after start failure', { - sourceKind: source.kind, - error: stopError, - }); - } - } - logger.error('Failed to start native display screen share', { - error, - sourceKind: source.kind, - }); - ActiveScreenShareSource.clear(); - return false; - } -} - -export async function startConfiguredNativeDisplayScreenShare( - source: NativeScreenCaptureSource, - options?: ConfiguredNativeDisplayScreenShareOptions, -): Promise { - let didStart = false; - await executeScreenShareOperation(async () => { - didStart = await runConfiguredNativeDisplayScreenShare(source, 'start', options); - }); - return didStart; -} - -export async function switchConfiguredNativeDisplayScreenShare( - source: NativeScreenCaptureSource, - options?: ConfiguredNativeDisplayScreenShareOptions, -): Promise { - let didSwitch = false; - await executeScreenShareOperation(async () => { - didSwitch = await runConfiguredNativeDisplayScreenShare(source, 'switch', options); - }); - return didSwitch; -} diff --git a/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts b/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts index e98944678..596fcc829 100644 --- a/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts +++ b/fluxer_app/src/features/voice/utils/StatsForNerdsCopy.ts @@ -15,11 +15,9 @@ import { } from '@app/features/ui/utils/NativeUtils'; import AdaptiveScreenShareEngine from '@app/features/voice/engine/AdaptiveScreenShareEngine'; import MediaEngine from '@app/features/voice/engine/MediaEngineFacade'; -import NativeVoiceStatsStore from '@app/features/voice/engine/native_voice_engine/NativeVoiceStatsStore'; import ScreenShareCodecNegotiation, { getScreenShareCodecPreferenceOrder, } from '@app/features/voice/engine/ScreenShareCodecNegotiation'; -import {isVoiceEngineV2NativeProjectionActiveFromMediaEngine} from '@app/features/voice/engine/VoiceMediaEngineBridge'; import {getScreenShareAudioPumpDiagnostics} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareAudioPump'; import {getNativeEngineAudioTrackPumpStats} from '@app/features/voice/engine/voice_screen_share_manager/NativeEngineAudioTrackPump'; import VoiceSettings from '@app/features/voice/state/VoiceSettings'; @@ -484,7 +482,7 @@ export function collectStatsForNerdsSnapshot(): StatsForNerdsData { stats: MediaEngine.voiceStats, perTrackStats: MediaEngine.perTrackStats, statsTimeSeries: MediaEngine.statsTimeSeries, - nativeStats: isVoiceEngineV2NativeProjectionActiveFromMediaEngine() ? NativeVoiceStatsStore.stats : null, + nativeStats: null, publisherTransport: MediaEngine.publisherTransport, subscriberTransport: MediaEngine.subscriberTransport, localParticipant, diff --git a/fluxer_app/src/features/voice/utils/VideoBackgroundFramePump.ts b/fluxer_app/src/features/voice/utils/VideoBackgroundFramePump.ts deleted file mode 100644 index b9108ecc7..000000000 --- a/fluxer_app/src/features/voice/utils/VideoBackgroundFramePump.ts +++ /dev/null @@ -1,501 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; -import {Logger} from '@app/features/platform/utils/AppLogger'; -import {getBackgroundImageURL} from '@app/features/theme/utils/BackgroundImageDB'; -import type {VoiceEngineV2BridgeProcessedCameraFrame} from '@fluxer/voice_engine_v2/bridge'; - -const logger = new Logger('VideoBackgroundFramePump'); - -export const VIDEO_BACKGROUND_PUMP_FPS_MAX = 24; -export const VIDEO_BACKGROUND_PUMP_DIMENSION_MIN = 2; -export const VIDEO_BACKGROUND_PUMP_IN_FLIGHT_FRAMES_MAX = 2; -export const VIDEO_BACKGROUND_PUMP_WEBCODECS_FAILURES_MAX = 3; -const VIDEO_BACKGROUND_PUMP_FALLBACK_INTERVAL_MS = Math.ceil(1000 / VIDEO_BACKGROUND_PUMP_FPS_MAX); -const RGBA_BYTES_PER_PIXEL = 4; -const VIDEO_READY_STATE_HAS_CURRENT_DATA = 2; - -const BT601_Y_R = 66; -const BT601_Y_G = 129; -const BT601_Y_B = 25; -const BT601_U_R = -38; -const BT601_U_G = -74; -const BT601_U_B = 112; -const BT601_V_R = 112; -const BT601_V_G = -94; -const BT601_V_B = -18; -const BT601_FIXED_POINT_ROUNDING = 128; -const BT601_FIXED_POINT_SHIFT = 8; -const BT601_Y_OFFSET = 16; -const BT601_CHROMA_OFFSET = 128; - -export interface VideoBackgroundFrameBridge { - pushCameraBackgroundFrame(frame: VoiceEngineV2BridgeProcessedCameraFrame): Promise; - clearCameraBackgroundFrame(): Promise; -} - -export interface VideoBackgroundCaptureDimensions { - width: number; - height: number; -} - -export interface VideoBackgroundFramePumpHooks { - resolveBackgroundUrl: (backgroundId: string) => Promise; - createVideoElement: () => HTMLVideoElement; - createCanvasContext: (width: number, height: number) => CanvasRenderingContext2D | null; - revokeObjectUrl: (url: string) => void; - now: () => number; -} - -export function createDefaultVideoBackgroundFramePumpHooks(): VideoBackgroundFramePumpHooks { - return { - resolveBackgroundUrl: (backgroundId) => getBackgroundImageURL(backgroundId), - createVideoElement: () => document.createElement('video'), - createCanvasContext: (width, height) => { - const canvas = document.createElement('canvas'); - canvas.width = width; - canvas.height = height; - return canvas.getContext('2d', {willReadFrequently: true}); - }, - revokeObjectUrl: (url) => URL.revokeObjectURL(url), - now: () => Date.now(), - }; -} - -export function floorToEvenDimension(value: number): number { - assert.ok(Number.isFinite(value), 'video background dimension must be finite'); - const floored = Math.max(VIDEO_BACKGROUND_PUMP_DIMENSION_MIN, Math.floor(value)); - return floored - (floored % 2); -} - -export function shouldEmitVideoBackgroundFrame(nowMs: number, lastEmitMs: number | null, fpsMax: number): boolean { - assert.ok(Number.isFinite(nowMs), 'video background frame clock must be finite'); - assert.ok(fpsMax > 0, 'video background frame fps cap must be positive'); - if (lastEmitMs === null) return true; - return nowMs - lastEmitMs >= 1000 / fpsMax; -} - -function clampByte(value: number): number { - return value < 0 ? 0 : value > 255 ? 255 : value; -} - -function packChromaPlanes(rgba: Uint8Array | Uint8ClampedArray, width: number, height: number, i420: Uint8Array): void { - const chromaWidth = width / 2; - const uPlaneOffset = width * height; - const vPlaneOffset = uPlaneOffset + chromaWidth * (height / 2); - for (let blockY = 0; blockY < height / 2; blockY++) { - for (let blockX = 0; blockX < chromaWidth; blockX++) { - let rSum = 0; - let gSum = 0; - let bSum = 0; - for (let dy = 0; dy < 2; dy++) { - for (let dx = 0; dx < 2; dx++) { - const offset = ((blockY * 2 + dy) * width + blockX * 2 + dx) * RGBA_BYTES_PER_PIXEL; - rSum += rgba[offset]; - gSum += rgba[offset + 1]; - bSum += rgba[offset + 2]; - } - } - const r = rSum >> 2; - const g = gSum >> 2; - const b = bSum >> 2; - const chromaIndex = blockY * chromaWidth + blockX; - i420[uPlaneOffset + chromaIndex] = clampByte( - ((BT601_U_R * r + BT601_U_G * g + BT601_U_B * b + BT601_FIXED_POINT_ROUNDING) >> BT601_FIXED_POINT_SHIFT) + - BT601_CHROMA_OFFSET, - ); - i420[vPlaneOffset + chromaIndex] = clampByte( - ((BT601_V_R * r + BT601_V_G * g + BT601_V_B * b + BT601_FIXED_POINT_ROUNDING) >> BT601_FIXED_POINT_SHIFT) + - BT601_CHROMA_OFFSET, - ); - } - } -} - -function i420FrameByteLength(width: number, height: number): number { - assert.equal(width % 2, 0, 'i420 width must be even'); - assert.equal(height % 2, 0, 'i420 height must be even'); - return width * height + (width / 2) * (height / 2) * 2; -} - -export function rgbaToI420Bt601Into( - rgba: Uint8Array | Uint8ClampedArray, - width: number, - height: number, - i420: Uint8Array, -): void { - assert.ok(width >= VIDEO_BACKGROUND_PUMP_DIMENSION_MIN, 'rgba to i420 width must be at least 2'); - assert.ok(height >= VIDEO_BACKGROUND_PUMP_DIMENSION_MIN, 'rgba to i420 height must be at least 2'); - assert.equal(width % 2, 0, 'rgba to i420 width must be even'); - assert.equal(height % 2, 0, 'rgba to i420 height must be even'); - assert.equal(rgba.length, width * height * RGBA_BYTES_PER_PIXEL, 'rgba buffer must match dimensions'); - assert.equal(i420.length, i420FrameByteLength(width, height), 'i420 buffer must match dimensions'); - for (let y = 0; y < height; y++) { - for (let x = 0; x < width; x++) { - const offset = (y * width + x) * RGBA_BYTES_PER_PIXEL; - const r = rgba[offset]; - const g = rgba[offset + 1]; - const b = rgba[offset + 2]; - i420[y * width + x] = clampByte( - ((BT601_Y_R * r + BT601_Y_G * g + BT601_Y_B * b + BT601_FIXED_POINT_ROUNDING) >> BT601_FIXED_POINT_SHIFT) + - BT601_Y_OFFSET, - ); - } - } - packChromaPlanes(rgba, width, height, i420); -} - -export function rgbaToI420Bt601(rgba: Uint8Array | Uint8ClampedArray, width: number, height: number): Uint8Array { - const i420 = new Uint8Array(i420FrameByteLength(width, height)); - rgbaToI420Bt601Into(rgba, width, height, i420); - return i420; -} - -interface VideoBackgroundI420Buffer { - data: Uint8Array; - inUse: boolean; -} - -interface VideoBackgroundPumpSession { - backgroundId: string; - generation: number; - video: HTMLVideoElement; - objectUrl: string; - context: CanvasRenderingContext2D | null; - contextWidth: number; - contextHeight: number; - frameCallbackId: number | null; - timeoutId: NodeJS.Timeout | null; - lastEmitMs: number | null; - i420Buffers: Array; - i420BufferWidth: number; - i420BufferHeight: number; - poolExhaustedWarned: boolean; - webCodecsI420Unavailable: boolean; - webCodecsI420ConsecutiveFailures: number; -} - -interface I420VideoFrame { - copyTo?: (destination: Uint8Array, options: {format: 'I420'}) => Promise; - close: () => void; -} - -interface I420VideoFrameConstructor { - new (source: CanvasImageSource, init: {timestamp: number}): I420VideoFrame; -} - -interface ResolvedVideoFrameTime { - nowMs: number; - timestampUs: number; -} - -function getI420VideoFrameConstructor(): I420VideoFrameConstructor | null { - const videoFrameConstructor = ( - window as typeof window & { - VideoFrame?: I420VideoFrameConstructor; - } - ).VideoFrame; - return videoFrameConstructor ?? null; -} - -function finiteNonNegative(value: number | undefined): value is number { - return typeof value === 'number' && Number.isFinite(value) && value >= 0; -} - -function resolveVideoFrameTime( - metadata: VideoFrameCallbackMetadata | null, - fallbackNowMs: number, -): ResolvedVideoFrameTime { - assert.ok(Number.isFinite(fallbackNowMs), 'video background fallback clock must be finite'); - const frameNowMs = finiteNonNegative(metadata?.presentationTime) - ? metadata.presentationTime - : finiteNonNegative(metadata?.expectedDisplayTime) - ? metadata.expectedDisplayTime - : fallbackNowMs; - const timestampUs = Math.max(1, Math.round(frameNowMs * 1000)); - return {nowMs: frameNowMs, timestampUs}; -} - -export class VideoBackgroundFramePump { - private readonly bridge: VideoBackgroundFrameBridge; - private readonly getCaptureDimensions: () => VideoBackgroundCaptureDimensions; - private readonly hooks: VideoBackgroundFramePumpHooks; - private session: VideoBackgroundPumpSession | null = null; - private generation = 0; - - constructor(options: { - bridge: VideoBackgroundFrameBridge; - getCaptureDimensions: () => VideoBackgroundCaptureDimensions; - hooks?: VideoBackgroundFramePumpHooks; - }) { - this.bridge = options.bridge; - this.getCaptureDimensions = options.getCaptureDimensions; - this.hooks = options.hooks ?? createDefaultVideoBackgroundFramePumpHooks(); - } - - isRunning(): boolean { - return this.session !== null; - } - - getRunningBackgroundId(): string | null { - return this.session?.backgroundId ?? null; - } - - async start(backgroundId: string): Promise { - assert.ok(backgroundId.length > 0, 'video background pump requires a background id'); - if (this.session?.backgroundId === backgroundId) return true; - if (this.session) { - await this.stop(); - } - const generation = ++this.generation; - const objectUrl = await this.hooks.resolveBackgroundUrl(backgroundId); - if (!objectUrl) { - logger.warn('Video background media is unavailable for pumping', {backgroundId}); - return false; - } - if (generation !== this.generation) { - this.hooks.revokeObjectUrl(objectUrl); - return false; - } - const video = this.hooks.createVideoElement(); - video.muted = true; - video.loop = true; - video.playsInline = true; - video.src = objectUrl; - this.session = { - backgroundId, - generation, - video, - objectUrl, - context: null, - contextWidth: 0, - contextHeight: 0, - frameCallbackId: null, - timeoutId: null, - lastEmitMs: null, - i420Buffers: [], - i420BufferWidth: 0, - i420BufferHeight: 0, - poolExhaustedWarned: false, - webCodecsI420Unavailable: false, - webCodecsI420ConsecutiveFailures: 0, - }; - const playResult = video.play(); - if (playResult && typeof playResult.catch === 'function') { - playResult.catch((error) => { - logger.warn('Video background playback failed to start', {backgroundId, error}); - }); - } - this.scheduleNextFrame(generation); - return true; - } - - async stop(): Promise { - const session = this.session; - this.generation++; - this.session = null; - if (!session) return; - this.clearScheduledFrame(session); - session.video.pause(); - session.video.removeAttribute('src'); - this.hooks.revokeObjectUrl(session.objectUrl); - try { - await this.bridge.clearCameraBackgroundFrame(); - } catch (error) { - logger.warn('Failed to clear native camera background frame on pump stop', {error}); - } - } - - private clearScheduledFrame(session: VideoBackgroundPumpSession): void { - if (session.frameCallbackId !== null && typeof session.video.cancelVideoFrameCallback === 'function') { - session.video.cancelVideoFrameCallback(session.frameCallbackId); - } - if (session.timeoutId !== null) { - clearTimeout(session.timeoutId); - } - session.frameCallbackId = null; - session.timeoutId = null; - } - - private scheduleNextFrame(generation: number): void { - const session = this.session; - if (!session || session.generation !== generation) return; - if (typeof session.video.requestVideoFrameCallback === 'function') { - const frameCallbackId = session.video.requestVideoFrameCallback((_now, metadata) => { - session.frameCallbackId = null; - if (session.timeoutId !== null) { - clearTimeout(session.timeoutId); - session.timeoutId = null; - } - this.pumpFrame(generation, metadata); - }); - session.frameCallbackId = frameCallbackId; - } - session.timeoutId = setTimeout(() => { - session.timeoutId = null; - if (session.frameCallbackId !== null && typeof session.video.cancelVideoFrameCallback === 'function') { - session.video.cancelVideoFrameCallback(session.frameCallbackId); - session.frameCallbackId = null; - } - this.pumpFrame(generation, null); - }, VIDEO_BACKGROUND_PUMP_FALLBACK_INTERVAL_MS); - } - - private resolveSessionContext(session: VideoBackgroundPumpSession): CanvasRenderingContext2D | null { - const dimensions = this.getCaptureDimensions(); - const width = floorToEvenDimension(dimensions.width); - const height = floorToEvenDimension(dimensions.height); - if (session.context && session.contextWidth === width && session.contextHeight === height) { - return session.context; - } - session.context = this.hooks.createCanvasContext(width, height); - session.contextWidth = width; - session.contextHeight = height; - session.i420Buffers = this.createI420BufferPool(width, height); - session.i420BufferWidth = width; - session.i420BufferHeight = height; - session.poolExhaustedWarned = false; - session.webCodecsI420Unavailable = false; - session.webCodecsI420ConsecutiveFailures = 0; - return session.context; - } - - private createI420BufferPool(width: number, height: number): Array { - const byteLength = i420FrameByteLength(width, height); - const buffers: Array = []; - for (let i = 0; i < VIDEO_BACKGROUND_PUMP_IN_FLIGHT_FRAMES_MAX; i += 1) { - buffers.push({data: new Uint8Array(byteLength), inUse: false}); - } - return buffers; - } - - private acquireI420Buffer(session: VideoBackgroundPumpSession): VideoBackgroundI420Buffer | null { - assert.equal(session.i420BufferWidth, session.contextWidth, 'i420 pool width must match capture width'); - assert.equal(session.i420BufferHeight, session.contextHeight, 'i420 pool height must match capture height'); - for (const buffer of session.i420Buffers) { - if (!buffer.inUse) { - buffer.inUse = true; - return buffer; - } - } - if (!session.poolExhaustedWarned) { - session.poolExhaustedWarned = true; - logger.warn('Video background pump skipped a frame because the bounded native push pool is exhausted', { - inFlightFramesMax: VIDEO_BACKGROUND_PUMP_IN_FLIGHT_FRAMES_MAX, - }); - } - return null; - } - - private releaseI420Buffer(buffer: VideoBackgroundI420Buffer): void { - assert.ok(buffer.inUse, 'released video background buffer must be in use'); - buffer.inUse = false; - } - - private async copyCanvasToI420WithWebCodecs( - session: VideoBackgroundPumpSession, - i420Buffer: VideoBackgroundI420Buffer, - timestampUs: number, - ): Promise { - if (session.webCodecsI420Unavailable) return false; - const VideoFrameImpl = getI420VideoFrameConstructor(); - if (!VideoFrameImpl) { - session.webCodecsI420Unavailable = true; - return false; - } - let frame: I420VideoFrame | null = null; - try { - assert.ok(session.context !== null, 'video background WebCodecs context must exist'); - frame = new VideoFrameImpl(session.context.canvas, {timestamp: timestampUs}); - if (typeof frame.copyTo !== 'function') { - session.webCodecsI420Unavailable = true; - return false; - } - await frame.copyTo(i420Buffer.data, {format: 'I420'}); - session.webCodecsI420ConsecutiveFailures = 0; - return true; - } catch (error) { - session.webCodecsI420ConsecutiveFailures += 1; - assert.ok(session.webCodecsI420ConsecutiveFailures > 0, 'webcodecs failure count must be positive'); - if (session.webCodecsI420ConsecutiveFailures >= VIDEO_BACKGROUND_PUMP_WEBCODECS_FAILURES_MAX) { - session.webCodecsI420Unavailable = true; - } - logger.debug('Video background WebCodecs I420 copy failed; falling back to canvas readback', { - error, - consecutiveFailures: session.webCodecsI420ConsecutiveFailures, - latched: session.webCodecsI420Unavailable, - }); - return false; - } finally { - frame?.close(); - } - } - - private copyCanvasToI420WithReadback( - context: CanvasRenderingContext2D, - width: number, - height: number, - i420Buffer: VideoBackgroundI420Buffer, - ): void { - const imageData = context.getImageData(0, 0, width, height); - rgbaToI420Bt601Into(imageData.data, width, height, i420Buffer.data); - } - - private async captureAndPushFrame( - session: VideoBackgroundPumpSession, - i420Buffer: VideoBackgroundI420Buffer, - frameTime: ResolvedVideoFrameTime, - ): Promise { - const context = session.context; - assert.ok(context !== null, 'video background capture context must exist'); - const width = session.contextWidth; - const height = session.contextHeight; - const timestampUs = frameTime.timestampUs; - try { - context.drawImage(session.video, 0, 0, width, height); - const copiedWithWebCodecs = await this.copyCanvasToI420WithWebCodecs(session, i420Buffer, timestampUs); - if (!copiedWithWebCodecs) { - this.copyCanvasToI420WithReadback(context, width, height, i420Buffer); - } - session.lastEmitMs = frameTime.nowMs; - await this.bridge.pushCameraBackgroundFrame({ - format: 'i420', - width, - height, - timestampUs, - data: i420Buffer.data, - }); - } catch (error) { - logger.warn('Failed to capture video background frame', {error}); - } finally { - this.releaseI420Buffer(i420Buffer); - } - } - - private pumpFrame(generation: number, metadata: VideoFrameCallbackMetadata | null = null): void { - const session = this.session; - if (!session || session.generation !== generation) return; - if (session.video.readyState < VIDEO_READY_STATE_HAS_CURRENT_DATA) { - this.scheduleNextFrame(generation); - return; - } - const frameTime = resolveVideoFrameTime(metadata, this.hooks.now()); - if (!shouldEmitVideoBackgroundFrame(frameTime.nowMs, session.lastEmitMs, VIDEO_BACKGROUND_PUMP_FPS_MAX)) { - this.scheduleNextFrame(generation); - return; - } - const context = this.resolveSessionContext(session); - if (!context) { - logger.warn('Video background pump canvas context is unavailable; stopping pump'); - void this.stop(); - return; - } - const i420Buffer = this.acquireI420Buffer(session); - if (!i420Buffer) { - this.scheduleNextFrame(generation); - return; - } - void this.captureAndPushFrame(session, i420Buffer, frameTime); - this.scheduleNextFrame(generation); - } -} diff --git a/fluxer_app/src/features/voice/utils/VoiceBackgroundAvailability.ts b/fluxer_app/src/features/voice/utils/VoiceBackgroundAvailability.ts index 59d8d49f9..c58f84026 100644 --- a/fluxer_app/src/features/voice/utils/VoiceBackgroundAvailability.ts +++ b/fluxer_app/src/features/voice/utils/VoiceBackgroundAvailability.ts @@ -1,17 +1,5 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import { - getNativeVoiceEngineCapabilitiesSnapshot, - getNativeVoiceEngineSelectionSnapshot, -} from '@app/features/voice/engine/native_voice_engine/NativeVoiceEngineSelection'; - export function areVoiceBackgroundsAvailable(): boolean { - const selection = getNativeVoiceEngineSelectionSnapshot(); - if (selection.state === 'checking' || selection.state === 'uninitialized') { - return true; - } - if (selection.state !== 'native') { - return false; - } - return getNativeVoiceEngineCapabilitiesSnapshot()?.nativeCameraBackgrounds === true; + return false; } diff --git a/fluxer_app/src/features/voice/utils/VoiceDeviceManager.ts b/fluxer_app/src/features/voice/utils/VoiceDeviceManager.ts index 87d68e7cb..3e328f296 100644 --- a/fluxer_app/src/features/voice/utils/VoiceDeviceManager.ts +++ b/fluxer_app/src/features/voice/utils/VoiceDeviceManager.ts @@ -4,16 +4,7 @@ import {ensureNativePermission} from '@app/features/permissions/system/utils/Nat import {Platform} from '@app/features/platform/types/Platform'; import {Logger} from '@app/features/platform/utils/AppLogger'; import {getNativePlatformSync, isDesktop} from '@app/features/ui/utils/NativeUtils'; -import { - type NativeAudioDeviceModuleStatus, - nativeAudioDeviceModuleState, -} from '@app/features/voice/engine/native_voice_engine/NativeAudioDeviceModuleState'; -import {isVoiceEngineV2AppNativeAudioDeviceBridgeAvailable} from '@app/features/voice/engine/v2/VoiceEngineV2AppNativeBridge'; -import type { - VoiceEngineV2AudioDeviceRole, - VoiceEngineV2AudioInputDevice, - VoiceEngineV2AudioOutputDevice, -} from '@fluxer/voice_engine_v2'; +import type {VoiceEngineV2AudioDeviceRole} from '@fluxer/voice_engine_v2'; const logger = new Logger('VoiceDeviceManager'); @@ -93,11 +84,6 @@ export type VoiceMediaDeviceInfo = MediaDeviceInfo & { fluxerVoiceAudioDevice?: VoiceAudioDeviceMetadata; }; -type NativeVoiceDeviceBridge = { - listAudioInputDevices?: () => Promise>; - listAudioOutputDevices?: () => Promise>; -}; - interface NormalizedAudioDeviceLabel { role: VoiceAudioDeviceRole | null; endpointLabel: string; @@ -129,11 +115,6 @@ const UUID_DEVICE_LABEL = /^[{(]?[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab] const OPAQUE_DEVICE_LABEL = /^[a-z0-9+/=_:-]{24,}$/i; const DEVICE_PATH_LABEL = /^(?:\\\\\?\\|[a-z]+:\/)/i; -function getNativeVoiceDeviceBridge(): NativeVoiceDeviceBridge | null { - if (typeof window === 'undefined') return null; - return window.electron?.voiceEngine ?? null; -} - function getAudioDefaultDevicePlatform(): VoiceAudioDefaultDevicePlatform { if (!isDesktop()) { return 'browser'; @@ -396,42 +377,6 @@ export function shapeBrowserAudioDevices(devices: ReadonlyArray ); } -export function shapeNativeAudioInputDevices( - devices: ReadonlyArray, -): Array { - return shapeAudioDevices( - devices.map((device) => ({ - deviceId: device.deviceId, - groupId: '', - kind: 'audioinput', - label: device.label, - isDefault: device.isDefault, - role: device.role, - endpointLabel: device.endpointLabel, - isDefaultRoute: device.isDefaultRoute, - })), - {synthesizeDefaultRoute: true}, - ); -} - -export function shapeNativeAudioOutputDevices( - devices: ReadonlyArray, -): Array { - return shapeAudioDevices( - devices.map((device) => ({ - deviceId: device.deviceId, - groupId: '', - kind: 'audiooutput', - label: device.label, - isDefault: device.isDefault, - role: device.role, - endpointLabel: device.endpointLabel, - isDefaultRoute: device.isDefaultRoute, - })), - {synthesizeDefaultRoute: true}, - ); -} - class VoiceDeviceManager { private state: VoiceDeviceState = { inputDevices: [], @@ -445,13 +390,11 @@ class VoiceDeviceManager { private currentEnumerationRequestsPermissions = false; private shouldRequestPermissions = false; private hasEnumeratedDevices = false; - private lastEnumerationUsedNativeAudio: boolean | null = null; constructor() { if (typeof navigator !== 'undefined' && navigator.mediaDevices?.addEventListener) { navigator.mediaDevices.addEventListener('devicechange', this.handleDeviceChange); } - nativeAudioDeviceModuleState.subscribe(this.handleNativeAudioDeviceModuleStatusChange); } public getState(): VoiceDeviceState { @@ -483,8 +426,7 @@ class VoiceDeviceManager { this.shouldRequestPermissions = true; } const shouldRequest = this.shouldRequestPermissions || requestPermissions; - const useNativeAudioDevices = await this.shouldUseNativeAudioDevices(); - if (!forceRefresh && !this.enumeratingPromise && this.canUseCachedState(shouldRequest, useNativeAudioDevices)) { + if (!forceRefresh && !this.enumeratingPromise && this.canUseCachedState(shouldRequest)) { logger.debug('Using cached device state'); return this.state; } @@ -494,7 +436,7 @@ class VoiceDeviceManager { if (!this.queuedPermissionEnumerationPromise) { this.queuedPermissionEnumerationPromise = this.enumeratingPromise .catch(() => this.state) - .then(async () => this.startEnumeration(true, await this.shouldUseNativeAudioDevices())) + .then(() => this.startEnumeration(true)) .finally(() => { this.queuedPermissionEnumerationPromise = null; }); @@ -505,25 +447,22 @@ class VoiceDeviceManager { return this.enumeratingPromise; } logger.debug('Creating new enumeration promise'); - return this.startEnumeration(shouldRequest, useNativeAudioDevices); + return this.startEnumeration(shouldRequest); } - private canUseCachedState(requestPermissions: boolean, useNativeAudioDevices: boolean): boolean { + private canUseCachedState(requestPermissions: boolean): boolean { if (!this.hasEnumeratedDevices) { return false; } - if (this.lastEnumerationUsedNativeAudio !== useNativeAudioDevices) { - return false; - } if (!requestPermissions) { return true; } return this.state.permissionStatus === 'granted'; } - private startEnumeration(requestPermissions: boolean, useNativeAudioDevices: boolean): Promise { + private startEnumeration(requestPermissions: boolean): Promise { this.currentEnumerationRequestsPermissions = requestPermissions; - const pendingPromise = this.enumerateDevices(requestPermissions, useNativeAudioDevices).catch((error) => { + const pendingPromise = this.enumerateDevices(requestPermissions).catch((error) => { logger.debug('Failed to enumerate media devices:', error); throw error; }); @@ -537,10 +476,7 @@ class VoiceDeviceManager { }); } - private async enumerateDevices( - requestPermissions: boolean, - useNativeAudioDevices: boolean, - ): Promise { + private async enumerateDevices(requestPermissions: boolean): Promise { logger.debug('enumerateDevices started', {requestPermissions}); if (!navigator.mediaDevices?.enumerateDevices) { logger.debug('Navigator or mediaDevices API not available'); @@ -599,7 +535,7 @@ class VoiceDeviceManager { logger.debug('No labels found, requesting permissions via getUserMedia'); try { const stream = await navigator.mediaDevices.getUserMedia({ - audio: !useNativeAudioDevices, + audio: true, video: true, }); logger.debug('getUserMedia succeeded, stopping tracks'); @@ -635,26 +571,12 @@ class VoiceDeviceManager { } } } - const nativeAudioWarming = useNativeAudioDevices - ? (await nativeAudioDeviceModuleState.ensureStatus()) === 'warming' - : false; - if (nativeAudioWarming) { - logger.debug('Native audio device module warming; deferring native device enumeration'); - } - const inputDevices = useNativeAudioDevices - ? nativeAudioWarming - ? [] - : await this.enumerateNativeInputDevices() - : shapeBrowserAudioDevices( - devices.filter((device) => device.kind === 'audioinput' && !isInternalVirtualAudioDevice(device)), - ); - const outputDevices = useNativeAudioDevices - ? nativeAudioWarming - ? [] - : await this.enumerateNativeOutputDevices() - : shapeBrowserAudioDevices( - devices.filter((device) => device.kind === 'audiooutput' && !isInternalVirtualAudioDevice(device)), - ); + const inputDevices = shapeBrowserAudioDevices( + devices.filter((device) => device.kind === 'audioinput' && !isInternalVirtualAudioDevice(device)), + ); + const outputDevices = shapeBrowserAudioDevices( + devices.filter((device) => device.kind === 'audiooutput' && !isInternalVirtualAudioDevice(device)), + ); const videoDevices = shapeVideoDevices(devices.filter((device) => device.kind === 'videoinput')); const nextState: VoiceDeviceState = { inputDevices, @@ -662,8 +584,7 @@ class VoiceDeviceManager { videoDevices, permissionStatus: this.resolvePermissionStatus(requestPermissions, permissionStatus), }; - this.hasEnumeratedDevices = !nativeAudioWarming; - this.lastEnumerationUsedNativeAudio = useNativeAudioDevices; + this.hasEnumeratedDevices = true; logger.debug('Final device state', { inputDeviceCount: inputDevices.length, outputDeviceCount: outputDevices.length, @@ -681,36 +602,6 @@ class VoiceDeviceManager { } } - private async shouldUseNativeAudioDevices(): Promise { - return isVoiceEngineV2AppNativeAudioDeviceBridgeAvailable(); - } - - private async enumerateNativeInputDevices(): Promise> { - const bridge = getNativeVoiceDeviceBridge(); - if (!bridge?.listAudioInputDevices) { - throw new Error('Native audio device bridge lost listAudioInputDevices after availability check'); - } - try { - return shapeNativeAudioInputDevices(await bridge.listAudioInputDevices()); - } catch (error) { - logger.error('Native audio input device enumeration failed', {error}); - return []; - } - } - - private async enumerateNativeOutputDevices(): Promise> { - const bridge = getNativeVoiceDeviceBridge(); - if (!bridge?.listAudioOutputDevices) { - throw new Error('Native audio device bridge lost listAudioOutputDevices after availability check'); - } - try { - return shapeNativeAudioOutputDevices(await bridge.listAudioOutputDevices()); - } catch (error) { - logger.error('Native audio output device enumeration failed', {error}); - return []; - } - } - private resolvePermissionStatus(requestPermissions: boolean, computedStatus: PermissionStatus): PermissionStatus { if (!requestPermissions) { if (this.state.permissionStatus === 'denied') { @@ -735,13 +626,6 @@ class VoiceDeviceManager { this.hasEnumeratedDevices = false; void this.ensureDevices({requestPermissions: this.shouldRequestPermissions}); }; - - private handleNativeAudioDeviceModuleStatusChange = (status: NativeAudioDeviceModuleStatus) => { - if (status !== 'ready') return; - logger.debug('Native audio device module became ready; re-enumerating devices'); - this.hasEnumeratedDevices = false; - void this.ensureDevices({requestPermissions: this.shouldRequestPermissions}); - }; } export const voiceDeviceManager = new VoiceDeviceManager(); diff --git a/fluxer_app/src/index.tsx b/fluxer_app/src/index.tsx index 2eb8e691b..298ba1adc 100644 --- a/fluxer_app/src/index.tsx +++ b/fluxer_app/src/index.tsx @@ -28,7 +28,6 @@ import { } from '@app/features/platform/utils/ClientInfo'; import {loadLazyModule} from '@app/features/platform/utils/LazyModuleLoader'; import {scheduleNonLatinScriptFaces} from '@app/features/theme/fonts/ScriptFontLoader'; -import {initializeNativeVoiceEngineSelectionForStartup} from '@app/features/voice/engine/native_voice_engine/NativeVoiceEngineSelection'; import {i18n} from '@lingui/core'; import {I18nProvider} from '@lingui/react'; import {configure} from 'mobx'; @@ -98,7 +97,6 @@ async function bootstrapThemeStudio(): Promise { } async function bootstrapApp(): Promise { - await initializeNativeVoiceEngineSelectionForStartup(); const [ {App}, {setupHttp},