diff --git a/fluxer_app/src/features/platform/types/Electron.ts b/fluxer_app/src/features/platform/types/Electron.ts index fb79fdf98..0ba724ad4 100644 --- a/fluxer_app/src/features/platform/types/Electron.ts +++ b/fluxer_app/src/features/platform/types/Electron.ts @@ -1,6 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import type {VoiceEngineV2BridgeApi} from '@fluxer/voice_engine_v2/bridge'; +import type {VoiceEngineV2BridgeHardwareEncoderApi} from '@fluxer/voice_engine_v2/bridge'; import type { AuthenticationResponseJSON, PublicKeyCredentialCreationOptionsJSON, @@ -440,7 +440,7 @@ export interface ElectronAPI { selectDisplayMediaSource: (requestId: string, sourceId: string | null, withAudio: boolean) => void; virtmic: VirtmicApi; nativeAudio: NativeAudioApi; - voiceEngine?: VoiceEngineV2BridgeApi; + voiceEngine?: VoiceEngineV2BridgeHardwareEncoderApi; } export type VirtmicUnavailableReason = diff --git a/fluxer_app/src/features/voice/utils/NativeHardwareEncoderCapabilities.ts b/fluxer_app/src/features/voice/utils/NativeHardwareEncoderCapabilities.ts index c44ff868a..761bdb5b0 100644 --- a/fluxer_app/src/features/voice/utils/NativeHardwareEncoderCapabilities.ts +++ b/fluxer_app/src/features/voice/utils/NativeHardwareEncoderCapabilities.ts @@ -18,9 +18,9 @@ let pendingPromise: Promise | n function fetchCapabilities(): Promise { if (!isDesktop()) return Promise.resolve(null); - const bridge = getElectronAPI()?.voiceEngine; - if (!bridge?.getHardwareEncoderCapabilities) return Promise.resolve(null); - return bridge + const voiceEngine = getElectronAPI()?.voiceEngine; + if (!voiceEngine?.getHardwareEncoderCapabilities) return Promise.resolve(null); + return voiceEngine .getHardwareEncoderCapabilities() .then((capabilities) => { const normalized = normalizeVoiceEngineV2HardwareEncoderCapabilities(capabilities); diff --git a/fluxer_app/src/types/electron.d.ts b/fluxer_app/src/types/electron.d.ts index 5614228e4..28e32162d 100644 --- a/fluxer_app/src/types/electron.d.ts +++ b/fluxer_app/src/types/electron.d.ts @@ -1,6 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import type {VoiceEngineV2BridgeApi} from '@fluxer/voice_engine_v2/bridge'; +import type {VoiceEngineV2BridgeHardwareEncoderApi} from '@fluxer/voice_engine_v2/bridge'; import type {AuthenticationResponseJSON, RegistrationResponseJSON} from '@simplewebauthn/browser'; export type InputMonitoringPermissionStatus = 'granted' | 'denied' | 'not-determined' | 'unsupported'; @@ -490,7 +490,7 @@ export interface ElectronAPI { virtmic?: VirtmicApi; nativeAudio?: NativeAudioApi; nativeScreenCapture?: NativeScreenCaptureApi; - voiceEngine?: VoiceEngineV2BridgeApi; + voiceEngine?: VoiceEngineV2BridgeHardwareEncoderApi; } export type VirtmicUnavailableReason = diff --git a/fluxer_desktop/src/common/Types.ts b/fluxer_desktop/src/common/Types.ts index 744f2aaee..f6ba2309e 100644 --- a/fluxer_desktop/src/common/Types.ts +++ b/fluxer_desktop/src/common/Types.ts @@ -1,6 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import type {VoiceEngineV2BridgeApi} from '@fluxer/voice_engine_v2/bridge'; +import type {VoiceEngineV2BridgeHardwareEncoderApi} from '@fluxer/voice_engine_v2/bridge'; import type { AuthenticationResponseJSON, PublicKeyCredentialCreationOptionsJSON, @@ -777,7 +777,7 @@ export interface ElectronAPI { virtmic: VirtmicApi; nativeAudio: NativeAudioApi; nativeScreenCapture: NativeScreenCaptureApi; - voiceEngine: VoiceEngineV2BridgeApi; + voiceEngine: VoiceEngineV2BridgeHardwareEncoderApi; getDesktopSources: ( types: Array<'screen' | 'window'>, requestId?: string, diff --git a/fluxer_desktop/src/main/NativeHardwareEncoder.ts b/fluxer_desktop/src/main/NativeHardwareEncoder.ts index 8cca877ab..27fcd467c 100644 --- a/fluxer_desktop/src/main/NativeHardwareEncoder.ts +++ b/fluxer_desktop/src/main/NativeHardwareEncoder.ts @@ -5,7 +5,7 @@ import {createChildLogger} from '@electron/common/Logger'; import { normalizeVoiceEngineV2BridgeHardwareEncoderCapabilities, unavailableVoiceEngineV2BridgeHardwareEncoderCapabilities, - VOICE_ENGINE_V2_IPC_CHANNELS, + VOICE_ENGINE_V2_HARDWARE_ENCODER_IPC_CHANNEL, type VoiceEngineV2BridgeHardwareEncoderCapabilities, } from '@fluxer/voice_engine_v2/bridge'; import {ipcMain} from 'electron'; @@ -56,13 +56,13 @@ export function registerNativeHardwareEncoderHandlers(): void { if (handlersRegistered) return; handlersRegistered = true; ipcMain.handle( - VOICE_ENGINE_V2_IPC_CHANNELS.getHardwareEncoderCapabilities, + VOICE_ENGINE_V2_HARDWARE_ENCODER_IPC_CHANNEL, (): VoiceEngineV2BridgeHardwareEncoderCapabilities => getHardwareEncoderCapabilities(), ); } export function cleanupNativeHardwareEncoderHandlers(): void { if (!handlersRegistered) return; - ipcMain.removeHandler(VOICE_ENGINE_V2_IPC_CHANNELS.getHardwareEncoderCapabilities); + ipcMain.removeHandler(VOICE_ENGINE_V2_HARDWARE_ENCODER_IPC_CHANNEL); handlersRegistered = false; } diff --git a/fluxer_desktop/src/preload/index.ts b/fluxer_desktop/src/preload/index.ts index e1d2f76e8..9baaa9d28 100644 --- a/fluxer_desktop/src/preload/index.ts +++ b/fluxer_desktop/src/preload/index.ts @@ -60,15 +60,9 @@ import type { VirtmicRoutingGraphResult, VirtmicSystemLinkOptions, } from '@electron/common/Types'; -import type { - VoiceEngineV2BridgeApi, - VoiceEngineV2BridgeEvent, - VoiceEngineV2BridgeVideoFrame, -} from '@fluxer/voice_engine_v2/bridge'; import { - VOICE_ENGINE_V2_BRIDGE_VERSION, - VOICE_ENGINE_V2_EVENT_CHANNELS, - VOICE_ENGINE_V2_IPC_CHANNELS, + VOICE_ENGINE_V2_HARDWARE_ENCODER_IPC_CHANNEL, + type VoiceEngineV2BridgeHardwareEncoderApi, } from '@fluxer/voice_engine_v2/bridge'; import type { AuthenticationResponseJSON, @@ -138,18 +132,6 @@ function validateNativeScreenCaptureLifecycleMessage(value: unknown): NativeScre : {captureId, kind: kind as NativeScreenCaptureLifecycleEventKind, message, source}; } -interface VoiceEngineV2BridgeVideoFrameWire { - meta: VoiceEngineV2BridgeVideoFrame['meta']; - data: Uint8Array; -} - -function videoFrameWireDataToArrayBuffer(data: Uint8Array): ArrayBuffer { - if (data.byteOffset === 0 && data.byteLength === data.buffer.byteLength) { - return data.buffer; - } - return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength); -} - function clampZoomLevel(level: number): number { if (!Number.isFinite(level)) return 1; const pct = Math.round(level * 100); @@ -758,71 +740,8 @@ const api: ElectronAPI = { }, }, voiceEngine: { - bridgeVersion: VOICE_ENGINE_V2_BRIDGE_VERSION, - isSupported: () => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.isSupported), - getCapabilities: () => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.getCapabilities), - prewarm: () => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.prewarm), - getHardwareEncoderCapabilities: () => - ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.getHardwareEncoderCapabilities), - connect: (options) => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.connect, options), - disconnect: () => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.disconnect), - isConnected: () => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.isConnected), - publishMicrophone: (options) => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.publishMicrophone, options), - pushPcm: (frame) => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.pushPcm, frame), - publishScreen: (options) => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.publishScreen, options), - updateScreenShareEncoding: (options) => - ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.updateScreenShareEncoding, options), - unpublishScreen: () => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.unpublishScreen), - publishScreenAudio: (options) => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.publishScreenAudio, options), - pushScreenAudioPcm: (frame) => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.pushScreenAudioPcm, frame), - pushScreenAudioFloat: (frame) => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.pushScreenAudioFloat, frame), - unpublishScreenAudio: () => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.unpublishScreenAudio), - setMicEnabled: (enabled) => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.setMicEnabled, enabled), - setSpeakingDetection: (options) => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.setSpeakingDetection, options), - listAudioInputDevices: () => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.listAudioInputDevices), - listAudioOutputDevices: () => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.listAudioOutputDevices), - setAudioOutputDevice: (deviceId) => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.setAudioOutputDevice, deviceId), - setParticipantVolume: (participantSid, volume) => - ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.setParticipantVolume, {participantSid, volume}), - setRemoteTrackSubscription: (options) => - ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.setRemoteTrackSubscription, options), - publishData: (options) => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.publishData, options), - listCameraDevices: () => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.listCameraDevices), - publishCamera: (options) => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.publishCamera, options), - publishNativeCameraSink: (options) => - ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.publishNativeCameraSink, options), - publishProcessedCamera: (options) => - ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.publishProcessedCamera, options), - pushProcessedCameraFrame: (frame) => - ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.pushProcessedCameraFrame, frame), - pushCameraBackgroundFrame: (frame) => - ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.pushCameraBackgroundFrame, frame), - clearCameraBackgroundFrame: () => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.clearCameraBackgroundFrame), - updateCameraCapture: (options) => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.updateCameraCapture, options), - publishDeviceScreenShare: (options) => - ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.publishDeviceScreenShare, options), - unpublishCamera: () => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.unpublishCamera), - isPublishingCamera: () => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.isPublishingCamera), - startCameraPreview: (options) => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.startCameraPreview, options), - stopCameraPreview: () => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.stopCameraPreview), - getConnectionStats: () => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.getConnectionStats), - getVoiceEngineReadiness: () => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.getVoiceEngineReadiness), - getAudioDeviceModuleState: () => ipcRenderer.invoke(VOICE_ENGINE_V2_IPC_CHANNELS.getAudioDeviceModuleState), - onEvent: (callback) => { - const handler = (_event: Electron.IpcRendererEvent, payload: VoiceEngineV2BridgeEvent): void => { - callback(payload); - }; - ipcRenderer.on(VOICE_ENGINE_V2_EVENT_CHANNELS.event, handler); - return () => ipcRenderer.removeListener(VOICE_ENGINE_V2_EVENT_CHANNELS.event, handler); - }, - onVideoFrame: (callback) => { - const handler = (_event: Electron.IpcRendererEvent, payload: VoiceEngineV2BridgeVideoFrameWire): void => { - callback({meta: payload.meta, data: videoFrameWireDataToArrayBuffer(payload.data)}); - }; - ipcRenderer.on(VOICE_ENGINE_V2_EVENT_CHANNELS.videoFrame, handler); - return () => ipcRenderer.removeListener(VOICE_ENGINE_V2_EVENT_CHANNELS.videoFrame, handler); - }, - } satisfies VoiceEngineV2BridgeApi, + getHardwareEncoderCapabilities: () => ipcRenderer.invoke(VOICE_ENGINE_V2_HARDWARE_ENCODER_IPC_CHANNEL), + } satisfies VoiceEngineV2BridgeHardwareEncoderApi, }; window.addEventListener( diff --git a/packages/voice_engine_v2/fixtures/bridge/bridge_contract.json b/packages/voice_engine_v2/fixtures/bridge/bridge_contract.json deleted file mode 100644 index 90965c8fc..000000000 --- a/packages/voice_engine_v2/fixtures/bridge/bridge_contract.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "version": 18, - "methods": [ - "isSupported", - "getCapabilities", - "prewarm", - "getHardwareEncoderCapabilities", - "connect", - "disconnect", - "isConnected", - "publishMicrophone", - "pushPcm", - "publishScreen", - "updateScreenShareEncoding", - "unpublishScreen", - "publishScreenAudio", - "pushScreenAudioPcm", - "pushScreenAudioFloat", - "unpublishScreenAudio", - "setMicEnabled", - "setSpeakingDetection", - "listAudioInputDevices", - "listAudioOutputDevices", - "setAudioOutputDevice", - "setParticipantVolume", - "setRemoteTrackSubscription", - "publishData", - "listCameraDevices", - "publishCamera", - "updateCameraCapture", - "publishNativeCameraSink", - "publishProcessedCamera", - "pushProcessedCameraFrame", - "pushCameraBackgroundFrame", - "clearCameraBackgroundFrame", - "publishDeviceScreenShare", - "unpublishCamera", - "isPublishingCamera", - "startCameraPreview", - "stopCameraPreview", - "getConnectionStats", - "getVoiceEngineReadiness", - "getAudioDeviceModuleState", - "onEvent", - "onVideoFrame" - ], - "ipcChannels": { - "isSupported": "voice-engine-v2:is-supported", - "getCapabilities": "voice-engine-v2:get-capabilities", - "prewarm": "voice-engine-v2:prewarm", - "getHardwareEncoderCapabilities": "voice-engine-v2:get-hardware-encoder-capabilities", - "connect": "voice-engine-v2:connect", - "disconnect": "voice-engine-v2:disconnect", - "isConnected": "voice-engine-v2:is-connected", - "publishMicrophone": "voice-engine-v2:publish-microphone", - "pushPcm": "voice-engine-v2:push-pcm", - "publishScreen": "voice-engine-v2:publish-screen", - "updateScreenShareEncoding": "voice-engine-v2:update-screen-share-encoding", - "unpublishScreen": "voice-engine-v2:unpublish-screen", - "publishScreenAudio": "voice-engine-v2:publish-screen-audio", - "pushScreenAudioPcm": "voice-engine-v2:push-screen-audio-pcm", - "pushScreenAudioFloat": "voice-engine-v2:push-screen-audio-float", - "unpublishScreenAudio": "voice-engine-v2:unpublish-screen-audio", - "setMicEnabled": "voice-engine-v2:set-mic-enabled", - "setSpeakingDetection": "voice-engine-v2:set-speaking-detection", - "listAudioInputDevices": "voice-engine-v2:list-audio-input-devices", - "listAudioOutputDevices": "voice-engine-v2:list-audio-output-devices", - "setAudioOutputDevice": "voice-engine-v2:set-audio-output-device", - "setParticipantVolume": "voice-engine-v2:set-participant-volume", - "setRemoteTrackSubscription": "voice-engine-v2:set-remote-track-subscription", - "publishData": "voice-engine-v2:publish-data", - "listCameraDevices": "voice-engine-v2:list-camera-devices", - "publishCamera": "voice-engine-v2:publish-camera", - "updateCameraCapture": "voice-engine-v2:update-camera-capture", - "publishNativeCameraSink": "voice-engine-v2:publish-native-camera-sink", - "publishProcessedCamera": "voice-engine-v2:publish-processed-camera", - "pushProcessedCameraFrame": "voice-engine-v2:push-processed-camera-frame", - "pushCameraBackgroundFrame": "voice-engine-v2:push-camera-background-frame", - "clearCameraBackgroundFrame": "voice-engine-v2:clear-camera-background-frame", - "publishDeviceScreenShare": "voice-engine-v2:publish-device-screen-share", - "unpublishCamera": "voice-engine-v2:unpublish-camera", - "isPublishingCamera": "voice-engine-v2:is-publishing-camera", - "startCameraPreview": "voice-engine-v2:start-camera-preview", - "stopCameraPreview": "voice-engine-v2:stop-camera-preview", - "getConnectionStats": "voice-engine-v2:get-connection-stats", - "getVoiceEngineReadiness": "voice-engine-v2:get-readiness", - "getAudioDeviceModuleState": "voice-engine-v2:get-audio-device-module-state" - }, - "eventChannels": { - "event": "voice-engine-v2:event", - "videoFrame": "voice-engine-v2:video-frame" - } -} diff --git a/packages/voice_engine_v2/fixtures/event_logs/app_voice_session.json b/packages/voice_engine_v2/fixtures/event_logs/app_voice_session.json index c656d6a1e..0a8b3a8f0 100644 --- a/packages/voice_engine_v2/fixtures/event_logs/app_voice_session.json +++ b/packages/voice_engine_v2/fixtures/event_logs/app_voice_session.json @@ -658,10 +658,6 @@ "serverRegion": "us-east", "failure": null }, - "nativeAudioDeviceModule": { - "status": "unsupported", - "detail": null - }, "microphone": { "status": "published", "desired": { diff --git a/packages/voice_engine_v2/fixtures/policies/e2ee_state_normalization.json b/packages/voice_engine_v2/fixtures/policies/e2ee_state_normalization.json deleted file mode 100644 index bdd001c2f..000000000 --- a/packages/voice_engine_v2/fixtures/policies/e2ee_state_normalization.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - {"raw": "encrypted", "expected": "encrypted"}, - {"raw": "ok", "expected": "encrypted"}, - {"raw": "decryption_failed", "expected": "decryption_failed"}, - {"raw": "decryptionFailed", "expected": "decryption_failed"}, - {"raw": "error", "expected": "decryption_failed"}, - {"raw": "broken", "expected": "decryption_failed"}, - {"raw": "unencrypted", "expected": "unencrypted"}, - {"raw": "missing_key", "expected": "unencrypted"}, - {"raw": "missingKey", "expected": "unencrypted"}, - {"raw": "internal_error", "expected": "unencrypted"}, - {"raw": "unexpected", "expected": "unknown"}, - {"raw": null, "expected": "unknown"} -] diff --git a/packages/voice_engine_v2/fixtures/policies/local_media_reconnect_suppression.json b/packages/voice_engine_v2/fixtures/policies/local_media_reconnect_suppression.json deleted file mode 100644 index 6c1782c1d..000000000 --- a/packages/voice_engine_v2/fixtures/policies/local_media_reconnect_suppression.json +++ /dev/null @@ -1,56 +0,0 @@ -[ - { - "name": "camera unpublish suppressed during reconnect when video restore is pending", - "input": {"source": "camera", "enabled": false, "reconnecting": true, "restoreVideo": true, "restoreStream": false}, - "expected": true - }, - { - "name": "screen share unpublish suppressed during reconnect when stream restore is pending", - "input": { - "source": "screen_share", - "enabled": false, - "reconnecting": true, - "restoreVideo": false, - "restoreStream": true - }, - "expected": true - }, - { - "name": "publish events are not suppressed", - "input": { - "source": "screen_share", - "enabled": true, - "reconnecting": true, - "restoreVideo": false, - "restoreStream": true - }, - "expected": false - }, - { - "name": "inactive reconnect is not suppressed", - "input": { - "source": "screen_share", - "enabled": false, - "reconnecting": false, - "restoreVideo": false, - "restoreStream": true - }, - "expected": false - }, - { - "name": "camera without video restore is not suppressed", - "input": {"source": "camera", "enabled": false, "reconnecting": true, "restoreVideo": false, "restoreStream": true}, - "expected": false - }, - { - "name": "microphone is not suppressed", - "input": { - "source": "microphone", - "enabled": false, - "reconnecting": true, - "restoreVideo": true, - "restoreStream": true - }, - "expected": false - } -] diff --git a/packages/voice_engine_v2/fixtures/policies/native_connect_retry.json b/packages/voice_engine_v2/fixtures/policies/native_connect_retry.json deleted file mode 100644 index 427bf94bf..000000000 --- a/packages/voice_engine_v2/fixtures/policies/native_connect_retry.json +++ /dev/null @@ -1,98 +0,0 @@ -[ - { - "name": "retry while accepted connection is still connecting before gateway echo", - "input": { - "connectionId": "connection-a", - "guildId": "guild-a", - "channelId": "channel-a", - "voiceState": null, - "connectionState": { - "connectionId": "connection-a", - "guildId": "guild-a", - "channelId": "channel-a", - "connecting": true, - "connected": false, - "reconnecting": false - } - }, - "expected": true - }, - { - "name": "retry when gateway voice state confirms channel", - "input": { - "connectionId": "connection-a", - "guildId": "guild-a", - "channelId": "channel-a", - "voiceState": { - "guild_id": "guild-a", - "channel_id": "channel-a" - }, - "connectionState": { - "connectionId": "stale-connection", - "guildId": "guild-a", - "channelId": "channel-a", - "connecting": false, - "connected": false, - "reconnecting": false - } - }, - "expected": true - }, - { - "name": "reject stale local connection", - "input": { - "connectionId": "connection-a", - "guildId": "guild-a", - "channelId": "channel-a", - "voiceState": null, - "connectionState": { - "connectionId": "connection-b", - "guildId": "guild-a", - "channelId": "channel-a", - "connecting": true, - "connected": false, - "reconnecting": false - } - }, - "expected": false - }, - { - "name": "reject inactive accepted connection", - "input": { - "connectionId": "connection-a", - "guildId": "guild-a", - "channelId": "channel-a", - "voiceState": null, - "connectionState": { - "connectionId": "connection-a", - "guildId": "guild-a", - "channelId": "channel-a", - "connecting": false, - "connected": false, - "reconnecting": false - } - }, - "expected": false - }, - { - "name": "treat null guild ids as equivalent private voice scope", - "input": { - "connectionId": "connection-a", - "guildId": null, - "channelId": "dm-channel-a", - "voiceState": { - "guild_id": null, - "channel_id": "dm-channel-a" - }, - "connectionState": { - "connectionId": "connection-b", - "guildId": null, - "channelId": "dm-channel-a", - "connecting": false, - "connected": false, - "reconnecting": false - } - }, - "expected": true - } -] diff --git a/packages/voice_engine_v2/fixtures/policies/native_participant_volume.json b/packages/voice_engine_v2/fixtures/policies/native_participant_volume.json deleted file mode 100644 index 66307b352..000000000 --- a/packages/voice_engine_v2/fixtures/policies/native_participant_volume.json +++ /dev/null @@ -1,37 +0,0 @@ -[ - { - "name": "local mute wins", - "input": {"userVolumePercent": 200, "outputVolumePercent": 100, "locallyMuted": true}, - "expected": 0 - }, - { - "name": "unity gain", - "input": {"userVolumePercent": 100, "outputVolumePercent": 100, "locallyMuted": false}, - "expected": 1 - }, - { - "name": "max user volume reaches ceiling", - "input": {"userVolumePercent": 200, "outputVolumePercent": 100, "locallyMuted": false}, - "expected": 2 - }, - { - "name": "quiet participant below unity", - "input": {"userVolumePercent": 50, "outputVolumePercent": 100, "locallyMuted": false}, - "expectedRange": {"minExclusive": 0, "maxExclusive": 1} - }, - { - "name": "zero composed volume", - "input": {"userVolumePercent": 0, "outputVolumePercent": 100, "locallyMuted": false}, - "expected": 0 - }, - { - "name": "non finite values fall back to unity inputs", - "input": {"userVolumePercent": null, "outputVolumePercent": null, "locallyMuted": false}, - "expected": 1 - }, - { - "name": "oversized values clamp to native range", - "input": {"userVolumePercent": 999, "outputVolumePercent": 999, "locallyMuted": false}, - "expectedRange": {"minInclusive": 0, "maxInclusive": 2} - } -] diff --git a/packages/voice_engine_v2/src/bridge/admWarmup.test.ts b/packages/voice_engine_v2/src/bridge/admWarmup.test.ts deleted file mode 100644 index f2518164a..000000000 --- a/packages/voice_engine_v2/src/bridge/admWarmup.test.ts +++ /dev/null @@ -1,104 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {describe, expect, it} from 'vitest'; -import { - runVoiceEngineV2AdmWarmup, - VOICE_ENGINE_V2_ADM_WARMUP_POLL_ATTEMPTS_MAX, - VOICE_ENGINE_V2_ADM_WARMUP_POLL_INTERVAL_MS, -} from './admWarmup'; - -describe('runVoiceEngineV2AdmWarmup', () => { - it('resolves ready when the first probe succeeds', async () => { - let probeCalls = 0; - const state = await runVoiceEngineV2AdmWarmup({ - probe: async () => { - probeCalls += 1; - return true; - }, - delay: async () => {}, - }); - expect(state).toEqual({status: 'ready'}); - expect(probeCalls).toBe(1); - }); - - it('polls at the fixed cadence until the probe reports ready', async () => { - let probeCalls = 0; - const delays: Array = []; - const state = await runVoiceEngineV2AdmWarmup({ - probe: async () => { - probeCalls += 1; - return probeCalls >= 5; - }, - delay: async (durationMs) => { - delays.push(durationMs); - }, - }); - expect(state).toEqual({status: 'ready'}); - expect(probeCalls).toBe(5); - expect(delays.every((durationMs) => durationMs === VOICE_ENGINE_V2_ADM_WARMUP_POLL_INTERVAL_MS)).toBe(true); - }); - - it('fails after the bounded attempt count when the probe never reports ready', async () => { - let probeCalls = 0; - const state = await runVoiceEngineV2AdmWarmup({ - probe: async () => { - probeCalls += 1; - return false; - }, - delay: async () => {}, - }); - expect(state.status).toBe('failed'); - expect(state.detail).toContain('deadline'); - expect(probeCalls).toBe(VOICE_ENGINE_V2_ADM_WARMUP_POLL_ATTEMPTS_MAX); - }); - - it('fails with the probe error detail when the probe throws', async () => { - let probeCalls = 0; - const state = await runVoiceEngineV2AdmWarmup({ - probe: async () => { - probeCalls += 1; - throw new Error('addon exploded'); - }, - delay: async () => {}, - }); - expect(state.status).toBe('failed'); - expect(state.detail).toContain('addon exploded'); - expect(probeCalls).toBe(1); - }); - - it('fails at the deadline without stacking probes when the probe hangs forever', async () => { - let probeCalls = 0; - const state = await runVoiceEngineV2AdmWarmup({ - probe: () => { - probeCalls += 1; - return new Promise(() => {}); - }, - delay: async () => {}, - }); - expect(state.status).toBe('failed'); - expect(state.detail).toContain('deadline'); - expect(probeCalls).toBe(1); - }); - - it('resolves ready when a slow probe settles after several poll intervals', async () => { - let probeCalls = 0; - let resolveProbe: ((ready: boolean) => void) | null = null; - let delayCalls = 0; - const state = await runVoiceEngineV2AdmWarmup({ - probe: () => { - probeCalls += 1; - return new Promise((resolve) => { - resolveProbe = resolve; - }); - }, - delay: async () => { - delayCalls += 1; - if (delayCalls === 3) { - resolveProbe?.(true); - } - }, - }); - expect(state).toEqual({status: 'ready'}); - expect(probeCalls).toBe(1); - }); -}); diff --git a/packages/voice_engine_v2/src/bridge/admWarmup.ts b/packages/voice_engine_v2/src/bridge/admWarmup.ts deleted file mode 100644 index 8e58dd9d4..000000000 --- a/packages/voice_engine_v2/src/bridge/admWarmup.ts +++ /dev/null @@ -1,62 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import type {VoiceEngineV2BridgeAudioDeviceModuleState} from './types'; - -export const VOICE_ENGINE_V2_ADM_WARMUP_POLL_INTERVAL_MS = 250; -export const VOICE_ENGINE_V2_ADM_WARMUP_DEADLINE_MS = 60_000; -export const VOICE_ENGINE_V2_ADM_WARMUP_POLL_ATTEMPTS_MAX = Math.floor( - VOICE_ENGINE_V2_ADM_WARMUP_DEADLINE_MS / VOICE_ENGINE_V2_ADM_WARMUP_POLL_INTERVAL_MS, -); - -export interface VoiceEngineV2AdmWarmupPorts { - probe(): Promise; - delay(durationMs: number): Promise; -} - -type AdmProbeOutcome = {kind: 'ready'; ready: boolean} | {kind: 'error'; detail: string}; - -const ADM_PROBE_STILL_PENDING = Symbol('voice-engine-v2-adm-probe-still-pending'); - -function startBoundedAdmProbe(ports: VoiceEngineV2AdmWarmupPorts): Promise { - return ports.probe().then( - (ready) => ({kind: 'ready', ready}), - (error) => { - const detail = error instanceof Error ? error.message : String(error); - return {kind: 'error', detail: `audio device module probe failed: ${detail}`}; - }, - ); -} - -export async function runVoiceEngineV2AdmWarmup( - ports: VoiceEngineV2AdmWarmupPorts, -): Promise { - if (VOICE_ENGINE_V2_ADM_WARMUP_POLL_ATTEMPTS_MAX < 1) { - throw new Error('ADM warmup poll attempt bound must be at least 1'); - } - let pendingProbe: Promise | null = null; - for (let attempt = 1; attempt <= VOICE_ENGINE_V2_ADM_WARMUP_POLL_ATTEMPTS_MAX; attempt++) { - if (pendingProbe === null) { - pendingProbe = startBoundedAdmProbe(ports); - } - const attemptTimeout = ports.delay(VOICE_ENGINE_V2_ADM_WARMUP_POLL_INTERVAL_MS).then(() => ADM_PROBE_STILL_PENDING); - const outcome = await Promise.race([pendingProbe, attemptTimeout]); - if (outcome === ADM_PROBE_STILL_PENDING) { - continue; - } - pendingProbe = null; - const probeOutcome = outcome as AdmProbeOutcome; - if (probeOutcome.kind === 'error') { - return {status: 'failed', detail: probeOutcome.detail}; - } - if (probeOutcome.ready) { - return {status: 'ready'}; - } - if (attempt < VOICE_ENGINE_V2_ADM_WARMUP_POLL_ATTEMPTS_MAX) { - await ports.delay(VOICE_ENGINE_V2_ADM_WARMUP_POLL_INTERVAL_MS); - } - } - return { - status: 'failed', - detail: `audio device module warmup exceeded ${VOICE_ENGINE_V2_ADM_WARMUP_DEADLINE_MS}ms deadline`, - }; -} diff --git a/packages/voice_engine_v2/src/bridge/bridge_contract.test.ts b/packages/voice_engine_v2/src/bridge/bridge_contract.test.ts index 3c32cb369..d047014b4 100644 --- a/packages/voice_engine_v2/src/bridge/bridge_contract.test.ts +++ b/packages/voice_engine_v2/src/bridge/bridge_contract.test.ts @@ -1,10 +1,8 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import {beforeEach, describe, expect, it} from 'vitest'; -import fixtureJson from '../../fixtures/bridge/bridge_contract.json'; +import {describe, expect, it} from 'vitest'; import { assertAudioFrameInvariants, - assertSchemaVersion, assertVideoFrameInvariants, VOICE_ENGINE_V2_AUDIO_FRAME_BYTES_MAX, VOICE_ENGINE_V2_VIDEO_DIMENSION_MAX, @@ -13,125 +11,15 @@ import { } from './ffi_assertions'; import { assertVoiceEngineV2BridgeAudioOptionsInvariants, - assertVoiceEngineV2BridgeEnvelopeSchema, assertVoiceEngineV2BridgeVideoOptionsInvariants, - getVoiceEngineV2BridgeDroppedEventCounts, - getVoiceEngineV2BridgeProcessedCameraI420ByteLength, - isVoiceEngineV2BridgeFloatPcmFrame, - isVoiceEngineV2BridgeKnownEventType, - isVoiceEngineV2BridgeProcessedCameraFrame, - isVoiceEngineV2BridgePublishCameraOptions, - isVoiceEngineV2BridgePublishProcessedCameraOptions, - isVoiceEngineV2BridgeReadiness, - resetVoiceEngineV2BridgeDroppedEventCounts, - translateVoiceEngineV2BridgeEventToEvents, - translateVoiceEngineV2BridgeVideoFrameToEvent, - VOICE_ENGINE_V2_ADM_STATUS_EVENT_TYPE, - VOICE_ENGINE_V2_BRIDGE_METHODS, - VOICE_ENGINE_V2_BRIDGE_VERSION, - VOICE_ENGINE_V2_ENGINE_READY_EVENT_TYPE, - VOICE_ENGINE_V2_EVENT_CHANNELS, - VOICE_ENGINE_V2_IPC_CHANNELS, + normalizeVoiceEngineV2BridgeHardwareEncoderCapabilities, + unavailableVoiceEngineV2BridgeHardwareEncoderCapabilities, + VOICE_ENGINE_V2_HARDWARE_ENCODER_IPC_CHANNEL, } from './index'; -interface VoiceEngineV2BridgeContractFixture { - version: number; - methods: Array; - ipcChannels: Record; - eventChannels: Record; -} - describe('voice engine v2 bridge contract', () => { - it('matches the shared bridge contract fixture', () => { - const fixture = fixtureJson as VoiceEngineV2BridgeContractFixture; - - expect(VOICE_ENGINE_V2_BRIDGE_VERSION).toBe(fixture.version); - expect(VOICE_ENGINE_V2_BRIDGE_METHODS).toEqual(fixture.methods); - expect(VOICE_ENGINE_V2_IPC_CHANNELS).toEqual(fixture.ipcChannels); - expect(VOICE_ENGINE_V2_EVENT_CHANNELS).toEqual(fixture.eventChannels); - expect(new Set(VOICE_ENGINE_V2_BRIDGE_METHODS).size).toBe(VOICE_ENGINE_V2_BRIDGE_METHODS.length); - expect(new Set(Object.values(VOICE_ENGINE_V2_IPC_CHANNELS)).size).toBe( - Object.values(VOICE_ENGINE_V2_IPC_CHANNELS).length, - ); - expect(new Set(Object.values(VOICE_ENGINE_V2_EVENT_CHANNELS)).size).toBe( - Object.values(VOICE_ENGINE_V2_EVENT_CHANNELS).length, - ); - }); - - it('accepts the canonical schema version on the FFI boundary', () => { - expect(() => assertSchemaVersion(VOICE_ENGINE_V2_BRIDGE_VERSION)).not.toThrow(); - }); - - it('rejects a schema version older than the bridge', () => { - try { - assertSchemaVersion(VOICE_ENGINE_V2_BRIDGE_VERSION - 1); - expect.fail('expected schema mismatch to throw'); - } catch (error) { - expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError); - expect((error as VoiceEngineV2FfiAssertError).code).toBe('schemaVersionMismatch'); - } - }); - - it('rejects a schema version newer than the bridge', () => { - try { - assertSchemaVersion(VOICE_ENGINE_V2_BRIDGE_VERSION + 1); - expect.fail('expected schema mismatch to throw'); - } catch (error) { - expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError); - expect((error as VoiceEngineV2FfiAssertError).code).toBe('schemaVersionMismatch'); - } - }); -}); - -describe('voice engine v2 readiness contract', () => { - it('registers the readiness method and channel in the shared contract', () => { - expect(VOICE_ENGINE_V2_BRIDGE_METHODS).toContain('getVoiceEngineReadiness'); - expect(VOICE_ENGINE_V2_IPC_CHANNELS.getVoiceEngineReadiness).toBe('voice-engine-v2:get-readiness'); - }); - - it('classifies engineReady as a known bridge event type', () => { - expect(VOICE_ENGINE_V2_ENGINE_READY_EVENT_TYPE).toBe('engineReady'); - expect(isVoiceEngineV2BridgeKnownEventType(VOICE_ENGINE_V2_ENGINE_READY_EVENT_TYPE)).toBe(true); - }); - - it('translates engineReady as a renderer-gating event with no reducer output', () => { - resetVoiceEngineV2BridgeDroppedEventCounts(); - const events = translateVoiceEngineV2BridgeEventToEvents({ - type: 'engineReady', - payload: {ready: true}, - }); - - expect(events).toEqual([]); - const counts = getVoiceEngineV2BridgeDroppedEventCounts(); - expect(counts.malformedPayloadCount).toBe(0); - expect(counts.unknownTypeCount).toBe(0); - }); - - it('translates audio device module status into a runtime event', () => { - resetVoiceEngineV2BridgeDroppedEventCounts(); - const events = translateVoiceEngineV2BridgeEventToEvents({ - type: VOICE_ENGINE_V2_ADM_STATUS_EVENT_TYPE, - payload: {status: 'warming'}, - }); - - expect(events).toEqual([{type: 'nativeAudioDeviceModule.statusChanged', status: 'warming'}]); - const counts = getVoiceEngineV2BridgeDroppedEventCounts(); - expect(counts.malformedPayloadCount).toBe(0); - expect(counts.unknownTypeCount).toBe(0); - }); - - it('accepts ready and not-ready readiness payloads', () => { - expect(isVoiceEngineV2BridgeReadiness({ready: true})).toBe(true); - expect(isVoiceEngineV2BridgeReadiness({ready: false})).toBe(true); - expect(isVoiceEngineV2BridgeReadiness({ready: false, reason: 'native voice engine prewarm pending'})).toBe(true); - }); - - it('rejects malformed readiness payloads', () => { - expect(isVoiceEngineV2BridgeReadiness(null)).toBe(false); - expect(isVoiceEngineV2BridgeReadiness({})).toBe(false); - expect(isVoiceEngineV2BridgeReadiness({ready: 'yes'})).toBe(false); - expect(isVoiceEngineV2BridgeReadiness({ready: false, reason: 42})).toBe(false); - expect(isVoiceEngineV2BridgeReadiness({ready: true, reason: 'ready engines carry no reason'})).toBe(false); + it('pins the hardware encoder ipc channel shared by preload and the main process', () => { + expect(VOICE_ENGINE_V2_HARDWARE_ENCODER_IPC_CHANNEL).toBe('voice-engine-v2:get-hardware-encoder-capabilities'); }); }); @@ -151,37 +39,37 @@ describe('voice engine v2 audio frame invariants', () => { expect(() => assertAudioFrameInvariants(canonicalAudioFrame, 500)).not.toThrow(); }); - it('rejects an audio frame with zero bytes', () => { + it('rejects an empty audio frame', () => { try { assertAudioFrameInvariants({...canonicalAudioFrame, frameBytes: 0}); - expect.fail('expected size assertion to throw'); + expect.fail('expected frame bytes assertion to throw'); } catch (error) { expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError); expect((error as VoiceEngineV2FfiAssertError).code).toBe('audioFrameBytesOutOfRange'); } }); - it('rejects an audio frame over the byte cap', () => { + it('rejects an audio frame past the byte cap', () => { try { assertAudioFrameInvariants({...canonicalAudioFrame, frameBytes: VOICE_ENGINE_V2_AUDIO_FRAME_BYTES_MAX + 1}); - expect.fail('expected size assertion to throw'); + expect.fail('expected frame bytes assertion to throw'); } catch (error) { expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError); expect((error as VoiceEngineV2FfiAssertError).code).toBe('audioFrameBytesOutOfRange'); } }); - it('rejects an audio frame with an unsupported sample rate', () => { + it('rejects an unsupported sample rate', () => { try { assertAudioFrameInvariants({...canonicalAudioFrame, sampleRateHz: 44100}); - expect.fail('expected sample-rate assertion to throw'); + expect.fail('expected sample rate assertion to throw'); } catch (error) { expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError); expect((error as VoiceEngineV2FfiAssertError).code).toBe('audioSampleRateInvalid'); } }); - it('rejects an audio frame with an unsupported channel count', () => { + it('rejects an unsupported channel count', () => { try { assertAudioFrameInvariants({...canonicalAudioFrame, numChannels: 3}); expect.fail('expected channels assertion to throw'); @@ -191,7 +79,7 @@ describe('voice engine v2 audio frame invariants', () => { } }); - it('rejects an audio frame whose timestamp does not advance', () => { + it('rejects a repeated audio timestamp', () => { try { assertAudioFrameInvariants(canonicalAudioFrame, canonicalAudioFrame.timestampNs); expect.fail('expected timestamp assertion to throw'); @@ -201,7 +89,7 @@ describe('voice engine v2 audio frame invariants', () => { } }); - it('rejects an audio frame whose timestamp moves backwards', () => { + it('rejects a regressed audio timestamp', () => { try { assertAudioFrameInvariants(canonicalAudioFrame, canonicalAudioFrame.timestampNs + 1); expect.fail('expected timestamp assertion to throw'); @@ -216,35 +104,35 @@ describe('voice engine v2 video frame invariants', () => { const canonicalVideoFrame = { widthPx: 1920, heightPx: 1080, - frameBytes: (1920 * 1080 * 3) / 2, + frameBytes: 1920 * 1080 * 4, timestampNs: 1000, }; - it('accepts a canonical video frame with no previous timestamp', () => { + it('accepts a canonical video frame', () => { expect(() => assertVideoFrameInvariants(canonicalVideoFrame)).not.toThrow(); }); - it('rejects a video frame with zero bytes', () => { + it('rejects an empty video frame', () => { try { assertVideoFrameInvariants({...canonicalVideoFrame, frameBytes: 0}); - expect.fail('expected size assertion to throw'); + expect.fail('expected frame bytes assertion to throw'); } catch (error) { expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError); expect((error as VoiceEngineV2FfiAssertError).code).toBe('videoFrameBytesOutOfRange'); } }); - it('rejects a video frame over the byte cap', () => { + it('rejects a video frame past the byte cap', () => { try { assertVideoFrameInvariants({...canonicalVideoFrame, frameBytes: VOICE_ENGINE_V2_VIDEO_FRAME_BYTES_MAX + 1}); - expect.fail('expected size assertion to throw'); + expect.fail('expected frame bytes assertion to throw'); } catch (error) { expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError); expect((error as VoiceEngineV2FfiAssertError).code).toBe('videoFrameBytesOutOfRange'); } }); - it('rejects a video frame with a zero dimension', () => { + it('rejects a zero video dimension', () => { try { assertVideoFrameInvariants({...canonicalVideoFrame, widthPx: 0}); expect.fail('expected dimension assertion to throw'); @@ -254,7 +142,7 @@ describe('voice engine v2 video frame invariants', () => { } }); - it('rejects a video frame with an oversized dimension', () => { + it('rejects a video dimension past the cap', () => { try { assertVideoFrameInvariants({...canonicalVideoFrame, heightPx: VOICE_ENGINE_V2_VIDEO_DIMENSION_MAX + 1}); expect.fail('expected dimension assertion to throw'); @@ -264,7 +152,7 @@ describe('voice engine v2 video frame invariants', () => { } }); - it('rejects a video frame whose timestamp regresses', () => { + it('rejects a repeated video timestamp', () => { try { assertVideoFrameInvariants(canonicalVideoFrame, canonicalVideoFrame.timestampNs); expect.fail('expected timestamp assertion to throw'); @@ -273,183 +161,6 @@ describe('voice engine v2 video frame invariants', () => { expect((error as VoiceEngineV2FfiAssertError).code).toBe('videoTimestampRegressed'); } }); - - it('documents that inboundVideo.frameReceived will activate this guard once the port leaves shadow', () => { - const hotPathTypes = ['inboundVideo.frameReceived', 'nativeCapture.frame']; - expect(hotPathTypes.length).toBeGreaterThan(0); - for (const type of hotPathTypes) { - expect(typeof type).toBe('string'); - } - }); -}); - -describe('voice engine v2 bridge schema version literal', () => { - it('pins the TS bridge version to the literal 18 to pair with the native addon bridge', () => { - expect(VOICE_ENGINE_V2_BRIDGE_VERSION).toBe(18); - }); - - it('reuses the schema-version airlock helper from bridge exports', () => { - expect(() => assertVoiceEngineV2BridgeEnvelopeSchema(VOICE_ENGINE_V2_BRIDGE_VERSION)).not.toThrow(); - }); - - it('rejects bridge schema mismatch via the exported airlock helper', () => { - try { - assertVoiceEngineV2BridgeEnvelopeSchema(VOICE_ENGINE_V2_BRIDGE_VERSION + 1); - expect.fail('expected schema mismatch to throw'); - } catch (error) { - expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError); - expect((error as VoiceEngineV2FfiAssertError).code).toBe('schemaVersionMismatch'); - } - }); -}); - -describe('voice engine v2 float pcm bridge frames', () => { - it('accepts Float32Array payloads for native-side PCM conversion', () => { - expect( - isVoiceEngineV2BridgeFloatPcmFrame({ - sampleRate: 48_000, - numChannels: 1, - samples: new Float32Array([0, 0.5, -0.5]), - }), - ).toBe(true); - }); - - it('rejects non-float payloads on the float PCM bridge', () => { - expect( - isVoiceEngineV2BridgeFloatPcmFrame({ - sampleRate: 48_000, - numChannels: 1, - samples: new Uint8Array([0, 1, 2, 3]).buffer, - }), - ).toBe(false); - }); -}); - -describe('voice engine v2 bridge translator airlock', () => { - function makeFrame(width: number, height: number, byteLength: number, timestampUs: number) { - return { - meta: {participantSid: 'PA', trackSid: 'TR', width, height, timestampUs}, - data: new ArrayBuffer(byteLength), - }; - } - - it('translates a canonical video frame received from Rust', () => { - const event = translateVoiceEngineV2BridgeVideoFrameToEvent(makeFrame(1920, 1080, 4096, 1)); - expect(event.type).toBe('inboundVideo.frameReceived'); - }); - - it('panics through the airlock when an inbound video frame has zero dimensions', () => { - try { - translateVoiceEngineV2BridgeVideoFrameToEvent(makeFrame(0, 1080, 4096, 1)); - expect.fail('expected dimension assertion to throw'); - } catch (error) { - expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError); - expect((error as VoiceEngineV2FfiAssertError).code).toBe('videoDimensionOutOfRange'); - } - }); - - it('panics through the airlock when an inbound video frame has zero bytes', () => { - try { - translateVoiceEngineV2BridgeVideoFrameToEvent(makeFrame(1920, 1080, 0, 1)); - expect.fail('expected size assertion to throw'); - } catch (error) { - expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError); - expect((error as VoiceEngineV2FfiAssertError).code).toBe('videoFrameBytesOutOfRange'); - } - }); -}); - -describe('voice engine v2 processed camera bridge airlock', () => { - it('accepts native camera background modes', () => { - expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundMode: 'none'})).toBe(true); - expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundMode: 'non'})).toBe(true); - expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundMode: 'blur'})).toBe(true); - expect( - isVoiceEngineV2BridgePublishCameraOptions({ - backgroundMode: 'custom', - backgroundCustomMediaPath: '/tmp/background.webm', - backgroundCustomMediaKind: 'video', - }), - ).toBe(true); - }); - - it('registers the camera capture update channel in the shared contract', () => { - expect(VOICE_ENGINE_V2_BRIDGE_METHODS).toContain('updateCameraCapture'); - expect(VOICE_ENGINE_V2_IPC_CHANNELS.updateCameraCapture).toBe('voice-engine-v2:update-camera-capture'); - }); - - it('accepts native camera mirroring as a typed boolean only', () => { - expect(isVoiceEngineV2BridgePublishCameraOptions({mirror: true})).toBe(true); - expect(isVoiceEngineV2BridgePublishCameraOptions({mirror: false})).toBe(true); - expect(isVoiceEngineV2BridgePublishCameraOptions({mirror: 'true'})).toBe(false); - }); - - it('accepts camera effect strengths across the full integer range', () => { - expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundBlurStrength: 0})).toBe(true); - expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundBlurStrength: 100})).toBe(true); - expect( - isVoiceEngineV2BridgePublishCameraOptions({ - backgroundMode: 'blur', - backgroundBlurStrength: 75, - }), - ).toBe(true); - }); - - it('rejects out-of-range or non-integer camera effect strengths', () => { - expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundBlurStrength: -1})).toBe(false); - expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundBlurStrength: 101})).toBe(false); - expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundBlurStrength: 49.5})).toBe(false); - expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundBlurStrength: '50'})).toBe(false); - expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundBlurStrength: Number.NaN})).toBe(false); - }); - - it('rejects invalid native camera background modes', () => { - expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundMode: 'sepia'})).toBe(false); - expect( - isVoiceEngineV2BridgePublishCameraOptions({ - backgroundMode: 'custom', - backgroundCustomMediaPath: '/tmp/background.webm', - backgroundCustomMediaKind: 'audio', - }), - ).toBe(false); - }); - - it('accepts canonical processed camera publish options', () => { - expect(isVoiceEngineV2BridgePublishProcessedCameraOptions({width: 1280, height: 720, frameRate: 30})).toBe(true); - }); - - it('rejects processed camera publish options with odd dimensions', () => { - expect(isVoiceEngineV2BridgePublishProcessedCameraOptions({width: 1279, height: 720, frameRate: 30})).toBe(false); - }); - - it('calculates tight i420 payload byte length', () => { - expect(getVoiceEngineV2BridgeProcessedCameraI420ByteLength(4, 2)).toBe(12); - }); - - it('accepts tight i420 processed camera frames', () => { - const data = new Uint8Array(getVoiceEngineV2BridgeProcessedCameraI420ByteLength(4, 2)); - expect( - isVoiceEngineV2BridgeProcessedCameraFrame({ - format: 'i420', - width: 4, - height: 2, - timestampUs: 1, - data, - }), - ).toBe(true); - }); - - it('rejects processed camera frames with loose byte lengths', () => { - expect( - isVoiceEngineV2BridgeProcessedCameraFrame({ - format: 'i420', - width: 4, - height: 2, - timestampUs: 1, - data: new ArrayBuffer(11), - }), - ).toBe(false); - }); }); describe('voice engine v2 bridge option airlock helpers', () => { @@ -502,61 +213,64 @@ describe('voice engine v2 bridge option airlock helpers', () => { }); }); -describe('voice engine v2 bridge event translation drop accounting', () => { - beforeEach(() => { - resetVoiceEngineV2BridgeDroppedEventCounts(); - }); - - it('returns no events for a known-but-ignored type without counting a drop', () => { - const events = translateVoiceEngineV2BridgeEventToEvents({ - type: 'stats', - payload: {rttMs: 12, outbound: [], inbound: []}, +describe('voice engine v2 bridge hardware encoder capabilities', () => { + it('reports an unavailable encoder with the supplied reason and detail', () => { + expect(unavailableVoiceEngineV2BridgeHardwareEncoderCapabilities('load-failed', 'addon missing')).toEqual({ + available: false, + backend: 'none', + compiled: false, + runtime: false, + codecs: [], + zeroCopy: false, + nativeInputs: [], + reason: 'load-failed', + detail: 'addon missing', }); - - expect(events).toEqual([]); - const counts = getVoiceEngineV2BridgeDroppedEventCounts(); - expect(counts.malformedPayloadCount).toBe(0); - expect(counts.unknownTypeCount).toBe(0); }); - it('counts a malformed payload for a translated known type', () => { - const events = translateVoiceEngineV2BridgeEventToEvents({ - type: 'trackMuted', - payload: {}, + it('omits the detail field when the caller supplies none', () => { + expect(unavailableVoiceEngineV2BridgeHardwareEncoderCapabilities('unsupported-addon-version')).not.toHaveProperty( + 'detail', + ); + }); + + it('normalizes a well-formed addon payload', () => { + expect( + normalizeVoiceEngineV2BridgeHardwareEncoderCapabilities({ + available: true, + backend: 'nvenc', + compiled: true, + runtime: true, + codecs: ['h264', 42, 'av1'], + zeroCopy: true, + nativeInputs: ['d3d11Texture', null], + }), + ).toEqual({ + available: true, + backend: 'nvenc', + compiled: true, + runtime: true, + codecs: ['h264', 'av1'], + zeroCopy: true, + nativeInputs: ['d3d11Texture'], }); - - expect(events).toEqual([]); - const counts = getVoiceEngineV2BridgeDroppedEventCounts(); - expect(counts.malformedPayloadCount).toBe(1); - expect(counts.unknownTypeCount).toBe(0); }); - it('counts an unknown event type as a dropped event', () => { - const events = translateVoiceEngineV2BridgeEventToEvents({ - type: 'someFutureEvent', - payload: {}, + it('falls back to an unavailable encoder when the addon returns a non-object', () => { + expect(normalizeVoiceEngineV2BridgeHardwareEncoderCapabilities(null)).toEqual({ + available: false, + backend: 'none', + compiled: false, + runtime: false, + codecs: [], + zeroCopy: false, + nativeInputs: [], + reason: 'query-failed', + detail: 'Native addon returned an invalid result', }); - - expect(events).toEqual([]); - const counts = getVoiceEngineV2BridgeDroppedEventCounts(); - expect(counts.malformedPayloadCount).toBe(0); - expect(counts.unknownTypeCount).toBe(1); }); - it('classifies every payload key as a known bridge event type', () => { - expect(isVoiceEngineV2BridgeKnownEventType('trackSubscribed')).toBe(true); - expect(isVoiceEngineV2BridgeKnownEventType('audioPlaybackUnavailable')).toBe(true); - expect(isVoiceEngineV2BridgeKnownEventType('someFutureEvent')).toBe(false); - }); - - it('still translates a well-formed known event after counting drops', () => { - translateVoiceEngineV2BridgeEventToEvents({type: 'trackMuted', payload: {}}); - const events = translateVoiceEngineV2BridgeEventToEvents({ - type: 'trackMuted', - payload: {trackSid: 'TR_1'}, - }); - - expect(events).toEqual([{type: 'room.trackMuted', trackSid: 'TR_1'}]); - expect(getVoiceEngineV2BridgeDroppedEventCounts().malformedPayloadCount).toBe(1); + it('falls back to the none backend when the addon omits one', () => { + expect(normalizeVoiceEngineV2BridgeHardwareEncoderCapabilities({available: true}).backend).toBe('none'); }); }); diff --git a/packages/voice_engine_v2/src/bridge/ffi_assertions.ts b/packages/voice_engine_v2/src/bridge/ffi_assertions.ts index 127087adc..a757f82c2 100644 --- a/packages/voice_engine_v2/src/bridge/ffi_assertions.ts +++ b/packages/voice_engine_v2/src/bridge/ffi_assertions.ts @@ -1,7 +1,5 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import {VOICE_ENGINE_V2_BRIDGE_VERSION} from './index'; - export const VOICE_ENGINE_V2_AUDIO_FRAME_BYTES_MAX = 1 << 20; export const VOICE_ENGINE_V2_VIDEO_FRAME_BYTES_MAX = 64 << 20; export const VOICE_ENGINE_V2_VIDEO_DIMENSION_MAX = 16384; @@ -24,7 +22,6 @@ interface VoiceEngineV2VideoFrameInvariants { export class VoiceEngineV2FfiAssertError extends Error { readonly code: - | 'schemaVersionMismatch' | 'audioFrameBytesOutOfRange' | 'audioSampleRateInvalid' | 'audioChannelsInvalid' @@ -44,21 +41,6 @@ function isFiniteInteger(value: number): boolean { return Number.isFinite(value) && Number.isInteger(value); } -export function assertSchemaVersion(received: number): void { - if (!isFiniteInteger(received)) { - throw new VoiceEngineV2FfiAssertError( - 'schemaVersionMismatch', - `schema version must be a finite integer (received ${received})`, - ); - } - if (received !== VOICE_ENGINE_V2_BRIDGE_VERSION) { - throw new VoiceEngineV2FfiAssertError( - 'schemaVersionMismatch', - `schema version mismatch: expected ${VOICE_ENGINE_V2_BRIDGE_VERSION}, received ${received}`, - ); - } -} - function assertAudioFrameBytes(bytes: number): void { if (!isFiniteInteger(bytes)) { throw new VoiceEngineV2FfiAssertError( diff --git a/packages/voice_engine_v2/src/bridge/guards.ts b/packages/voice_engine_v2/src/bridge/guards.ts deleted file mode 100644 index 1389588a3..000000000 --- a/packages/voice_engine_v2/src/bridge/guards.ts +++ /dev/null @@ -1,482 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {assertAudioFrameInvariants, assertSchemaVersion, assertVideoFrameInvariants} from './ffi_assertions'; -import type { - VoiceEngineV2BridgeCameraBackgroundCustomMediaKind, - VoiceEngineV2BridgeCameraBackgroundMode, - VoiceEngineV2BridgeCameraPreviewInfo, - VoiceEngineV2BridgeCapabilities, - VoiceEngineV2BridgeConnectOptions, - VoiceEngineV2BridgeFloatPcmFrame, - VoiceEngineV2BridgeHardwareEncoderCapabilities, - VoiceEngineV2BridgeParticipantVolumeOptions, - VoiceEngineV2BridgePcmFrame, - VoiceEngineV2BridgeProcessedCameraFrame, - VoiceEngineV2BridgePublishCameraOptions, - VoiceEngineV2BridgePublishDataOptions, - VoiceEngineV2BridgePublishDeviceScreenShareOptions, - VoiceEngineV2BridgePublishMicrophoneOptions, - VoiceEngineV2BridgePublishProcessedCameraOptions, - VoiceEngineV2BridgePublishProcessedCameraResult, - VoiceEngineV2BridgePublishScreenAudioOptions, - VoiceEngineV2BridgePublishScreenOptions, - VoiceEngineV2BridgeReadiness, - VoiceEngineV2BridgeRemoteTrackSubscriptionOptions, - VoiceEngineV2BridgeRemoteTrackSubscriptionQuality, - VoiceEngineV2BridgeSpeakingDetectionOptions, - VoiceEngineV2BridgeUpdateScreenShareEncodingOptions, -} from './types'; -import {VOICE_ENGINE_V2_CAMERA_EFFECT_STRENGTH_MAX, VOICE_ENGINE_V2_CAMERA_EFFECT_STRENGTH_MIN} from './types'; - -export function assertVoiceEngineV2BridgeAudioOptionsInvariants(options: { - sampleRate: number; - numChannels: number; -}): void { - assertAudioFrameInvariants({ - sampleRateHz: options.sampleRate, - numChannels: options.numChannels, - frameBytes: 1, - timestampNs: 1, - }); -} - -export function assertVoiceEngineV2BridgeVideoOptionsInvariants(options: {width: number; height: number}): void { - assertVideoFrameInvariants({ - widthPx: options.width, - heightPx: options.height, - frameBytes: 1, - timestampNs: 1, - }); -} - -export function assertVoiceEngineV2BridgeEnvelopeSchema(received: number): void { - assertSchemaVersion(received); -} - -export function unavailableVoiceEngineV2BridgeCapabilities(): VoiceEngineV2BridgeCapabilities { - return { - microphoneCapture: false, - syntheticMicrophonePcm: false, - cameraCapture: false, - nativeCameraBackgrounds: false, - screenShare: false, - screenShareEncodingUpdate: false, - screenShareAudio: false, - deviceLists: false, - outputDeviceSelection: false, - participantVolume: false, - remoteTrackSubscription: false, - dataChannel: false, - connectionStats: false, - nativeVideoFrames: false, - hardwareEncoderCapabilities: false, - }; -} - -export function normalizeVoiceEngineV2BridgeCapabilities(value: unknown): VoiceEngineV2BridgeCapabilities { - if (typeof value !== 'object' || value === null) { - return unavailableVoiceEngineV2BridgeCapabilities(); - } - const candidate = value as Partial & { - microphone?: boolean; - camera?: boolean; - cameraBackgrounds?: boolean; - screen?: boolean; - screenAudio?: boolean; - outputDevice?: boolean; - dataChannel?: boolean; - stats?: boolean; - hardwareEncoding?: boolean; - }; - return { - microphoneCapture: candidate.microphoneCapture === true || candidate.microphone === true, - syntheticMicrophonePcm: candidate.syntheticMicrophonePcm === true, - cameraCapture: candidate.cameraCapture === true || candidate.camera === true, - nativeCameraBackgrounds: candidate.nativeCameraBackgrounds === true || candidate.cameraBackgrounds === true, - screenShare: candidate.screenShare === true || candidate.screen === true, - screenShareEncodingUpdate: candidate.screenShareEncodingUpdate === true, - screenShareAudio: candidate.screenShareAudio === true || candidate.screenAudio === true, - deviceLists: candidate.deviceLists === true, - outputDeviceSelection: candidate.outputDeviceSelection === true || candidate.outputDevice === true, - participantVolume: candidate.participantVolume === true, - remoteTrackSubscription: candidate.remoteTrackSubscription === true, - dataChannel: candidate.dataChannel === true, - connectionStats: candidate.connectionStats === true || candidate.stats === true, - nativeVideoFrames: candidate.nativeVideoFrames === true, - hardwareEncoderCapabilities: candidate.hardwareEncoderCapabilities === true || candidate.hardwareEncoding === true, - }; -} - -export function unavailableVoiceEngineV2BridgeHardwareEncoderCapabilities( - reason: NonNullable, - detail?: string, -): VoiceEngineV2BridgeHardwareEncoderCapabilities { - return { - available: false, - backend: 'none', - compiled: false, - runtime: false, - codecs: [], - zeroCopy: false, - nativeInputs: [], - reason, - ...(detail ? {detail} : {}), - }; -} - -export function normalizeVoiceEngineV2BridgeHardwareEncoderCapabilities( - value: unknown, -): VoiceEngineV2BridgeHardwareEncoderCapabilities { - if (typeof value !== 'object' || value === null) { - return unavailableVoiceEngineV2BridgeHardwareEncoderCapabilities( - 'query-failed', - 'Native addon returned an invalid result', - ); - } - const candidate = value as Partial; - const backend = typeof candidate.backend === 'string' && candidate.backend.length > 0 ? candidate.backend : 'none'; - return { - available: candidate.available === true, - backend, - compiled: candidate.compiled === true, - runtime: candidate.runtime === true, - codecs: Array.isArray(candidate.codecs) ? candidate.codecs.filter((codec) => typeof codec === 'string') : [], - zeroCopy: candidate.zeroCopy === true, - nativeInputs: Array.isArray(candidate.nativeInputs) - ? candidate.nativeInputs.filter((input) => typeof input === 'string') - : [], - ...(typeof candidate.reason === 'string' ? {reason: candidate.reason} : {}), - ...(typeof candidate.detail === 'string' ? {detail: candidate.detail} : {}), - }; -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null; -} - -function isFiniteNumber(value: unknown): value is number { - return typeof value === 'number' && Number.isFinite(value); -} - -function isPositiveFiniteNumber(value: unknown): value is number { - return isFiniteNumber(value) && value > 0; -} - -function isOptionalPositiveFiniteNumber(value: unknown): value is number | undefined { - return value === undefined || isPositiveFiniteNumber(value); -} - -function isOptionalDeepFilterNoiseReductionLevel(value: unknown): value is number | undefined { - if (value === undefined) return true; - return isFiniteNumber(value) && value >= 0 && value <= 100; -} - -function isOptionalBoolean(value: unknown): value is boolean | undefined { - return value === undefined || typeof value === 'boolean'; -} - -function isOptionalString(value: unknown): value is string | undefined { - return value === undefined || typeof value === 'string'; -} - -function isOptionalCameraBackgroundMode(value: unknown): value is VoiceEngineV2BridgeCameraBackgroundMode | undefined { - return value === undefined || value === 'none' || value === 'non' || value === 'blur' || value === 'custom'; -} - -export function isVoiceEngineV2BridgeCameraEffectStrength(value: unknown): value is number { - return ( - typeof value === 'number' && - Number.isInteger(value) && - value >= VOICE_ENGINE_V2_CAMERA_EFFECT_STRENGTH_MIN && - value <= VOICE_ENGINE_V2_CAMERA_EFFECT_STRENGTH_MAX - ); -} - -function isOptionalCameraEffectStrength(value: unknown): value is number | undefined { - return value === undefined || isVoiceEngineV2BridgeCameraEffectStrength(value); -} - -function isOptionalCameraBackgroundCustomMediaKind( - value: unknown, -): value is VoiceEngineV2BridgeCameraBackgroundCustomMediaKind | undefined { - return value === undefined || value === 'static' || value === 'animated' || value === 'video'; -} - -function isStringArray(value: unknown): value is Array { - return Array.isArray(value) && value.every((item) => typeof item === 'string'); -} - -function isOptionalStringArray(value: unknown): value is Array | undefined { - return value === undefined || isStringArray(value); -} - -function isArrayBuffer(value: unknown): value is ArrayBuffer { - return value instanceof ArrayBuffer || Object.prototype.toString.call(value) === '[object ArrayBuffer]'; -} - -function isFloat32Array(value: unknown): value is Float32Array { - return value instanceof Float32Array || Object.prototype.toString.call(value) === '[object Float32Array]'; -} - -export function isVoiceEngineV2BridgeBinaryPayload(value: unknown): value is ArrayBuffer | ArrayBufferView { - return isArrayBuffer(value) || ArrayBuffer.isView(value); -} - -export function getVoiceEngineV2BridgeProcessedCameraI420ByteLength(width: number, height: number): number { - return (width * height * 3) / 2; -} - -function isPositiveFiniteInteger(value: unknown): value is number { - return isPositiveFiniteNumber(value) && Number.isInteger(value); -} - -function isPositiveEvenFiniteInteger(value: unknown): value is number { - return isPositiveFiniteInteger(value) && value % 2 === 0; -} - -function getVoiceEngineV2BridgeBinaryPayloadByteLength(value: ArrayBuffer | ArrayBufferView): number { - return value.byteLength; -} - -export function isVoiceEngineV2BridgeRemoteTrackSubscriptionQuality( - value: unknown, -): value is VoiceEngineV2BridgeRemoteTrackSubscriptionQuality { - return value === 'low' || value === 'medium' || value === 'high'; -} - -export function isVoiceEngineV2BridgeConnectOptions(value: unknown): value is VoiceEngineV2BridgeConnectOptions { - if (!isRecord(value)) return false; - return ( - typeof value.url === 'string' && - value.url.length > 0 && - typeof value.token === 'string' && - value.token.length > 0 && - (value.e2eeKey === undefined || value.e2eeKey === null || isArrayBuffer(value.e2eeKey)) - ); -} - -const VOICE_ENGINE_V2_SCREEN_CODECS = new Set(['', 'vp8', 'vp9', 'h264', 'h265', 'av1']); -const VOICE_ENGINE_V2_SCREEN_PACING_MODES = new Set(['sender', 'source']); - -export function isVoiceEngineV2BridgePublishScreenOptions( - value: unknown, -): value is VoiceEngineV2BridgePublishScreenOptions { - if (!isRecord(value)) return false; - return ( - typeof value.captureId === 'string' && - value.captureId.length > 0 && - isPositiveFiniteNumber(value.width) && - isPositiveFiniteNumber(value.height) && - (value.codec === undefined || - (typeof value.codec === 'string' && VOICE_ENGINE_V2_SCREEN_CODECS.has(value.codec))) && - isOptionalBoolean(value.hardwareEncoding) && - isOptionalBoolean(value.zeroCopyRequired) && - isOptionalPositiveFiniteNumber(value.maxBitrateBps) && - isOptionalPositiveFiniteNumber(value.maxFramerate) && - isOptionalBoolean(value.adaptiveSend) && - isOptionalPositiveFiniteNumber(value.minVideoFps) && - isOptionalPositiveFiniteNumber(value.maxAudioBufferMs) && - (value.trackName === undefined || (typeof value.trackName === 'string' && value.trackName.length > 0)) && - (value.pacing === undefined || - (typeof value.pacing === 'string' && VOICE_ENGINE_V2_SCREEN_PACING_MODES.has(value.pacing))) - ); -} - -export function isVoiceEngineV2BridgeUpdateScreenShareEncodingOptions( - value: unknown, -): value is VoiceEngineV2BridgeUpdateScreenShareEncodingOptions { - if (!isRecord(value)) return false; - return ( - typeof value.captureId === 'string' && - value.captureId.length > 0 && - isPositiveFiniteNumber(value.width) && - isPositiveFiniteNumber(value.height) && - isOptionalPositiveFiniteNumber(value.frameRate) && - isOptionalPositiveFiniteNumber(value.maxBitrateBps) && - (value.codec === undefined || - (typeof value.codec === 'string' && VOICE_ENGINE_V2_SCREEN_CODECS.has(value.codec))) && - isOptionalBoolean(value.hardwareEncoding) && - isOptionalBoolean(value.zeroCopyRequired) - ); -} - -export function isVoiceEngineV2BridgePublishMicrophoneOptions( - value: unknown, -): value is VoiceEngineV2BridgePublishMicrophoneOptions { - if (!isRecord(value)) return false; - const mode = value.mode ?? 'device'; - const pcmTestValuesAreValid = - mode !== 'pcm-test' || (isPositiveFiniteNumber(value.sampleRate) && isPositiveFiniteNumber(value.numChannels)); - return ( - (mode === 'device' || mode === 'pcm-test') && - isOptionalString(value.deviceId) && - isOptionalBoolean(value.echoCancellation) && - isOptionalBoolean(value.noiseSuppression) && - isOptionalBoolean(value.autoGainControl) && - isOptionalBoolean(value.deepFilter) && - isOptionalDeepFilterNoiseReductionLevel(value.deepFilterNoiseReductionLevel) && - isOptionalPositiveFiniteNumber(value.sampleRate) && - isOptionalPositiveFiniteNumber(value.numChannels) && - isOptionalPositiveFiniteNumber(value.maxBitrateBps) && - pcmTestValuesAreValid - ); -} - -export function isVoiceEngineV2BridgePcmFrame(value: unknown): value is VoiceEngineV2BridgePcmFrame { - if (!isRecord(value)) return false; - return ( - isPositiveFiniteNumber(value.sampleRate) && - isPositiveFiniteNumber(value.numChannels) && - isArrayBuffer(value.samples) - ); -} - -export function isVoiceEngineV2BridgeFloatPcmFrame(value: unknown): value is VoiceEngineV2BridgeFloatPcmFrame { - if (!isRecord(value)) return false; - return ( - isPositiveFiniteNumber(value.sampleRate) && - isPositiveFiniteNumber(value.numChannels) && - isFloat32Array(value.samples) - ); -} - -export function isVoiceEngineV2BridgePublishScreenAudioOptions( - value: unknown, -): value is VoiceEngineV2BridgePublishScreenAudioOptions { - if (!isRecord(value)) return false; - return ( - isPositiveFiniteNumber(value.sampleRate) && - isPositiveFiniteNumber(value.numChannels) && - (value.route === undefined || value.route === 'browser' || value.route === 'native') && - isOptionalString(value.captureId) && - isOptionalString(value.tapId) - ); -} - -export function isVoiceEngineV2ParticipantVolumeOptions( - value: unknown, -): value is VoiceEngineV2BridgeParticipantVolumeOptions { - if (!isRecord(value)) return false; - return typeof value.participantSid === 'string' && value.participantSid.length > 0 && isFiniteNumber(value.volume); -} - -export function clampVoiceEngineV2ParticipantVolume(volume: number): number { - if (!Number.isFinite(volume)) return 1; - return Math.max(0, Math.min(2, volume)); -} - -export function isVoiceEngineV2BridgeSpeakingDetectionOptions( - value: unknown, -): value is VoiceEngineV2BridgeSpeakingDetectionOptions { - if (!isRecord(value)) return false; - return isPositiveFiniteNumber(value.localThresholdRms) && isPositiveFiniteNumber(value.remoteThresholdRms); -} - -export function isVoiceEngineV2BridgeRemoteTrackSubscriptionOptions( - value: unknown, -): value is VoiceEngineV2BridgeRemoteTrackSubscriptionOptions { - if (!isRecord(value)) return false; - return ( - typeof value.participantIdentity === 'string' && - value.participantIdentity.length > 0 && - typeof value.source === 'string' && - value.source.length > 0 && - typeof value.subscribed === 'boolean' && - isOptionalBoolean(value.enabled) && - (value.quality === undefined || isVoiceEngineV2BridgeRemoteTrackSubscriptionQuality(value.quality)) - ); -} - -export function isVoiceEngineV2BridgePublishDataOptions( - value: unknown, -): value is VoiceEngineV2BridgePublishDataOptions { - if (!isRecord(value)) return false; - return ( - isVoiceEngineV2BridgeBinaryPayload(value.payload) && - isOptionalBoolean(value.reliable) && - isOptionalString(value.topic) && - isOptionalStringArray(value.destinationIdentities) - ); -} - -export function isVoiceEngineV2BridgePublishCameraOptions( - value: unknown, -): value is VoiceEngineV2BridgePublishCameraOptions { - if (!isRecord(value)) return false; - return ( - isOptionalString(value.deviceId) && - isOptionalPositiveFiniteNumber(value.width) && - isOptionalPositiveFiniteNumber(value.height) && - isOptionalPositiveFiniteNumber(value.frameRate) && - isOptionalBoolean(value.mirror) && - isOptionalCameraBackgroundMode(value.backgroundMode) && - isOptionalString(value.backgroundCustomMediaPath) && - isOptionalCameraBackgroundCustomMediaKind(value.backgroundCustomMediaKind) && - isOptionalCameraEffectStrength(value.backgroundBlurStrength) - ); -} - -export function isVoiceEngineV2BridgePublishProcessedCameraOptions( - value: unknown, -): value is VoiceEngineV2BridgePublishProcessedCameraOptions { - if (!isRecord(value)) return false; - return ( - isPositiveEvenFiniteInteger(value.width) && - isPositiveEvenFiniteInteger(value.height) && - isPositiveFiniteInteger(value.frameRate) - ); -} - -export function isVoiceEngineV2BridgePublishProcessedCameraResult( - value: unknown, -): value is VoiceEngineV2BridgePublishProcessedCameraResult { - if (!isRecord(value)) return false; - return typeof value.trackSid === 'string' && value.trackSid.length > 0; -} - -export function isVoiceEngineV2BridgeCameraPreviewInfo(value: unknown): value is VoiceEngineV2BridgeCameraPreviewInfo { - if (!isRecord(value)) return false; - return ( - typeof value.trackSid === 'string' && - value.trackSid.length > 0 && - isPositiveEvenFiniteInteger(value.width) && - isPositiveEvenFiniteInteger(value.height) && - isPositiveFiniteInteger(value.frameRate) - ); -} - -export function isVoiceEngineV2BridgeProcessedCameraFrame( - value: unknown, -): value is VoiceEngineV2BridgeProcessedCameraFrame { - if (!isRecord(value)) return false; - if (value.format !== 'i420') return false; - if (!isPositiveEvenFiniteInteger(value.width)) return false; - if (!isPositiveEvenFiniteInteger(value.height)) return false; - if (!isPositiveFiniteInteger(value.timestampUs)) return false; - if (!isVoiceEngineV2BridgeBinaryPayload(value.data)) return false; - const expectedBytes = getVoiceEngineV2BridgeProcessedCameraI420ByteLength(value.width, value.height); - return getVoiceEngineV2BridgeBinaryPayloadByteLength(value.data) === expectedBytes; -} - -export function isVoiceEngineV2BridgeReadiness(value: unknown): value is VoiceEngineV2BridgeReadiness { - if (!isRecord(value)) return false; - if (typeof value.ready !== 'boolean') return false; - if (value.ready) return value.reason === undefined; - return isOptionalString(value.reason); -} - -export function isVoiceEngineV2BridgePublishDeviceScreenShareOptions( - value: unknown, -): value is VoiceEngineV2BridgePublishDeviceScreenShareOptions { - if (!isRecord(value)) return false; - return ( - isOptionalString(value.deviceId) && - isOptionalPositiveFiniteNumber(value.width) && - isOptionalPositiveFiniteNumber(value.height) && - isOptionalPositiveFiniteNumber(value.frameRate) && - (value.codec === undefined || - (typeof value.codec === 'string' && VOICE_ENGINE_V2_SCREEN_CODECS.has(value.codec))) && - isOptionalPositiveFiniteNumber(value.maxBitrateBps) && - isOptionalPositiveFiniteNumber(value.maxFramerate) - ); -} diff --git a/packages/voice_engine_v2/src/bridge/index.ts b/packages/voice_engine_v2/src/bridge/index.ts index b41398885..ba612ca6c 100644 --- a/packages/voice_engine_v2/src/bridge/index.ts +++ b/packages/voice_engine_v2/src/bridge/index.ts @@ -1,7 +1,76 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -export * from './admWarmup'; -export * from './guards'; -export * from './stats'; -export * from './translate'; -export * from './types'; +import type {VoiceEngineV2HardwareEncoderCapabilities} from '../protocol'; +import {assertAudioFrameInvariants, assertVideoFrameInvariants} from './ffi_assertions'; + +export const VOICE_ENGINE_V2_HARDWARE_ENCODER_IPC_CHANNEL = 'voice-engine-v2:get-hardware-encoder-capabilities'; + +export type VoiceEngineV2BridgeHardwareEncoderCapabilities = VoiceEngineV2HardwareEncoderCapabilities; + +export interface VoiceEngineV2BridgeHardwareEncoderApi { + getHardwareEncoderCapabilities(): Promise; +} + +export function assertVoiceEngineV2BridgeAudioOptionsInvariants(options: { + sampleRate: number; + numChannels: number; +}): void { + assertAudioFrameInvariants({ + sampleRateHz: options.sampleRate, + numChannels: options.numChannels, + frameBytes: 1, + timestampNs: 1, + }); +} + +export function assertVoiceEngineV2BridgeVideoOptionsInvariants(options: {width: number; height: number}): void { + assertVideoFrameInvariants({ + widthPx: options.width, + heightPx: options.height, + frameBytes: 1, + timestampNs: 1, + }); +} + +export function unavailableVoiceEngineV2BridgeHardwareEncoderCapabilities( + reason: NonNullable, + detail?: string, +): VoiceEngineV2BridgeHardwareEncoderCapabilities { + return { + available: false, + backend: 'none', + compiled: false, + runtime: false, + codecs: [], + zeroCopy: false, + nativeInputs: [], + reason, + ...(detail ? {detail} : {}), + }; +} + +export function normalizeVoiceEngineV2BridgeHardwareEncoderCapabilities( + value: unknown, +): VoiceEngineV2BridgeHardwareEncoderCapabilities { + if (typeof value !== 'object' || value === null) { + return unavailableVoiceEngineV2BridgeHardwareEncoderCapabilities( + 'query-failed', + 'Native addon returned an invalid result', + ); + } + const candidate = value as Partial; + const backend = typeof candidate.backend === 'string' && candidate.backend.length > 0 ? candidate.backend : 'none'; + return { + available: candidate.available === true, + backend, + compiled: candidate.compiled === true, + runtime: candidate.runtime === true, + codecs: Array.isArray(candidate.codecs) ? candidate.codecs.filter((codec) => typeof codec === 'string') : [], + zeroCopy: candidate.zeroCopy === true, + nativeInputs: Array.isArray(candidate.nativeInputs) + ? candidate.nativeInputs.filter((input) => typeof input === 'string') + : [], + ...(typeof candidate.reason === 'string' ? {reason: candidate.reason} : {}), + ...(typeof candidate.detail === 'string' ? {detail: candidate.detail} : {}), + }; +} diff --git a/packages/voice_engine_v2/src/bridge/stats.ts b/packages/voice_engine_v2/src/bridge/stats.ts deleted file mode 100644 index 556f73d27..000000000 --- a/packages/voice_engine_v2/src/bridge/stats.ts +++ /dev/null @@ -1,141 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import type { - VoiceEngineV2BridgeInboundStat, - VoiceEngineV2BridgeOutboundStat, - VoiceEngineV2BridgeSendStats, - VoiceEngineV2BridgeStats, - VoiceEngineV2BridgeTrackKind, -} from './types'; - -function asTrackKind(value: unknown): VoiceEngineV2BridgeTrackKind { - return value === 'video' ? 'video' : 'audio'; -} - -function asNumber(value: unknown): number { - return typeof value === 'number' && Number.isFinite(value) ? value : 0; -} - -function asNonNegativeNumber(value: unknown): number { - return Math.max(0, asNumber(value)); -} - -function asOptionalNumber(value: unknown): number | undefined { - return typeof value === 'number' && Number.isFinite(value) ? value : undefined; -} - -function asOptionalCount(value: unknown): number | undefined { - const number = asOptionalNumber(value); - return number == null ? undefined : Math.max(0, Math.trunc(number)); -} - -function asString(value: unknown): string | undefined { - return typeof value === 'string' && value.length > 0 ? value : undefined; -} - -function coerceVoiceEngineV2BridgeSendStats(value: unknown): VoiceEngineV2BridgeSendStats | null | undefined { - if (value == null || typeof value !== 'object') return value === null ? null : undefined; - const payload = value as Record; - return { - outgoingVideoQueueDepth: asNonNegativeNumber(payload.outgoingVideoQueueDepth), - outgoingVideoQueueCapacity: asNonNegativeNumber(payload.outgoingVideoQueueCapacity), - outgoingVideoMaxQueueDepth: asNonNegativeNumber(payload.outgoingVideoMaxQueueDepth), - outgoingVideoFramesProduced: asNonNegativeNumber(payload.outgoingVideoFramesProduced), - outgoingVideoFramesAccepted: asNonNegativeNumber(payload.outgoingVideoFramesAccepted), - outgoingVideoFramesDropped: asNonNegativeNumber(payload.outgoingVideoFramesDropped), - outgoingVideoFramesCoalesced: asNonNegativeNumber(payload.outgoingVideoFramesCoalesced), - outgoingVideoFramesCaptured: asNonNegativeNumber(payload.outgoingVideoFramesCaptured), - outgoingVideoCaptureFailures: asNonNegativeNumber(payload.outgoingVideoCaptureFailures), - outgoingVideoEffectiveFps: asNonNegativeNumber(payload.outgoingVideoEffectiveFps), - outgoingVideoTargetFps: asNonNegativeNumber(payload.outgoingVideoTargetFps), - outgoingVideoPacingTargetFps: asNonNegativeNumber(payload.outgoingVideoPacingTargetFps), - outgoingVideoMaxQueueAgeMs: asNonNegativeNumber(payload.outgoingVideoMaxQueueAgeMs), - outgoingVideoMaxPushLatencyMs: asNonNegativeNumber(payload.outgoingVideoMaxPushLatencyMs), - outgoingVideoPacingMode: asString(payload.outgoingVideoPacingMode) ?? 'unknown', - outgoingVideoBusActive: payload.outgoingVideoBusActive === true, - outgoingAudioBufferTargetMs: asNonNegativeNumber(payload.outgoingAudioBufferTargetMs), - outgoingAudioBufferMaxMs: asNonNegativeNumber(payload.outgoingAudioBufferMaxMs), - outgoingAudioUnderruns: asNonNegativeNumber(payload.outgoingAudioUnderruns), - outgoingAudioRebuffers: asNonNegativeNumber(payload.outgoingAudioRebuffers), - outgoingAudioMaxFrameGapMs: asNonNegativeNumber(payload.outgoingAudioMaxFrameGapMs), - adaptiveSendTier: asString(payload.adaptiveSendTier) ?? 'unknown', - adaptiveSendReason: asString(payload.adaptiveSendReason) ?? 'unknown', - }; -} - -function coerceVoiceEngineV2BridgeOutboundStat(entry: unknown): Array { - if (entry == null || typeof entry !== 'object') return []; - const e = entry as Record; - const zeroCopy = e.zeroCopy === true ? true : undefined; - return [ - { - trackSid: asString(e.trackSid) ?? '', - source: asString(e.source) ?? '', - kind: asTrackKind(e.kind), - codec: asString(e.codec), - bitrateKbps: asNumber(e.bitrateKbps), - packetsLost: asNumber(e.packetsLost), - fps: asOptionalNumber(e.fps), - audioLevel: asOptionalNumber(e.audioLevel), - width: asOptionalCount(e.width), - height: asOptionalCount(e.height), - sourceWidth: asOptionalCount(e.sourceWidth), - sourceHeight: asOptionalCount(e.sourceHeight), - targetBitrateKbps: asOptionalNumber(e.targetBitrateKbps), - configuredFps: asOptionalNumber(e.configuredFps), - targetFps: asOptionalNumber(e.targetFps), - effectiveFps: asOptionalNumber(e.effectiveFps), - framesProduced: asOptionalCount(e.framesProduced), - framesAccepted: asOptionalCount(e.framesAccepted), - framesDropped: asOptionalCount(e.framesDropped), - framesCoalesced: asOptionalCount(e.framesCoalesced), - framesCaptured: asOptionalCount(e.framesCaptured), - captureFailures: asOptionalCount(e.captureFailures), - maxQueueAgeMs: asOptionalNumber(e.maxQueueAgeMs), - maxPushLatencyMs: asOptionalNumber(e.maxPushLatencyMs), - adaptiveSendTier: asString(e.adaptiveSendTier), - adaptiveSendReason: asString(e.adaptiveSendReason), - ...(zeroCopy === true ? {zeroCopy} : {}), - }, - ]; -} - -function coerceVoiceEngineV2BridgeInboundStat(entry: unknown): Array { - if (entry == null || typeof entry !== 'object') return []; - const e = entry as Record; - const participantIdentity = asString(e.participantIdentity); - return [ - { - ...(participantIdentity ? {participantIdentity} : {}), - participantSid: asString(e.participantSid) ?? '', - trackSid: asString(e.trackSid) ?? '', - source: asString(e.source), - kind: asTrackKind(e.kind), - codec: asString(e.codec), - bitrateKbps: asNumber(e.bitrateKbps), - packetsLost: asNumber(e.packetsLost), - jitterMs: asOptionalNumber(e.jitterMs), - audioLevel: asOptionalNumber(e.audioLevel), - width: asOptionalCount(e.width), - height: asOptionalCount(e.height), - fps: asOptionalNumber(e.fps), - sourceWidth: asOptionalCount(e.sourceWidth), - sourceHeight: asOptionalCount(e.sourceHeight), - }, - ]; -} - -export function coerceVoiceEngineV2BridgeStats(payload: Record): VoiceEngineV2BridgeStats { - const rttRaw = payload.rttMs; - const rttMs = typeof rttRaw === 'number' && Number.isFinite(rttRaw) ? rttRaw : null; - const droppedNativeVideoFrames = asOptionalCount(payload.droppedNativeVideoFrames); - const droppedVideoFrameCallbacks = asOptionalCount(payload.droppedVideoFrameCallbacks); - const outbound = Array.isArray(payload.outbound) - ? payload.outbound.flatMap((entry) => coerceVoiceEngineV2BridgeOutboundStat(entry)) - : []; - const inbound = Array.isArray(payload.inbound) - ? payload.inbound.flatMap((entry) => coerceVoiceEngineV2BridgeInboundStat(entry)) - : []; - const send = coerceVoiceEngineV2BridgeSendStats(payload.send); - return {rttMs, outbound, inbound, droppedNativeVideoFrames, droppedVideoFrameCallbacks, send}; -} diff --git a/packages/voice_engine_v2/src/bridge/translate.ts b/packages/voice_engine_v2/src/bridge/translate.ts deleted file mode 100644 index 4f8a9d1d4..000000000 --- a/packages/voice_engine_v2/src/bridge/translate.ts +++ /dev/null @@ -1,310 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import type { - VoiceEngineV2Event, - VoiceEngineV2NativeAudioDeviceModuleStatus, - VoiceEngineV2TrackKind, - VoiceEngineV2TrackSource, -} from '../protocol'; -import {assertVideoFrameInvariants} from './ffi_assertions'; -import type {VoiceEngineV2BridgeEvent, VoiceEngineV2BridgeKnownEventType, VoiceEngineV2BridgeVideoFrame} from './types'; - -type VoiceEngineV2RoomTrackPublishedEvent = Extract; - -type VoiceEngineV2BridgeRecord = Record; - -function bridgeString(value: unknown): string | null { - return typeof value === 'string' && value.length > 0 ? value : null; -} - -function bridgeBoolean(value: unknown): boolean { - return value === true; -} - -function bridgeAudioDeviceModuleStatus(value: unknown): VoiceEngineV2NativeAudioDeviceModuleStatus | null { - if (value === 'warming' || value === 'ready' || value === 'failed') return value; - return null; -} - -function normalizeVoiceEngineV2BridgeTrackSource(value: unknown): VoiceEngineV2TrackSource | string { - switch (value) { - case 'microphone': - return 'microphone'; - case 'camera': - return 'camera'; - case 'screen': - case 'screen_share': - case 'screenshare': - return 'screen'; - case 'screenAudio': - case 'screen_share_audio': - case 'screenshareAudio': - case 'screenshare_audio': - case 'screen_audio': - case 'system_audio': - return 'screenAudio'; - default: - return 'unknown'; - } -} - -function normalizeVoiceEngineV2BridgeTrackKind(value: unknown): VoiceEngineV2TrackKind | null { - return value === 'audio' || value === 'video' ? value : null; -} - -function participantEventFromBridgePayload(payload: VoiceEngineV2BridgeRecord): VoiceEngineV2Event | null { - const identity = bridgeString(payload.identity); - const sid = bridgeString(payload.sid); - if (!identity || !sid) return null; - return { - type: 'room.participantJoined', - participant: { - identity, - sid, - name: bridgeString(payload.name) ?? identity, - }, - }; -} - -function trackEventFromBridgePayload(payload: VoiceEngineV2BridgeRecord): VoiceEngineV2RoomTrackPublishedEvent | null { - const participantIdentity = bridgeString(payload.identity) ?? bridgeString(payload.participantIdentity); - const participantSid = bridgeString(payload.participantSid); - const trackSid = bridgeString(payload.trackSid); - const kind = normalizeVoiceEngineV2BridgeTrackKind(payload.kind); - if (!participantIdentity || !participantSid || !trackSid || !kind) return null; - const source = normalizeVoiceEngineV2BridgeTrackSource(payload.source); - return { - type: 'room.trackPublished', - track: { - participantIdentity, - participantSid, - trackSid, - trackName: bridgeString(payload.trackName) ?? String(source), - kind, - source, - muted: bridgeBoolean(payload.muted), - }, - }; -} - -function nativeAudioDeviceModuleStatusEventFromBridgePayload( - payload: VoiceEngineV2BridgeRecord, -): VoiceEngineV2Event | null { - const status = bridgeAudioDeviceModuleStatus(payload.status); - if (!status) return null; - const detail = bridgeString(payload.detail); - return { - type: 'nativeAudioDeviceModule.statusChanged', - status, - ...(detail ? {detail} : {}), - }; -} - -const VOICE_ENGINE_V2_BRIDGE_KNOWN_EVENT_TYPE_FLAGS: {[Type in VoiceEngineV2BridgeKnownEventType]: true} = { - connected: true, - connectionState: true, - disconnected: true, - participantJoined: true, - participantLeft: true, - participantNameChanged: true, - participantMetadataChanged: true, - participantAttributesChanged: true, - trackPublished: true, - trackUnpublished: true, - trackSubscribed: true, - trackUnsubscribed: true, - trackSubscriptionFailed: true, - trackMuted: true, - trackUnmuted: true, - localTrackPublished: true, - localTrackUnpublished: true, - localTrackRepublished: true, - activeSpeakers: true, - speakingChanged: true, - connectionQuality: true, - dataReceived: true, - e2eeState: true, - stats: true, - audioPlaybackUnavailable: true, - engineReady: true, - audioDeviceModuleStatus: true, -}; - -const VOICE_ENGINE_V2_BRIDGE_KNOWN_EVENT_TYPE_SET: ReadonlySet = new Set( - Object.keys(VOICE_ENGINE_V2_BRIDGE_KNOWN_EVENT_TYPE_FLAGS), -); - -export function isVoiceEngineV2BridgeKnownEventType(value: string): value is VoiceEngineV2BridgeKnownEventType { - return VOICE_ENGINE_V2_BRIDGE_KNOWN_EVENT_TYPE_SET.has(value); -} - -export const VOICE_ENGINE_V2_BRIDGE_DROPPED_EVENTS_COUNT_MAX = 1_000_000_000; - -export interface VoiceEngineV2BridgeDroppedEventCounts { - malformedPayloadCount: number; - unknownTypeCount: number; -} - -let bridgeMalformedPayloadCount = 0; -let bridgeUnknownTypeCount = 0; - -function saturatingDroppedEventCount(count: number): number { - return Math.min(count + 1, VOICE_ENGINE_V2_BRIDGE_DROPPED_EVENTS_COUNT_MAX); -} - -export function getVoiceEngineV2BridgeDroppedEventCounts(): VoiceEngineV2BridgeDroppedEventCounts { - return { - malformedPayloadCount: bridgeMalformedPayloadCount, - unknownTypeCount: bridgeUnknownTypeCount, - }; -} - -export function resetVoiceEngineV2BridgeDroppedEventCounts(): void { - bridgeMalformedPayloadCount = 0; - bridgeUnknownTypeCount = 0; -} - -type VoiceEngineV2BridgeTranslationResult = Array | 'ignored' | 'malformed'; - -export function translateVoiceEngineV2BridgeEventToEvents(event: VoiceEngineV2BridgeEvent): Array { - if (!isVoiceEngineV2BridgeKnownEventType(event.type)) { - bridgeUnknownTypeCount = saturatingDroppedEventCount(bridgeUnknownTypeCount); - return []; - } - const payload = event.payload as VoiceEngineV2BridgeRecord; - const translated = translateKnownVoiceEngineV2BridgeEvent(event.type, payload); - if (translated === 'ignored') return []; - if (translated === 'malformed') { - bridgeMalformedPayloadCount = saturatingDroppedEventCount(bridgeMalformedPayloadCount); - return []; - } - return translated; -} - -function translateKnownVoiceEngineV2BridgeEvent( - type: VoiceEngineV2BridgeKnownEventType, - payload: VoiceEngineV2BridgeRecord, -): VoiceEngineV2BridgeTranslationResult { - switch (type) { - case 'participantJoined': - case 'participantNameChanged': - case 'participantMetadataChanged': - case 'participantAttributesChanged': { - const participantEvent = participantEventFromBridgePayload(payload); - return participantEvent ? [participantEvent] : 'malformed'; - } - case 'participantLeft': - return participantLeftEventsFromBridgePayload(payload); - case 'trackPublished': - case 'localTrackPublished': { - const trackEvent = trackEventFromBridgePayload(payload); - return trackEvent ? [trackEvent] : 'malformed'; - } - case 'trackSubscribed': - return trackSubscribedEventsFromBridgePayload(payload); - case 'trackUnpublished': - case 'localTrackUnpublished': { - const trackSid = bridgeString(payload.trackSid); - return trackSid - ? [ - {type: 'room.trackUnpublished', trackSid}, - {type: 'inboundVideo.trackUnsubscribed', trackSid}, - ] - : 'malformed'; - } - case 'trackUnsubscribed': - case 'trackSubscriptionFailed': { - const trackSid = bridgeString(payload.trackSid); - return trackSid ? [{type: 'inboundVideo.trackUnsubscribed', trackSid}] : 'malformed'; - } - case 'trackMuted': { - const trackSid = bridgeString(payload.trackSid); - return trackSid ? [{type: 'room.trackMuted', trackSid}] : 'malformed'; - } - case 'trackUnmuted': { - const trackSid = bridgeString(payload.trackSid); - return trackSid ? [{type: 'room.trackUnmuted', trackSid}] : 'malformed'; - } - case 'activeSpeakers': - return Array.isArray(payload.participants) - ? payload.participants.flatMap((participant) => - typeof participant === 'object' && participant !== null - ? (participantEventFromBridgePayload(participant as VoiceEngineV2BridgeRecord) ?? []) - : [], - ) - : 'malformed'; - case 'connected': - case 'connectionState': - case 'disconnected': - case 'localTrackRepublished': - case 'speakingChanged': - case 'connectionQuality': - case 'dataReceived': - case 'e2eeState': - case 'stats': - case 'audioPlaybackUnavailable': - case 'engineReady': - return 'ignored'; - case 'audioDeviceModuleStatus': { - const event = nativeAudioDeviceModuleStatusEventFromBridgePayload(payload); - return event ? [event] : 'malformed'; - } - } -} - -function participantLeftEventsFromBridgePayload( - payload: VoiceEngineV2BridgeRecord, -): VoiceEngineV2BridgeTranslationResult { - const participantIdentity = bridgeString(payload.identity); - const participantSid = bridgeString(payload.sid); - if (!participantIdentity && !participantSid) return 'malformed'; - return [ - { - type: 'room.participantLeft', - ...(participantIdentity ? {participantIdentity} : {}), - ...(participantSid ? {participantSid} : {}), - }, - ]; -} - -function trackSubscribedEventsFromBridgePayload( - payload: VoiceEngineV2BridgeRecord, -): VoiceEngineV2BridgeTranslationResult { - const trackEvent = trackEventFromBridgePayload(payload); - if (!trackEvent) return 'malformed'; - if (trackEvent.track.kind !== 'video') return [trackEvent]; - return [ - trackEvent, - { - type: 'inboundVideo.trackSubscribed', - track: { - participantSid: trackEvent.track.participantSid, - participantIdentity: trackEvent.track.participantIdentity, - trackSid: trackEvent.track.trackSid, - source: trackEvent.track.source, - }, - }, - ]; -} - -export function translateVoiceEngineV2BridgeVideoFrameToEvent( - frame: VoiceEngineV2BridgeVideoFrame, -): VoiceEngineV2Event { - assertVideoFrameInvariants({ - widthPx: frame.meta.width, - heightPx: frame.meta.height, - frameBytes: frame.data.byteLength, - timestampNs: frame.meta.timestampUs * 1000, - }); - return { - type: 'inboundVideo.frameReceived', - frame: { - participantSid: frame.meta.participantSid, - trackSid: frame.meta.trackSid, - width: frame.meta.width, - height: frame.meta.height, - timestampUs: frame.meta.timestampUs, - byteLength: frame.data.byteLength, - }, - }; -} diff --git a/packages/voice_engine_v2/src/bridge/types.ts b/packages/voice_engine_v2/src/bridge/types.ts deleted file mode 100644 index ec813b3f6..000000000 --- a/packages/voice_engine_v2/src/bridge/types.ts +++ /dev/null @@ -1,680 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later -// biome-ignore-all lint/suspicious/noConfusingVoidType: IPC result metadata mirrors void-returning bridge methods. - -import type { - VoiceEngineV2CameraOptions, - VoiceEngineV2ConnectOptions, - VoiceEngineV2DataOptions, - VoiceEngineV2HardwareEncoderCapabilities, - VoiceEngineV2PcmFrame, - VoiceEngineV2RemoteTrackQuality, - VoiceEngineV2RemoteTrackSubscriptionOptions, - VoiceEngineV2ScreenAudioOptions, - VoiceEngineV2ScreenOptions, - VoiceEngineV2SendStats, - VoiceEngineV2TrackKind, - VoiceEngineV2TrackSource, - VoiceEngineV2VideoCodec, -} from '../protocol'; - -export const VOICE_ENGINE_V2_BRIDGE_VERSION = 18; - -export type VoiceEngineV2BridgeMethodName = - | 'isSupported' - | 'getCapabilities' - | 'prewarm' - | 'getHardwareEncoderCapabilities' - | 'connect' - | 'disconnect' - | 'isConnected' - | 'publishMicrophone' - | 'pushPcm' - | 'publishScreen' - | 'updateScreenShareEncoding' - | 'unpublishScreen' - | 'publishScreenAudio' - | 'pushScreenAudioPcm' - | 'pushScreenAudioFloat' - | 'unpublishScreenAudio' - | 'setMicEnabled' - | 'setSpeakingDetection' - | 'listAudioInputDevices' - | 'listAudioOutputDevices' - | 'setAudioOutputDevice' - | 'setParticipantVolume' - | 'setRemoteTrackSubscription' - | 'publishData' - | 'listCameraDevices' - | 'publishCamera' - | 'updateCameraCapture' - | 'publishNativeCameraSink' - | 'publishProcessedCamera' - | 'pushProcessedCameraFrame' - | 'pushCameraBackgroundFrame' - | 'clearCameraBackgroundFrame' - | 'publishDeviceScreenShare' - | 'unpublishCamera' - | 'isPublishingCamera' - | 'startCameraPreview' - | 'stopCameraPreview' - | 'getConnectionStats' - | 'getVoiceEngineReadiness' - | 'getAudioDeviceModuleState' - | 'onEvent' - | 'onVideoFrame'; - -export type VoiceEngineV2InvokeMethodName = Exclude; - -export type VoiceEngineV2BridgeConnectOptions = VoiceEngineV2ConnectOptions; - -export type VoiceEngineV2BridgeScreenPacing = 'sender' | 'source'; - -export interface VoiceEngineV2BridgePublishScreenOptions extends VoiceEngineV2ScreenOptions { - codec?: VoiceEngineV2VideoCodec; - pacing?: VoiceEngineV2BridgeScreenPacing; - trackName?: string; -} - -export interface VoiceEngineV2BridgeUpdateScreenShareEncodingOptions { - captureId: string; - width: number; - height: number; - frameRate?: number; - maxBitrateBps?: number; - codec?: VoiceEngineV2VideoCodec; - hardwareEncoding?: boolean; - zeroCopyRequired?: boolean; -} - -export interface VoiceEngineV2BridgePublishMicrophoneOptions { - mode?: 'device' | 'pcm-test'; - deviceId?: string; - echoCancellation?: boolean; - noiseSuppression?: boolean; - autoGainControl?: boolean; - deepFilter?: boolean; - deepFilterNoiseReductionLevel?: number; - sampleRate?: number; - numChannels?: number; - maxBitrateBps?: number; -} - -export type VoiceEngineV2BridgePcmFrame = VoiceEngineV2PcmFrame; -export interface VoiceEngineV2BridgeFloatPcmFrame { - sampleRate: number; - numChannels: number; - samples: Float32Array; -} -export type VoiceEngineV2BridgePublishScreenAudioOptions = VoiceEngineV2ScreenAudioOptions; -export type VoiceEngineV2BridgePublishDataOptions = VoiceEngineV2DataOptions; - -export type VoiceEngineV2BridgeTrackKind = VoiceEngineV2TrackKind; -export type VoiceEngineV2BridgeTrackSource = - | VoiceEngineV2TrackSource - | 'screen_share' - | 'screen_share_audio' - | 'screenshare' - | 'screenshareAudio'; -export type VoiceEngineV2BridgeSubscriptionStatus = 'desired' | 'subscribed' | 'unsubscribed'; -export type VoiceEngineV2BridgeConnectionQuality = 'excellent' | 'good' | 'poor' | 'lost'; - -export interface VoiceEngineV2BridgeParticipantEventPayload { - sid: string; - identity: string; - name: string; -} - -export interface VoiceEngineV2BridgeTrackEventPayload { - participantSid: string; - identity: string; - participantName: string; - trackSid: string; - trackName: string; - kind: VoiceEngineV2BridgeTrackKind; - source: VoiceEngineV2BridgeTrackSource; - muted: boolean; -} - -export interface VoiceEngineV2BridgeSubscribedTrackEventPayload extends VoiceEngineV2BridgeTrackEventPayload { - subscribed: boolean; - subscriptionStatus: VoiceEngineV2BridgeSubscriptionStatus; -} - -export interface VoiceEngineV2BridgeTrackSubscriptionFailedEventPayload { - participantSid: string; - identity: string; - participantName: string; - trackSid: string; - error: string; - trackName?: string; - kind?: VoiceEngineV2BridgeTrackKind; - source?: VoiceEngineV2BridgeTrackSource; - muted?: boolean; - subscribed?: boolean; - subscriptionStatus?: VoiceEngineV2BridgeSubscriptionStatus; -} - -export interface VoiceEngineV2BridgeLocalTrackRepublishedEventPayload extends VoiceEngineV2BridgeTrackEventPayload { - previousTrackSid: string; -} - -export interface VoiceEngineV2BridgeEventPayloads { - connected: Record; - connectionState: {state: string}; - disconnected: {reason: string}; - participantJoined: VoiceEngineV2BridgeParticipantEventPayload; - participantLeft: VoiceEngineV2BridgeParticipantEventPayload; - participantNameChanged: {sid: string; identity: string; oldName: string; name: string}; - participantMetadataChanged: { - sid: string; - identity: string; - name: string; - oldMetadata: string; - metadata: string; - attributes: Record; - }; - participantAttributesChanged: { - sid: string; - identity: string; - name: string; - attributes: Record; - changedAttributes: Record; - }; - trackPublished: VoiceEngineV2BridgeSubscribedTrackEventPayload; - trackUnpublished: VoiceEngineV2BridgeSubscribedTrackEventPayload; - trackSubscribed: VoiceEngineV2BridgeSubscribedTrackEventPayload; - trackUnsubscribed: VoiceEngineV2BridgeSubscribedTrackEventPayload; - trackSubscriptionFailed: VoiceEngineV2BridgeTrackSubscriptionFailedEventPayload; - trackMuted: VoiceEngineV2BridgeTrackEventPayload; - trackUnmuted: VoiceEngineV2BridgeTrackEventPayload; - localTrackPublished: VoiceEngineV2BridgeTrackEventPayload; - localTrackUnpublished: VoiceEngineV2BridgeTrackEventPayload; - localTrackRepublished: VoiceEngineV2BridgeLocalTrackRepublishedEventPayload; - activeSpeakers: { - sids: Array; - participants: Array; - }; - speakingChanged: { - participantSid: string; - identity: string; - trackSid: string; - source: VoiceEngineV2BridgeTrackSource; - isLocal: boolean; - speaking: boolean; - }; - connectionQuality: { - sid: string; - identity: string; - name: string; - quality: VoiceEngineV2BridgeConnectionQuality; - }; - dataReceived: { - payloadBytes: Array; - payloadText?: string; - topic?: string; - reliable: boolean; - kind: 'reliable' | 'lossy'; - participantSid?: string; - identity?: string; - participantName?: string; - }; - e2eeState: {sid: string; identity: string; name: string; state: string}; - stats: VoiceEngineV2BridgeStats; - audioPlaybackUnavailable: {message: string}; - engineReady: {ready: true}; - audioDeviceModuleStatus: VoiceEngineV2BridgeAudioDeviceModuleState; -} - -export const VOICE_ENGINE_V2_ENGINE_READY_EVENT_TYPE = 'engineReady'; -export const VOICE_ENGINE_V2_ADM_STATUS_EVENT_TYPE = 'audioDeviceModuleStatus'; - -export interface VoiceEngineV2BridgeReadiness { - ready: boolean; - reason?: string; -} - -export type VoiceEngineV2BridgeAudioDeviceModuleStatus = 'warming' | 'ready' | 'failed'; - -export interface VoiceEngineV2BridgeAudioDeviceModuleState { - status: VoiceEngineV2BridgeAudioDeviceModuleStatus; - detail?: string; -} - -export type VoiceEngineV2BridgeKnownEventType = keyof VoiceEngineV2BridgeEventPayloads; -export type VoiceEngineV2BridgeEventType = VoiceEngineV2BridgeKnownEventType | (string & {}); -export type VoiceEngineV2BridgeKnownEvent = { - [Type in VoiceEngineV2BridgeKnownEventType]: {type: Type; payload: VoiceEngineV2BridgeEventPayloads[Type]}; -}[VoiceEngineV2BridgeKnownEventType]; -export type VoiceEngineV2BridgeEvent = - | VoiceEngineV2BridgeKnownEvent - | {type: string & {}; payload: Record}; - -export interface VoiceEngineV2BridgeVideoFrameMeta { - bridgeVersion?: number; - participantSid: string; - participantIdentity?: string; - trackSid: string; - trackName?: string; - source?: VoiceEngineV2BridgeTrackSource; - width: number; - height: number; - timestampUs: number; -} - -export interface VoiceEngineV2BridgeVideoFrame { - meta: VoiceEngineV2BridgeVideoFrameMeta; - data: ArrayBuffer; -} - -export type VoiceEngineV2BridgeAudioDeviceRole = 'default' | 'communications' | 'endpoint'; - -export interface VoiceEngineV2BridgeAudioOutputDevice { - deviceId: string; - label: string; - isDefault: boolean; - role?: VoiceEngineV2BridgeAudioDeviceRole; - endpointLabel?: string; - isDefaultRoute?: boolean; -} - -export interface VoiceEngineV2BridgeAudioInputDevice { - deviceId: string; - label: string; - isDefault: boolean; - role?: VoiceEngineV2BridgeAudioDeviceRole; - endpointLabel?: string; - isDefaultRoute?: boolean; -} - -export interface VoiceEngineV2BridgeCameraDevice { - deviceId: string; - label: string; - description: string; - index?: number | null; - deviceIdAliases?: Array; -} - -export const VOICE_ENGINE_V2_CAMERA_EFFECT_STRENGTH_MIN = 0; -export const VOICE_ENGINE_V2_CAMERA_EFFECT_STRENGTH_MAX = 100; -export const VOICE_ENGINE_V2_CAMERA_EFFECT_STRENGTH_DEFAULT = 50; - -export interface VoiceEngineV2BridgePublishCameraOptions extends VoiceEngineV2CameraOptions { - mirror?: boolean; - backgroundBlurStrength?: number; -} - -export type VoiceEngineV2BridgeUpdateCameraCaptureOptions = VoiceEngineV2BridgePublishCameraOptions; -export type VoiceEngineV2BridgeCameraBackgroundMode = 'none' | 'non' | 'blur' | 'custom'; -export type VoiceEngineV2BridgeCameraBackgroundCustomMediaKind = 'static' | 'animated' | 'video'; - -export interface VoiceEngineV2BridgePublishProcessedCameraOptions { - width: number; - height: number; - frameRate: number; -} - -export interface VoiceEngineV2BridgePublishProcessedCameraResult { - trackSid: string; -} - -export type VoiceEngineV2BridgePublishNativeCameraSinkResult = VoiceEngineV2BridgePublishProcessedCameraResult; - -export type VoiceEngineV2BridgeStartCameraPreviewOptions = VoiceEngineV2BridgePublishCameraOptions; - -export interface VoiceEngineV2BridgeCameraPreviewInfo { - trackSid: string; - width: number; - height: number; - frameRate: number; -} - -export type VoiceEngineV2BridgeProcessedCameraFrameFormat = 'i420'; - -export interface VoiceEngineV2BridgeProcessedCameraFrame { - format: VoiceEngineV2BridgeProcessedCameraFrameFormat; - width: number; - height: number; - timestampUs: number; - data: ArrayBuffer | ArrayBufferView; -} - -export interface VoiceEngineV2BridgePublishDeviceScreenShareOptions { - deviceId?: string; - width?: number; - height?: number; - frameRate?: number; - codec?: VoiceEngineV2VideoCodec; - maxBitrateBps?: number; - maxFramerate?: number; -} - -export interface VoiceEngineV2BridgeOutboundStat { - trackSid: string; - source: string; - kind: VoiceEngineV2BridgeTrackKind; - codec?: string; - bitrateKbps: number; - packetsLost: number; - fps?: number; - audioLevel?: number; - width?: number; - height?: number; - sourceWidth?: number; - sourceHeight?: number; - targetBitrateKbps?: number; - configuredFps?: number; - targetFps?: number; - effectiveFps?: number; - framesProduced?: number; - framesAccepted?: number; - framesDropped?: number; - framesCoalesced?: number; - framesCaptured?: number; - captureFailures?: number; - maxQueueAgeMs?: number; - maxPushLatencyMs?: number; - adaptiveSendTier?: string; - adaptiveSendReason?: string; - zeroCopy?: boolean; -} - -export interface VoiceEngineV2BridgeInboundStat { - participantSid: string; - participantIdentity?: string; - trackSid: string; - source?: string; - kind: VoiceEngineV2BridgeTrackKind; - codec?: string; - bitrateKbps: number; - packetsLost: number; - jitterMs?: number; - audioLevel?: number; - width?: number; - height?: number; - fps?: number; - sourceWidth?: number; - sourceHeight?: number; -} - -export type VoiceEngineV2BridgeSendStats = VoiceEngineV2SendStats; - -export interface VoiceEngineV2BridgeStats { - rttMs: number | null; - outbound: Array; - inbound: Array; - droppedNativeVideoFrames?: number; - droppedVideoFrameCallbacks?: number; - send?: VoiceEngineV2BridgeSendStats | null; -} - -export type VoiceEngineV2BridgeRemoteTrackSubscriptionQuality = VoiceEngineV2RemoteTrackQuality; -export type VoiceEngineV2BridgeRemoteTrackSubscriptionOptions = VoiceEngineV2RemoteTrackSubscriptionOptions; - -export type VoiceEngineV2BridgeHardwareEncoderBackend = 'nvenc' | 'videotoolbox' | 'none' | string; -export type VoiceEngineV2BridgeHardwareEncoderCapabilities = VoiceEngineV2HardwareEncoderCapabilities; - -export interface VoiceEngineV2BridgeCapabilities { - microphoneCapture: boolean; - syntheticMicrophonePcm: boolean; - cameraCapture: boolean; - nativeCameraBackgrounds: boolean; - screenShare: boolean; - screenShareEncodingUpdate: boolean; - screenShareAudio: boolean; - deviceLists: boolean; - outputDeviceSelection: boolean; - participantVolume: boolean; - remoteTrackSubscription: boolean; - dataChannel: boolean; - connectionStats: boolean; - nativeVideoFrames: boolean; - hardwareEncoderCapabilities: boolean; -} - -export type VoiceEngineV2BridgeOperationErrorCode = - | 'unsupported-capability' - | 'not-connected' - | 'invalid-args' - | 'native-error'; - -export interface VoiceEngineV2BridgeOperationError { - code: VoiceEngineV2BridgeOperationErrorCode; - message: string; - capability?: keyof VoiceEngineV2BridgeCapabilities | string; -} - -export interface VoiceEngineV2BridgeOperationSuccess { - ok: true; -} - -export interface VoiceEngineV2BridgeOperationFailure { - ok: false; - error: VoiceEngineV2BridgeOperationError; -} - -export type VoiceEngineV2BridgeOperationResult = - | VoiceEngineV2BridgeOperationSuccess - | VoiceEngineV2BridgeOperationFailure; - -export const VOICE_ENGINE_V2_OPERATION_SUCCESS: VoiceEngineV2BridgeOperationSuccess = {ok: true}; - -export function createVoiceEngineV2OperationFailure( - error: VoiceEngineV2BridgeOperationError, -): VoiceEngineV2BridgeOperationFailure { - return {ok: false, error}; -} - -export interface VoiceEngineV2BridgeParticipantVolumeOptions { - participantSid: string; - volume: number; -} - -export interface VoiceEngineV2BridgeSpeakingDetectionOptions { - localThresholdRms: number; - remoteThresholdRms: number; -} - -export interface VoiceEngineV2BridgeIpcMethods { - isSupported: {args: []; result: boolean}; - getCapabilities: {args: []; result: VoiceEngineV2BridgeCapabilities}; - prewarm: {args: []; result: void}; - getHardwareEncoderCapabilities: {args: []; result: VoiceEngineV2BridgeHardwareEncoderCapabilities}; - connect: {args: [VoiceEngineV2BridgeConnectOptions]; result: void}; - disconnect: {args: []; result: void}; - isConnected: {args: []; result: boolean}; - publishMicrophone: {args: [VoiceEngineV2BridgePublishMicrophoneOptions]; result: VoiceEngineV2BridgeOperationResult}; - pushPcm: {args: [VoiceEngineV2BridgePcmFrame]; result: boolean}; - publishScreen: {args: [VoiceEngineV2BridgePublishScreenOptions]; result: void}; - updateScreenShareEncoding: {args: [VoiceEngineV2BridgeUpdateScreenShareEncodingOptions]; result: void}; - unpublishScreen: {args: []; result: void}; - publishScreenAudio: {args: [VoiceEngineV2BridgePublishScreenAudioOptions]; result: void}; - pushScreenAudioPcm: {args: [VoiceEngineV2BridgePcmFrame]; result: boolean}; - pushScreenAudioFloat: {args: [VoiceEngineV2BridgeFloatPcmFrame]; result: boolean}; - unpublishScreenAudio: {args: []; result: void}; - setMicEnabled: {args: [boolean]; result: VoiceEngineV2BridgeOperationResult}; - setSpeakingDetection: {args: [VoiceEngineV2BridgeSpeakingDetectionOptions]; result: void}; - listAudioInputDevices: {args: []; result: Array}; - listAudioOutputDevices: {args: []; result: Array}; - setAudioOutputDevice: {args: [string]; result: void}; - setParticipantVolume: {args: [VoiceEngineV2BridgeParticipantVolumeOptions]; result: void}; - setRemoteTrackSubscription: {args: [VoiceEngineV2BridgeRemoteTrackSubscriptionOptions]; result: void}; - publishData: {args: [VoiceEngineV2BridgePublishDataOptions]; result: void}; - listCameraDevices: {args: []; result: Array}; - publishCamera: {args: [VoiceEngineV2BridgePublishCameraOptions]; result: void}; - updateCameraCapture: {args: [VoiceEngineV2BridgeUpdateCameraCaptureOptions]; result: void}; - publishNativeCameraSink: { - args: [VoiceEngineV2BridgePublishCameraOptions]; - result: VoiceEngineV2BridgePublishNativeCameraSinkResult; - }; - publishProcessedCamera: { - args: [VoiceEngineV2BridgePublishProcessedCameraOptions]; - result: VoiceEngineV2BridgePublishProcessedCameraResult; - }; - pushProcessedCameraFrame: {args: [VoiceEngineV2BridgeProcessedCameraFrame]; result: boolean}; - pushCameraBackgroundFrame: {args: [VoiceEngineV2BridgeProcessedCameraFrame]; result: boolean}; - clearCameraBackgroundFrame: {args: []; result: void}; - publishDeviceScreenShare: {args: [VoiceEngineV2BridgePublishDeviceScreenShareOptions]; result: void}; - unpublishCamera: {args: []; result: void}; - isPublishingCamera: {args: []; result: boolean}; - startCameraPreview: { - args: [VoiceEngineV2BridgeStartCameraPreviewOptions]; - result: VoiceEngineV2BridgeCameraPreviewInfo; - }; - stopCameraPreview: {args: []; result: void}; - getConnectionStats: {args: []; result: VoiceEngineV2BridgeStats}; - getVoiceEngineReadiness: {args: []; result: VoiceEngineV2BridgeReadiness}; - getAudioDeviceModuleState: {args: []; result: VoiceEngineV2BridgeAudioDeviceModuleState}; -} - -export const VOICE_ENGINE_V2_BRIDGE_METHODS = [ - 'isSupported', - 'getCapabilities', - 'prewarm', - 'getHardwareEncoderCapabilities', - 'connect', - 'disconnect', - 'isConnected', - 'publishMicrophone', - 'pushPcm', - 'publishScreen', - 'updateScreenShareEncoding', - 'unpublishScreen', - 'publishScreenAudio', - 'pushScreenAudioPcm', - 'pushScreenAudioFloat', - 'unpublishScreenAudio', - 'setMicEnabled', - 'setSpeakingDetection', - 'listAudioInputDevices', - 'listAudioOutputDevices', - 'setAudioOutputDevice', - 'setParticipantVolume', - 'setRemoteTrackSubscription', - 'publishData', - 'listCameraDevices', - 'publishCamera', - 'updateCameraCapture', - 'publishNativeCameraSink', - 'publishProcessedCamera', - 'pushProcessedCameraFrame', - 'pushCameraBackgroundFrame', - 'clearCameraBackgroundFrame', - 'publishDeviceScreenShare', - 'unpublishCamera', - 'isPublishingCamera', - 'startCameraPreview', - 'stopCameraPreview', - 'getConnectionStats', - 'getVoiceEngineReadiness', - 'getAudioDeviceModuleState', - 'onEvent', - 'onVideoFrame', -] as const satisfies ReadonlyArray; - -export const VOICE_ENGINE_V2_IPC_CHANNELS = { - isSupported: 'voice-engine-v2:is-supported', - getCapabilities: 'voice-engine-v2:get-capabilities', - prewarm: 'voice-engine-v2:prewarm', - getHardwareEncoderCapabilities: 'voice-engine-v2:get-hardware-encoder-capabilities', - connect: 'voice-engine-v2:connect', - disconnect: 'voice-engine-v2:disconnect', - isConnected: 'voice-engine-v2:is-connected', - publishMicrophone: 'voice-engine-v2:publish-microphone', - pushPcm: 'voice-engine-v2:push-pcm', - publishScreen: 'voice-engine-v2:publish-screen', - updateScreenShareEncoding: 'voice-engine-v2:update-screen-share-encoding', - unpublishScreen: 'voice-engine-v2:unpublish-screen', - publishScreenAudio: 'voice-engine-v2:publish-screen-audio', - pushScreenAudioPcm: 'voice-engine-v2:push-screen-audio-pcm', - pushScreenAudioFloat: 'voice-engine-v2:push-screen-audio-float', - unpublishScreenAudio: 'voice-engine-v2:unpublish-screen-audio', - setMicEnabled: 'voice-engine-v2:set-mic-enabled', - setSpeakingDetection: 'voice-engine-v2:set-speaking-detection', - listAudioInputDevices: 'voice-engine-v2:list-audio-input-devices', - listAudioOutputDevices: 'voice-engine-v2:list-audio-output-devices', - setAudioOutputDevice: 'voice-engine-v2:set-audio-output-device', - setParticipantVolume: 'voice-engine-v2:set-participant-volume', - setRemoteTrackSubscription: 'voice-engine-v2:set-remote-track-subscription', - publishData: 'voice-engine-v2:publish-data', - listCameraDevices: 'voice-engine-v2:list-camera-devices', - publishCamera: 'voice-engine-v2:publish-camera', - updateCameraCapture: 'voice-engine-v2:update-camera-capture', - publishNativeCameraSink: 'voice-engine-v2:publish-native-camera-sink', - publishProcessedCamera: 'voice-engine-v2:publish-processed-camera', - pushProcessedCameraFrame: 'voice-engine-v2:push-processed-camera-frame', - pushCameraBackgroundFrame: 'voice-engine-v2:push-camera-background-frame', - clearCameraBackgroundFrame: 'voice-engine-v2:clear-camera-background-frame', - publishDeviceScreenShare: 'voice-engine-v2:publish-device-screen-share', - unpublishCamera: 'voice-engine-v2:unpublish-camera', - isPublishingCamera: 'voice-engine-v2:is-publishing-camera', - startCameraPreview: 'voice-engine-v2:start-camera-preview', - stopCameraPreview: 'voice-engine-v2:stop-camera-preview', - getConnectionStats: 'voice-engine-v2:get-connection-stats', - getVoiceEngineReadiness: 'voice-engine-v2:get-readiness', - getAudioDeviceModuleState: 'voice-engine-v2:get-audio-device-module-state', -} as const satisfies {[Method in VoiceEngineV2InvokeMethodName]: string}; - -export const VOICE_ENGINE_V2_EVENT_CHANNELS = { - event: 'voice-engine-v2:event', - videoFrame: 'voice-engine-v2:video-frame', -} as const satisfies { - event: string; - videoFrame: string; -}; - -export interface VoiceEngineV2BridgePushedEvents { - event: VoiceEngineV2BridgeEvent; - videoFrame: VoiceEngineV2BridgeVideoFrame; -} - -export interface VoiceEngineV2BridgeApi { - bridgeVersion: number; - isSupported(): Promise; - getCapabilities(): Promise; - prewarm(): Promise; - getHardwareEncoderCapabilities(): Promise; - connect(options: VoiceEngineV2BridgeConnectOptions): Promise; - disconnect(): Promise; - isConnected(): Promise; - publishMicrophone(options: VoiceEngineV2BridgePublishMicrophoneOptions): Promise; - pushPcm(frame: VoiceEngineV2BridgePcmFrame): Promise; - publishScreen(options: VoiceEngineV2BridgePublishScreenOptions): Promise; - updateScreenShareEncoding(options: VoiceEngineV2BridgeUpdateScreenShareEncodingOptions): Promise; - unpublishScreen(): Promise; - publishScreenAudio(options: VoiceEngineV2BridgePublishScreenAudioOptions): Promise; - pushScreenAudioPcm(frame: VoiceEngineV2BridgePcmFrame): Promise; - pushScreenAudioFloat(frame: VoiceEngineV2BridgeFloatPcmFrame): Promise; - unpublishScreenAudio(): Promise; - setMicEnabled(enabled: boolean): Promise; - setSpeakingDetection(options: VoiceEngineV2BridgeSpeakingDetectionOptions): Promise; - listAudioInputDevices(): Promise>; - listAudioOutputDevices(): Promise>; - setAudioOutputDevice(deviceId: string): Promise; - setParticipantVolume(participantSid: string, volume: number): Promise; - setRemoteTrackSubscription(options: VoiceEngineV2BridgeRemoteTrackSubscriptionOptions): Promise; - publishData(options: VoiceEngineV2BridgePublishDataOptions): Promise; - listCameraDevices(): Promise>; - publishCamera(options: VoiceEngineV2BridgePublishCameraOptions): Promise; - updateCameraCapture(options: VoiceEngineV2BridgeUpdateCameraCaptureOptions): Promise; - publishNativeCameraSink( - options: VoiceEngineV2BridgePublishCameraOptions, - ): Promise; - publishProcessedCamera( - options: VoiceEngineV2BridgePublishProcessedCameraOptions, - ): Promise; - pushProcessedCameraFrame(frame: VoiceEngineV2BridgeProcessedCameraFrame): Promise; - pushCameraBackgroundFrame(frame: VoiceEngineV2BridgeProcessedCameraFrame): Promise; - clearCameraBackgroundFrame(): Promise; - publishDeviceScreenShare(options: VoiceEngineV2BridgePublishDeviceScreenShareOptions): Promise; - unpublishCamera(): Promise; - isPublishingCamera(): Promise; - startCameraPreview( - options: VoiceEngineV2BridgeStartCameraPreviewOptions, - ): Promise; - stopCameraPreview(): Promise; - getConnectionStats(): Promise; - getVoiceEngineReadiness(): Promise; - getAudioDeviceModuleState?(): Promise; - onEvent(callback: (event: VoiceEngineV2BridgeEvent) => void): () => void; - onVideoFrame(callback: (frame: VoiceEngineV2BridgeVideoFrame) => void): () => void; -} diff --git a/packages/voice_engine_v2/src/core/participants_tracks.test.ts b/packages/voice_engine_v2/src/core/participants_tracks.test.ts index 60c9c21bb..9daeb2f3b 100644 --- a/packages/voice_engine_v2/src/core/participants_tracks.test.ts +++ b/packages/voice_engine_v2/src/core/participants_tracks.test.ts @@ -6,7 +6,6 @@ import inboundVideoFramesFixtureJson from '../../fixtures/event_logs/inbound_vid import participantsTracksFixtureJson from '../../fixtures/event_logs/participants_tracks.json'; import subscriptionQualityFixtureJson from '../../fixtures/event_logs/subscription_quality.json'; import watchedStreamsFixtureJson from '../../fixtures/event_logs/watched_streams.json'; -import {translateVoiceEngineV2BridgeEventToEvents, translateVoiceEngineV2BridgeVideoFrameToEvent} from '../bridge'; import type {VoiceEngineV2Command} from '../protocol/commands'; import type {VoiceEngineV2Event} from '../protocol/events'; import type { @@ -136,39 +135,45 @@ describe('voice engine v2 participants, subscriptions, and inbound video fixture assertFixture(fixture); }); - it('translates native bridge participant, track, and frame events into v2 projections', () => { - let snapshot = createVoiceEngineV2InitialSnapshot(availableVoiceEngineV2Capabilities()); - const bridgeEvents = translateVoiceEngineV2BridgeEventToEvents({ - type: 'trackSubscribed', - payload: { - participantSid: 'PA_alice', - identity: 'alice', - participantName: 'Alice', - trackSid: 'TR_screen', - trackName: 'screen_share', - kind: 'video', - source: 'screen_share', - muted: false, - subscribed: true, - subscriptionStatus: 'subscribed', + it('projects a subscribed video track and its first frame into room and inbound-video state', () => { + const events: Array = [ + { + type: 'room.trackPublished', + track: { + participantIdentity: 'alice', + participantSid: 'PA_alice', + trackSid: 'TR_screen', + trackName: 'screen_share', + kind: 'video', + source: 'screen', + muted: false, + }, }, - }); - for (const event of bridgeEvents) { - snapshot = transitionVoiceEngineV2(snapshot, event).snapshot; - } - snapshot = transitionVoiceEngineV2( - snapshot, - translateVoiceEngineV2BridgeVideoFrameToEvent({ - meta: { + { + type: 'inboundVideo.trackSubscribed', + track: { + participantSid: 'PA_alice', + participantIdentity: 'alice', + trackSid: 'TR_screen', + source: 'screen', + }, + }, + { + type: 'inboundVideo.frameReceived', + frame: { participantSid: 'PA_alice', trackSid: 'TR_screen', width: 320, height: 180, timestampUs: 100, + byteLength: 86_400, }, - data: new ArrayBuffer(86_400), - }), - ).snapshot; + }, + ]; + let snapshot = createVoiceEngineV2InitialSnapshot(availableVoiceEngineV2Capabilities()); + for (const event of events) { + snapshot = transitionVoiceEngineV2(snapshot, event).snapshot; + } expect(snapshot.room.tracks.TR_screen).toMatchObject({ participantIdentity: 'alice', diff --git a/packages/voice_engine_v2/src/core/reducer.test.ts b/packages/voice_engine_v2/src/core/reducer.test.ts index 8a05c9e3d..7bb94a13d 100644 --- a/packages/voice_engine_v2/src/core/reducer.test.ts +++ b/packages/voice_engine_v2/src/core/reducer.test.ts @@ -121,94 +121,6 @@ describe('transitionVoiceEngineV2', () => { ]); }); - it('defers native microphone publish until the audio device module is ready', () => { - const queued = applyEvents(initialSnapshot(), [ - {type: 'nativeAudioDeviceModule.statusChanged', status: 'warming'}, - {type: 'microphone.publishRequested', options: {deviceId: 'default'}}, - ]); - - const established = transitionVoiceEngineV2(queued, { - type: 'connection.externallyEstablished', - options: {url: 'wss://voice', token: ''}, - }); - - expect(established.snapshot.connection.status).toBe('connected'); - expect(established.snapshot.microphone.status).toBe('idle'); - expect(established.snapshot.microphone.desired).toEqual({deviceId: 'default'}); - expect(established.commands).toEqual([]); - - const ready = transitionVoiceEngineV2(established.snapshot, { - type: 'nativeAudioDeviceModule.statusChanged', - status: 'ready', - }); - - expect(ready.snapshot.microphone.status).toBe('publishing'); - expect(ready.commands).toEqual([{type: 'microphone.publish', operationId: 1, options: {deviceId: 'default'}}]); - }); - - it('fails deferred native microphone publish when the audio device module fails', () => { - const established = applyEvents(initialSnapshot(), [ - {type: 'nativeAudioDeviceModule.statusChanged', status: 'warming'}, - {type: 'microphone.publishRequested', options: {deviceId: 'default'}}, - {type: 'connection.externallyEstablished', options: {url: 'wss://voice', token: ''}}, - ]); - - const failed = transitionVoiceEngineV2(established, { - type: 'nativeAudioDeviceModule.statusChanged', - status: 'failed', - detail: 'ADM probe failed', - }); - - expect(failed.snapshot.microphone.status).toBe('failed'); - expect(failed.snapshot.microphone.failure).toEqual({ - code: 'deviceUnavailable', - capability: 'microphone', - message: 'ADM probe failed', - }); - expect(failed.commands).toEqual([]); - }); - - it('does not republish an already published microphone on repeated ADM ready events', () => { - const established = applyEvents(initialSnapshot(), [ - {type: 'nativeAudioDeviceModule.statusChanged', status: 'warming'}, - {type: 'microphone.publishRequested', options: {deviceId: 'default'}}, - {type: 'connection.externallyEstablished', options: {url: 'wss://voice', token: ''}}, - ]); - const ready = transitionVoiceEngineV2(established, { - type: 'nativeAudioDeviceModule.statusChanged', - status: 'ready', - }); - const operationId = ready.commands[0]?.operationId ?? -1; - const published = transitionVoiceEngineV2(ready.snapshot, { - type: 'microphone.publishSucceeded', - operationId, - }).snapshot; - - const repeated = transitionVoiceEngineV2(published, { - type: 'nativeAudioDeviceModule.statusChanged', - status: 'ready', - }); - - expect(repeated.snapshot.microphone.status).toBe('published'); - expect(repeated.commands).toEqual([]); - }); - - it('does not let set-microphone-enabled implicitly publish while native ADM is warming', () => { - const established = applyEvents(initialSnapshot(), [ - {type: 'nativeAudioDeviceModule.statusChanged', status: 'warming'}, - {type: 'connection.externallyEstablished', options: {url: 'wss://voice', token: ''}}, - ]); - - const enabled = transitionVoiceEngineV2(established, { - type: 'microphone.setEnabledRequested', - enabled: true, - }); - - expect(enabled.snapshot.microphone.enabled).toBe(true); - expect(enabled.snapshot.microphone.status).toBe('idle'); - expect(enabled.commands).toEqual([]); - }); - it('treats a repeated connection.externallyEstablished for the same endpoint as a no-op', () => { const established = applyEvents(initialSnapshot(), [ {type: 'connection.externallyEstablished', options: {url: 'wss://voice', token: ''}}, diff --git a/packages/voice_engine_v2/src/core/reducers/_dispatch.ts b/packages/voice_engine_v2/src/core/reducers/_dispatch.ts index 08f1d5b60..4e3f33bc8 100644 --- a/packages/voice_engine_v2/src/core/reducers/_dispatch.ts +++ b/packages/voice_engine_v2/src/core/reducers/_dispatch.ts @@ -17,7 +17,6 @@ import {transitionImplementation} from './implementation'; import {transitionInboundVideo} from './inboundVideo'; import {transitionLifecycle} from './lifecycle'; import {transitionMicrophone} from './microphone'; -import {transitionNativeAudioDeviceModule} from './nativeAudioDeviceModule'; import {transitionNativeAudioTap} from './nativeAudioTap'; import {transitionNativeCapture} from './nativeCapture'; import {transitionNativeFrameSink} from './nativeFrameSink'; @@ -179,8 +178,6 @@ export function dispatchPlatformEvent( case 'devices.selectAudioOutputRequested': case 'devices.selectCameraRequested': return transitionDevices(snapshot, event); - case 'nativeAudioDeviceModule.statusChanged': - return transitionNativeAudioDeviceModule(snapshot, event); case 'nativeCapture.startRequested': case 'nativeCapture.updateRequested': case 'nativeCapture.stopRequested': diff --git a/packages/voice_engine_v2/src/core/reducers/microphone.ts b/packages/voice_engine_v2/src/core/reducers/microphone.ts index 4e567f99e..1709a1af1 100644 --- a/packages/voice_engine_v2/src/core/reducers/microphone.ts +++ b/packages/voice_engine_v2/src/core/reducers/microphone.ts @@ -203,46 +203,6 @@ function microphoneOptionsWithAudioInputDevice( return {...options, deviceId}; } -function nativeAudioDeviceModuleNotReadyError(snapshot: VoiceEngineV2Snapshot): VoiceEngineV2Error { - assert.ok(snapshot != null, 'nativeAudioDeviceModuleNotReadyError snapshot must not be null'); - assert.ok( - snapshot.nativeAudioDeviceModule.status === 'failed', - 'nativeAudioDeviceModuleNotReadyError requires failed ADM status', - ); - return { - code: 'deviceUnavailable', - capability: 'microphone', - message: snapshot.nativeAudioDeviceModule.detail ?? 'Native audio device module failed to become ready', - }; -} - -function planNativeAudioDeviceModuleGatedMicrophone(snapshot: VoiceEngineV2Snapshot): VoiceEngineV2Transition | null { - assert.ok(snapshot != null, 'planNativeAudioDeviceModuleGatedMicrophone snapshot must not be null'); - assert.ok(snapshot.nativeAudioDeviceModule != null, 'snapshot.nativeAudioDeviceModule must not be null'); - switch (snapshot.nativeAudioDeviceModule.status) { - case 'unsupported': - case 'ready': - return null; - case 'unknown': - case 'warming': - return { - snapshot: {...snapshot, microphone: {...snapshot.microphone, status: 'idle', failure: null}}, - commands: [], - }; - case 'failed': { - const error = nativeAudioDeviceModuleNotReadyError(snapshot); - return { - snapshot: { - ...snapshot, - microphone: {...snapshot.microphone, status: 'failed', published: null, operationId: null, failure: error}, - lastFailure: error, - }, - commands: [], - }; - } - } -} - export function planMicrophoneDeviceChange( snapshot: VoiceEngineV2Snapshot, deviceId: string | null, @@ -327,8 +287,6 @@ export function beginMicrophonePublish(snapshot: VoiceEngineV2Snapshot): VoiceEn }; } if (!isConnected(snapshot)) return {snapshot, commands: []}; - const gated = planNativeAudioDeviceModuleGatedMicrophone(snapshot); - if (gated) return gated; if (snapshot.microphone.status === 'publishing') return {snapshot, commands: []}; if (snapshot.microphone.status === 'published' && sameMicrophoneOptions(snapshot.microphone.published, desired)) { return {snapshot, commands: []}; diff --git a/packages/voice_engine_v2/src/core/reducers/nativeAudioDeviceModule.ts b/packages/voice_engine_v2/src/core/reducers/nativeAudioDeviceModule.ts deleted file mode 100644 index b6093c784..000000000 --- a/packages/voice_engine_v2/src/core/reducers/nativeAudioDeviceModule.ts +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; -import type {VoiceEngineV2Event} from '../../protocol/events'; -import type {VoiceEngineV2Snapshot, VoiceEngineV2Transition} from '../state'; -import {beginMicrophonePublish} from './microphone'; - -type VoiceEngineV2NativeAudioDeviceModuleEvent = Extract< - VoiceEngineV2Event, - {type: 'nativeAudioDeviceModule.statusChanged'} ->; - -export function transitionNativeAudioDeviceModule( - snapshot: VoiceEngineV2Snapshot, - event: VoiceEngineV2NativeAudioDeviceModuleEvent, -): VoiceEngineV2Transition { - assert.ok(snapshot != null, 'transitionNativeAudioDeviceModule snapshot must not be null'); - assert.ok(event != null, 'transitionNativeAudioDeviceModule event must not be null'); - assert.equal(event.type, 'nativeAudioDeviceModule.statusChanged', 'unexpected ADM event type'); - assert.equal(typeof event.status, 'string', 'ADM status must be a string'); - const next: VoiceEngineV2Snapshot = { - ...snapshot, - nativeAudioDeviceModule: { - status: event.status, - detail: event.detail ?? null, - }, - }; - return beginMicrophonePublish(next); -} diff --git a/packages/voice_engine_v2/src/core/state.ts b/packages/voice_engine_v2/src/core/state.ts index 8f1f16276..264c35722 100644 --- a/packages/voice_engine_v2/src/core/state.ts +++ b/packages/voice_engine_v2/src/core/state.ts @@ -27,7 +27,6 @@ import type { VoiceEngineV2LiveKitRoomState, VoiceEngineV2MediaStatus, VoiceEngineV2MicrophoneOptions, - VoiceEngineV2NativeAudioDeviceModuleState, VoiceEngineV2NativeAudioTapOptions, VoiceEngineV2NativeCaptureOptions, VoiceEngineV2NativeFrameSinkOptions, @@ -162,7 +161,6 @@ export interface VoiceEngineV2Snapshot { connection: VoiceEngineV2ConnectionState; gateway: VoiceEngineV2GatewayState; liveKit: VoiceEngineV2LiveKitState; - nativeAudioDeviceModule: VoiceEngineV2NativeAudioDeviceModuleState; microphone: VoiceEngineV2MicrophoneState; camera: VoiceEngineV2LocalMediaState; screen: VoiceEngineV2LocalMediaState; @@ -286,10 +284,6 @@ export function createVoiceEngineV2InitialSnapshot( serverRegion: null, failure: null, }, - nativeAudioDeviceModule: { - status: 'unsupported', - detail: null, - }, microphone: { status: 'idle', desired: null, diff --git a/packages/voice_engine_v2/src/index.ts b/packages/voice_engine_v2/src/index.ts index bc4fac68c..c32f744f2 100644 --- a/packages/voice_engine_v2/src/index.ts +++ b/packages/voice_engine_v2/src/index.ts @@ -1,95 +1,5 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -export type { - VoiceEngineV2BridgeApi, - VoiceEngineV2BridgeAudioDeviceRole, - VoiceEngineV2BridgeAudioInputDevice, - VoiceEngineV2BridgeAudioOutputDevice, - VoiceEngineV2BridgeCameraDevice, - VoiceEngineV2BridgeCapabilities, - VoiceEngineV2BridgeConnectionQuality, - VoiceEngineV2BridgeConnectOptions, - VoiceEngineV2BridgeDroppedEventCounts, - VoiceEngineV2BridgeEvent, - VoiceEngineV2BridgeEventPayloads, - VoiceEngineV2BridgeEventType, - VoiceEngineV2BridgeFloatPcmFrame, - VoiceEngineV2BridgeHardwareEncoderBackend, - VoiceEngineV2BridgeHardwareEncoderCapabilities, - VoiceEngineV2BridgeInboundStat, - VoiceEngineV2BridgeIpcMethods, - VoiceEngineV2BridgeKnownEvent, - VoiceEngineV2BridgeKnownEventType, - VoiceEngineV2BridgeLocalTrackRepublishedEventPayload, - VoiceEngineV2BridgeMethodName, - VoiceEngineV2BridgeOperationError, - VoiceEngineV2BridgeOperationErrorCode, - VoiceEngineV2BridgeOperationFailure, - VoiceEngineV2BridgeOperationResult, - VoiceEngineV2BridgeOperationSuccess, - VoiceEngineV2BridgeOutboundStat, - VoiceEngineV2BridgeParticipantEventPayload, - VoiceEngineV2BridgeParticipantVolumeOptions, - VoiceEngineV2BridgePcmFrame, - VoiceEngineV2BridgePublishCameraOptions, - VoiceEngineV2BridgePublishDataOptions, - VoiceEngineV2BridgePublishDeviceScreenShareOptions, - VoiceEngineV2BridgePublishMicrophoneOptions, - VoiceEngineV2BridgePublishScreenAudioOptions, - VoiceEngineV2BridgePublishScreenOptions, - VoiceEngineV2BridgePushedEvents, - VoiceEngineV2BridgeRemoteTrackSubscriptionOptions, - VoiceEngineV2BridgeRemoteTrackSubscriptionQuality, - VoiceEngineV2BridgeScreenPacing, - VoiceEngineV2BridgeSendStats, - VoiceEngineV2BridgeStats, - VoiceEngineV2BridgeSubscribedTrackEventPayload, - VoiceEngineV2BridgeSubscriptionStatus, - VoiceEngineV2BridgeTrackEventPayload, - VoiceEngineV2BridgeTrackKind, - VoiceEngineV2BridgeTrackSource, - VoiceEngineV2BridgeTrackSubscriptionFailedEventPayload, - VoiceEngineV2BridgeUpdateScreenShareEncodingOptions, - VoiceEngineV2BridgeVideoFrame, - VoiceEngineV2BridgeVideoFrameMeta, - VoiceEngineV2InvokeMethodName, -} from './bridge'; -export { - assertVoiceEngineV2BridgeAudioOptionsInvariants, - assertVoiceEngineV2BridgeVideoOptionsInvariants, - clampVoiceEngineV2ParticipantVolume, - coerceVoiceEngineV2BridgeStats, - createVoiceEngineV2OperationFailure, - getVoiceEngineV2BridgeDroppedEventCounts, - isVoiceEngineV2BridgeBinaryPayload, - isVoiceEngineV2BridgeConnectOptions, - isVoiceEngineV2BridgeFloatPcmFrame, - isVoiceEngineV2BridgeKnownEventType, - isVoiceEngineV2BridgePcmFrame, - isVoiceEngineV2BridgePublishCameraOptions, - isVoiceEngineV2BridgePublishDataOptions, - isVoiceEngineV2BridgePublishDeviceScreenShareOptions, - isVoiceEngineV2BridgePublishMicrophoneOptions, - isVoiceEngineV2BridgePublishScreenAudioOptions, - isVoiceEngineV2BridgePublishScreenOptions, - isVoiceEngineV2BridgeRemoteTrackSubscriptionOptions, - isVoiceEngineV2BridgeRemoteTrackSubscriptionQuality, - isVoiceEngineV2BridgeUpdateScreenShareEncodingOptions, - isVoiceEngineV2ParticipantVolumeOptions, - normalizeVoiceEngineV2BridgeCapabilities, - normalizeVoiceEngineV2BridgeHardwareEncoderCapabilities, - resetVoiceEngineV2BridgeDroppedEventCounts, - translateVoiceEngineV2BridgeEventToEvents, - translateVoiceEngineV2BridgeVideoFrameToEvent, - unavailableVoiceEngineV2BridgeCapabilities, - unavailableVoiceEngineV2BridgeHardwareEncoderCapabilities, - VOICE_ENGINE_V2_BRIDGE_DROPPED_EVENTS_COUNT_MAX, - VOICE_ENGINE_V2_BRIDGE_METHODS, - VOICE_ENGINE_V2_BRIDGE_VERSION, - VOICE_ENGINE_V2_EVENT_CHANNELS, - VOICE_ENGINE_V2_IPC_CHANNELS, - VOICE_ENGINE_V2_OPERATION_SUCCESS, -} from './bridge'; export type { VoiceEngineV2CapabilitiesProjection, VoiceEngineV2ConnectionState, @@ -161,17 +71,10 @@ export type { VoiceEngineV2CameraEncodingPlanAction, VoiceEngineV2CameraEncodingPlanInput, VoiceEngineV2CameraEncodingPlanReason, - VoiceEngineV2LocalTrackReconnectState, VoiceEngineV2MicrophoneFailureContext, VoiceEngineV2MicrophoneOperationFailureAction, - VoiceEngineV2NativeConnectRetryPolicyInput, - VoiceEngineV2NativeParticipantVolumeInput, VoiceEngineV2OperationFailureLike, VoiceEngineV2OperationResultLike, - VoiceEngineV2ParticipantE2eeState, - VoiceEngineV2ReconnectTrackSource, - VoiceEngineV2RetryConnectionState, - VoiceEngineV2RetryVoiceState, VoiceEngineV2ScreenEncodingPlan, VoiceEngineV2ScreenEncodingPlanAction, VoiceEngineV2ScreenEncodingPlanInput, @@ -195,12 +98,9 @@ export { applyVoiceEngineV2CameraEncodingOptions, applyVoiceEngineV2ScreenEncodingOptions, asVoiceEngineV2StatsTrackSource, - boostedVoiceEngineV2VolumePercentToTrackVolume, - clampVoiceEngineV2VolumePercent, classifyVoiceEngineV2TrackStats, coalesceVoiceEngineV2OutboundStats, coerceVoiceEngineV2Stats, - computeVoiceEngineV2NativeParticipantVolume, getVoiceEngineV2MicrophoneOperationFailureAction, hasVoiceEngineV2NativeHardwareEncoder, hasVoiceEngineV2NativeNvencEncoder, @@ -208,17 +108,13 @@ export { isMoreEfficientVoiceEngineV2VideoCodec, maxDecodableVoiceEngineV2VideoCodec, normalizeVoiceEngineV2HardwareEncoderCapabilities, - normalizeVoiceEngineV2ParticipantE2eeState, planVoiceEngineV2CameraEncodingChange, planVoiceEngineV2NegotiatedVideoCodec, planVoiceEngineV2ScreenEncodingChange, - shouldRetryVoiceEngineV2NativeConnectTimeout, - shouldSuppressVoiceEngineV2LocalTrackStateDuringReconnect, summarizeVoiceEngineV2Stats, unavailableVoiceEngineV2HardwareEncoderCapabilities, VOICE_ENGINE_V2_VIDEO_CODEC_FLOOR, VOICE_ENGINE_V2_VIDEO_CODEC_PREFERENCE, - VOICE_ENGINE_V2_VOLUME_MAX_PERCENT, VoiceEngineV2StatsTrackSource, voiceEngineV2VideoCodecRank, worseVoiceEngineV2VideoCodec, @@ -292,8 +188,6 @@ export type { VoiceEngineV2MediaStatus, VoiceEngineV2MicrophoneOptions, VoiceEngineV2Model, - VoiceEngineV2NativeAudioDeviceModuleState, - VoiceEngineV2NativeAudioDeviceModuleStatus, VoiceEngineV2NativeAudioTapOptions, VoiceEngineV2NativeCaptureFrame, VoiceEngineV2NativeCaptureKind, @@ -344,7 +238,6 @@ export { export type { VoiceEngineV2EventLogEntry, VoiceEngineV2EventLogSpillSink, - VoiceEngineV2FrameStatsEvent, VoiceEngineV2MemoryEventLogSpillSink, VoiceEngineV2RuntimeClock, VoiceEngineV2RuntimeListener, @@ -354,10 +247,7 @@ export type { export { commandResultToEvent, createVoiceEngineV2MemoryEventLogSpillSink, - VOICE_ENGINE_V2_FRAME_STATS_FLUSH_INTERVAL_MS, - VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP, VoiceEngineV2Controller, - VoiceEngineV2FrameStatsAccumulator, VoiceEngineV2Runtime, } from './runtime'; export type { diff --git a/packages/voice_engine_v2/src/policies/e2eeState.ts b/packages/voice_engine_v2/src/policies/e2eeState.ts deleted file mode 100644 index aa3b5f152..000000000 --- a/packages/voice_engine_v2/src/policies/e2eeState.ts +++ /dev/null @@ -1,23 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -export type VoiceEngineV2ParticipantE2eeState = 'encrypted' | 'decryption_failed' | 'unencrypted' | 'unknown'; - -export function normalizeVoiceEngineV2ParticipantE2eeState(raw: unknown): VoiceEngineV2ParticipantE2eeState { - switch (raw) { - case 'encrypted': - case 'ok': - return 'encrypted'; - case 'decryption_failed': - case 'decryptionFailed': - case 'error': - case 'broken': - return 'decryption_failed'; - case 'unencrypted': - case 'missing_key': - case 'missingKey': - case 'internal_error': - return 'unencrypted'; - default: - return 'unknown'; - } -} diff --git a/packages/voice_engine_v2/src/policies/index.ts b/packages/voice_engine_v2/src/policies/index.ts index fa0d2294b..1908c1add 100644 --- a/packages/voice_engine_v2/src/policies/index.ts +++ b/packages/voice_engine_v2/src/policies/index.ts @@ -21,8 +21,6 @@ export { voiceEngineV2VideoCodecRank, worseVoiceEngineV2VideoCodec, } from './codecNegotiation'; -export type {VoiceEngineV2ParticipantE2eeState} from './e2eeState'; -export {normalizeVoiceEngineV2ParticipantE2eeState} from './e2eeState'; export { hasVoiceEngineV2NativeHardwareEncoder, hasVoiceEngineV2NativeNvencEncoder, @@ -30,11 +28,6 @@ export { normalizeVoiceEngineV2HardwareEncoderCapabilities, unavailableVoiceEngineV2HardwareEncoderCapabilities, } from './hardwareEncoderCapabilities'; -export type { - VoiceEngineV2LocalTrackReconnectState, - VoiceEngineV2ReconnectTrackSource, -} from './localMediaReconnectSuppression'; -export {shouldSuppressVoiceEngineV2LocalTrackStateDuringReconnect} from './localMediaReconnectSuppression'; export type { VoiceEngineV2MicrophoneFailureContext, VoiceEngineV2MicrophoneOperationFailureAction, @@ -42,19 +35,6 @@ export type { VoiceEngineV2OperationResultLike, } from './microphoneFailureAction'; export {getVoiceEngineV2MicrophoneOperationFailureAction} from './microphoneFailureAction'; -export type { - VoiceEngineV2NativeConnectRetryPolicyInput, - VoiceEngineV2RetryConnectionState, - VoiceEngineV2RetryVoiceState, -} from './nativeConnectRetry'; -export {shouldRetryVoiceEngineV2NativeConnectTimeout} from './nativeConnectRetry'; -export type {VoiceEngineV2NativeParticipantVolumeInput} from './nativeParticipantVolume'; -export { - boostedVoiceEngineV2VolumePercentToTrackVolume, - clampVoiceEngineV2VolumePercent, - computeVoiceEngineV2NativeParticipantVolume, - VOICE_ENGINE_V2_VOLUME_MAX_PERCENT, -} from './nativeParticipantVolume'; export type { VoiceEngineV2ScreenEncodingPlan, VoiceEngineV2ScreenEncodingPlanAction, diff --git a/packages/voice_engine_v2/src/policies/localMediaReconnectSuppression.ts b/packages/voice_engine_v2/src/policies/localMediaReconnectSuppression.ts deleted file mode 100644 index ed63125a0..000000000 --- a/packages/voice_engine_v2/src/policies/localMediaReconnectSuppression.ts +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -export type VoiceEngineV2ReconnectTrackSource = - | 'microphone' - | 'camera' - | 'screen_share' - | 'screen' - | 'unknown' - | string; - -export interface VoiceEngineV2LocalTrackReconnectState { - source: VoiceEngineV2ReconnectTrackSource; - enabled: boolean; - reconnecting: boolean; - restoreVideo: boolean; - restoreStream: boolean; -} - -export function shouldSuppressVoiceEngineV2LocalTrackStateDuringReconnect({ - source, - enabled, - reconnecting, - restoreVideo, - restoreStream, -}: VoiceEngineV2LocalTrackReconnectState): boolean { - if (enabled || !reconnecting) return false; - if (source === 'camera') return restoreVideo; - if (source === 'screen_share' || source === 'screen') return restoreStream; - return false; -} diff --git a/packages/voice_engine_v2/src/policies/nativeConnectRetry.ts b/packages/voice_engine_v2/src/policies/nativeConnectRetry.ts deleted file mode 100644 index 37a60749a..000000000 --- a/packages/voice_engine_v2/src/policies/nativeConnectRetry.ts +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -const PRIVATE_VOICE_SCOPE_ID = '@me'; - -export interface VoiceEngineV2RetryVoiceState { - guild_id: string | null; - channel_id: string | null; -} - -export interface VoiceEngineV2RetryConnectionState { - connectionId: string | null; - guildId: string | null; - channelId: string | null; - connecting: boolean; - connected: boolean; - reconnecting: boolean; -} - -export interface VoiceEngineV2NativeConnectRetryPolicyInput { - connectionId: string; - guildId: string | null; - channelId: string; - voiceState: VoiceEngineV2RetryVoiceState | null; - connectionState: VoiceEngineV2RetryConnectionState; -} - -function normalizeVoiceScope(guildId: string | null | undefined): string { - return guildId ?? PRIVATE_VOICE_SCOPE_ID; -} - -function isGatewayVoiceStateActiveForConnection({ - guildId, - channelId, - voiceState, -}: VoiceEngineV2NativeConnectRetryPolicyInput): boolean { - if (!voiceState?.channel_id) return false; - return ( - voiceState.channel_id === channelId && normalizeVoiceScope(voiceState.guild_id) === normalizeVoiceScope(guildId) - ); -} - -function isAcceptedVoiceConnectionStillActive({ - connectionId, - guildId, - channelId, - connectionState, -}: VoiceEngineV2NativeConnectRetryPolicyInput): boolean { - return ( - connectionState.connectionId === connectionId && - connectionState.channelId === channelId && - normalizeVoiceScope(connectionState.guildId) === normalizeVoiceScope(guildId) && - (connectionState.connecting || connectionState.connected || connectionState.reconnecting) - ); -} - -export function shouldRetryVoiceEngineV2NativeConnectTimeout( - input: VoiceEngineV2NativeConnectRetryPolicyInput, -): boolean { - return isGatewayVoiceStateActiveForConnection(input) || isAcceptedVoiceConnectionStillActive(input); -} diff --git a/packages/voice_engine_v2/src/policies/nativeParticipantVolume.test.ts b/packages/voice_engine_v2/src/policies/nativeParticipantVolume.test.ts deleted file mode 100644 index c147853f7..000000000 --- a/packages/voice_engine_v2/src/policies/nativeParticipantVolume.test.ts +++ /dev/null @@ -1,195 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {describe, expect, it} from 'vitest'; -import { - computeVoiceEngineV2NativeParticipantVolume, - VOICE_ENGINE_V2_VOLUME_MAX_PERCENT, -} from './nativeParticipantVolume'; - -const USER_VOLUME_PERCENTS = [0, 25, 50, 100, 150, VOICE_ENGINE_V2_VOLUME_MAX_PERCENT]; - -describe('computeVoiceEngineV2NativeParticipantVolume deafen handling', () => { - it.each(USER_VOLUME_PERCENTS)('forces volume 0 when effectiveDeaf regardless of user volume %d', (percent) => { - const volume = computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: percent, - outputVolumePercent: 100, - locallyMuted: false, - effectiveDeaf: true, - }); - expect(volume).toBe(0); - }); - - it('forces volume 0 when effectiveDeaf and locallyMuted', () => { - const volume = computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: 100, - outputVolumePercent: 100, - locallyMuted: true, - effectiveDeaf: true, - }); - expect(volume).toBe(0); - }); - - it('forces volume 0 when effectiveDeaf with boosted output volume', () => { - const volume = computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: VOICE_ENGINE_V2_VOLUME_MAX_PERCENT, - outputVolumePercent: VOICE_ENGINE_V2_VOLUME_MAX_PERCENT, - locallyMuted: false, - effectiveDeaf: true, - }); - expect(volume).toBe(0); - }); - - it('returns 0 when locallyMuted and not deafened', () => { - const volume = computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: 100, - outputVolumePercent: 100, - locallyMuted: true, - effectiveDeaf: false, - }); - expect(volume).toBe(0); - }); - - it('returns unity gain at 100/100 when not deafened and not muted', () => { - const volume = computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: 100, - outputVolumePercent: 100, - locallyMuted: false, - effectiveDeaf: false, - }); - expect(volume).toBeCloseTo(1, 5); - }); - - it('treats omitted effectiveDeaf as not deafened', () => { - const volume = computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: 100, - outputVolumePercent: 100, - locallyMuted: false, - }); - expect(volume).toBeCloseTo(1, 5); - }); - - it.each(USER_VOLUME_PERCENTS)('stays within [0, 2] when undeafened at user volume %d', (percent) => { - const volume = computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: percent, - outputVolumePercent: VOICE_ENGINE_V2_VOLUME_MAX_PERCENT, - locallyMuted: false, - effectiveDeaf: false, - }); - expect(volume).toBeGreaterThanOrEqual(0); - expect(volume).toBeLessThanOrEqual(2); - }); - - it('restores a positive volume after undeafen for an audible participant', () => { - const deafened = computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: 150, - outputVolumePercent: 100, - locallyMuted: false, - effectiveDeaf: true, - }); - const restored = computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: 150, - outputVolumePercent: 100, - locallyMuted: false, - effectiveDeaf: false, - }); - expect(deafened).toBe(0); - expect(restored).toBeGreaterThan(1); - }); - - it('forces volume 0 when streamMuted regardless of stream volume', () => { - const volume = computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: 100, - outputVolumePercent: 100, - locallyMuted: false, - effectiveDeaf: false, - streamVolumePercent: 100, - streamMuted: true, - }); - expect(volume).toBe(0); - }); - - it('returns 0 when stream volume is 0', () => { - const volume = computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: 100, - outputVolumePercent: 100, - locallyMuted: false, - effectiveDeaf: false, - streamVolumePercent: 0, - }); - expect(volume).toBe(0); - }); - - it('composes a reduced stream volume below unity gain', () => { - const volume = computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: 100, - outputVolumePercent: 100, - locallyMuted: false, - effectiveDeaf: false, - streamVolumePercent: 50, - }); - expect(volume).toBeGreaterThan(0); - expect(volume).toBeLessThan(1); - }); - - it('treats omitted stream inputs as full stream volume', () => { - const withStream = computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: 100, - outputVolumePercent: 100, - locallyMuted: false, - effectiveDeaf: false, - streamVolumePercent: 100, - streamMuted: false, - }); - const withoutStream = computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: 100, - outputVolumePercent: 100, - locallyMuted: false, - effectiveDeaf: false, - }); - expect(withStream).toBeCloseTo(withoutStream, 5); - }); - - it('forces volume 0 when deafened even if stream is unmuted at full volume', () => { - const volume = computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: 100, - outputVolumePercent: 100, - locallyMuted: false, - effectiveDeaf: true, - streamVolumePercent: 100, - streamMuted: false, - }); - expect(volume).toBe(0); - }); - - it('throws on non-boolean streamMuted', () => { - expect(() => - computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: 100, - outputVolumePercent: 100, - locallyMuted: false, - streamMuted: 'no' as unknown as boolean, - }), - ).toThrow(); - }); - - it('throws on non-boolean locallyMuted', () => { - expect(() => - computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: 100, - outputVolumePercent: 100, - locallyMuted: 'yes' as unknown as boolean, - }), - ).toThrow(); - }); - - it('throws on non-boolean effectiveDeaf', () => { - expect(() => - computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: 100, - outputVolumePercent: 100, - locallyMuted: false, - effectiveDeaf: 1 as unknown as boolean, - }), - ).toThrow(); - }); -}); diff --git a/packages/voice_engine_v2/src/policies/nativeParticipantVolume.ts b/packages/voice_engine_v2/src/policies/nativeParticipantVolume.ts deleted file mode 100644 index 894fbfb18..000000000 --- a/packages/voice_engine_v2/src/policies/nativeParticipantVolume.ts +++ /dev/null @@ -1,58 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; - -export const VOICE_ENGINE_V2_VOLUME_MAX_PERCENT = 200; - -const UNITY_GAIN_PERCENT = 100; -const QUIET_RANGE_EXPONENT = 1.2; -const PARTICIPANT_GAIN_MAX = 2; - -export interface VoiceEngineV2NativeParticipantVolumeInput { - userVolumePercent: number; - outputVolumePercent: number; - locallyMuted: boolean; - effectiveDeaf?: boolean; - streamVolumePercent?: number; - streamMuted?: boolean; -} - -export function clampVoiceEngineV2VolumePercent(value: number): number { - if (!Number.isFinite(value)) return UNITY_GAIN_PERCENT; - return Math.max(0, Math.min(VOICE_ENGINE_V2_VOLUME_MAX_PERCENT, value)); -} - -export function boostedVoiceEngineV2VolumePercentToTrackVolume(value: number): number { - const clamped = clampVoiceEngineV2VolumePercent(value); - if (clamped === 0) return 0; - if (clamped <= UNITY_GAIN_PERCENT) { - return 10 ** (((clamped - UNITY_GAIN_PERCENT) / UNITY_GAIN_PERCENT) * QUIET_RANGE_EXPONENT); - } - return 2 ** ((clamped - UNITY_GAIN_PERCENT) / (VOICE_ENGINE_V2_VOLUME_MAX_PERCENT - UNITY_GAIN_PERCENT)); -} - -export function computeVoiceEngineV2NativeParticipantVolume(input: VoiceEngineV2NativeParticipantVolumeInput): number { - assert.equal(typeof input.locallyMuted, 'boolean', 'locallyMuted must be a boolean'); - if (input.effectiveDeaf !== undefined) { - assert.equal(typeof input.effectiveDeaf, 'boolean', 'effectiveDeaf must be a boolean when provided'); - } - if (input.streamMuted !== undefined) { - assert.equal(typeof input.streamMuted, 'boolean', 'streamMuted must be a boolean when provided'); - } - if (input.effectiveDeaf === true) return 0; - if (input.locallyMuted) return 0; - if (input.streamMuted === true) return 0; - const streamVolumePercent = - input.streamVolumePercent === undefined - ? UNITY_GAIN_PERCENT - : clampVoiceEngineV2VolumePercent(input.streamVolumePercent); - const composedPercent = - (clampVoiceEngineV2VolumePercent(input.userVolumePercent) / UNITY_GAIN_PERCENT) * - (streamVolumePercent / UNITY_GAIN_PERCENT) * - clampVoiceEngineV2VolumePercent(input.outputVolumePercent); - const gain = boostedVoiceEngineV2VolumePercentToTrackVolume(clampVoiceEngineV2VolumePercent(composedPercent)); - const volume = Math.max(0, Math.min(PARTICIPANT_GAIN_MAX, gain)); - assert.ok(volume >= 0, 'participant volume must be non-negative'); - assert.ok(volume <= PARTICIPANT_GAIN_MAX, 'participant volume must not exceed gain cap'); - return volume; -} diff --git a/packages/voice_engine_v2/src/policies/policies.test.ts b/packages/voice_engine_v2/src/policies/policies.test.ts index 9f9161e60..9dad2e953 100644 --- a/packages/voice_engine_v2/src/policies/policies.test.ts +++ b/packages/voice_engine_v2/src/policies/policies.test.ts @@ -2,12 +2,8 @@ import {describe, expect, it} from 'vitest'; import cameraShareEncodingPlanFixtures from '../../fixtures/policies/camera_share_encoding_plan.json'; -import e2eeFixtures from '../../fixtures/policies/e2ee_state_normalization.json'; import hardwareEncoderCapabilityFixtures from '../../fixtures/policies/hardware_encoder_capabilities.json'; -import localMediaReconnectFixtures from '../../fixtures/policies/local_media_reconnect_suppression.json'; import microphoneFailureFixtures from '../../fixtures/policies/microphone_failure_action.json'; -import connectRetryFixtures from '../../fixtures/policies/native_connect_retry.json'; -import volumeFixtures from '../../fixtures/policies/native_participant_volume.json'; import screenShareEncodingPlanFixtures from '../../fixtures/policies/screen_share_encoding_plan.json'; import voiceStatsCoercionFixtures from '../../fixtures/policies/voice_stats_coercion.json'; import voiceStatsSummaryFixtures from '../../fixtures/policies/voice_stats_summary.json'; @@ -22,31 +18,19 @@ import type { import { classifyVoiceEngineV2TrackStats, coerceVoiceEngineV2Stats, - computeVoiceEngineV2NativeParticipantVolume, getVoiceEngineV2MicrophoneOperationFailureAction, hasVoiceEngineV2NativeNvencEncoder, normalizeVoiceEngineV2HardwareEncoderCapabilities, - normalizeVoiceEngineV2ParticipantE2eeState, planVoiceEngineV2CameraEncodingChange, planVoiceEngineV2ScreenEncodingChange, - shouldRetryVoiceEngineV2NativeConnectTimeout, - shouldSuppressVoiceEngineV2LocalTrackStateDuringReconnect, summarizeVoiceEngineV2Stats, - type VoiceEngineV2LocalTrackReconnectState, type VoiceEngineV2MicrophoneFailureContext, - type VoiceEngineV2NativeConnectRetryPolicyInput, type VoiceEngineV2OperationResultLike, type VoiceEngineV2StatsSummary, type VoiceEngineV2StatsTrackClassificationInput, type VoiceEngineV2StatsTrackRoleSelection, } from './index'; -interface ConnectRetryFixture { - name: string; - input: VoiceEngineV2NativeConnectRetryPolicyInput; - expected: boolean; -} - interface MicrophoneFailureFixture { name: string; result: VoiceEngineV2OperationResultLike; @@ -55,33 +39,6 @@ interface MicrophoneFailureFixture { expected: string; } -interface VolumeFixture { - name: string; - input: { - userVolumePercent: number | null; - outputVolumePercent: number | null; - locallyMuted: boolean; - }; - expected?: number; - expectedRange?: { - minExclusive?: number; - maxExclusive?: number; - minInclusive?: number; - maxInclusive?: number; - }; -} - -interface LocalMediaReconnectFixture { - name: string; - input: VoiceEngineV2LocalTrackReconnectState; - expected: boolean; -} - -interface E2eeFixture { - raw: unknown; - expected: string; -} - interface HardwareEncoderCapabilityFixture { name: string; input: unknown; @@ -141,52 +98,12 @@ interface VoiceTrackClassificationFixture { } describe('voice engine v2 policies', () => { - it.each(connectRetryFixtures as Array)('replays native connect retry fixture: $name', ({ - input, - expected, - }) => { - expect(shouldRetryVoiceEngineV2NativeConnectTimeout(input)).toBe(expected); - }); - it.each( microphoneFailureFixtures as Array, )('replays microphone failure action fixture: $name', ({result, requestedEnabled, context, expected}) => { expect(getVoiceEngineV2MicrophoneOperationFailureAction(result, requestedEnabled, context)).toBe(expected); }); - it.each(volumeFixtures as Array)('replays native volume fixture: $name', (fixture) => { - const gain = computeVoiceEngineV2NativeParticipantVolume({ - userVolumePercent: fixture.input.userVolumePercent ?? Number.NaN, - outputVolumePercent: fixture.input.outputVolumePercent ?? Number.NaN, - locallyMuted: fixture.input.locallyMuted, - }); - if (fixture.expected !== undefined) { - expect(gain).toBeCloseTo(fixture.expected, 5); - } - if (fixture.expectedRange?.minExclusive !== undefined) { - expect(gain).toBeGreaterThan(fixture.expectedRange.minExclusive); - } - if (fixture.expectedRange?.maxExclusive !== undefined) { - expect(gain).toBeLessThan(fixture.expectedRange.maxExclusive); - } - if (fixture.expectedRange?.minInclusive !== undefined) { - expect(gain).toBeGreaterThanOrEqual(fixture.expectedRange.minInclusive); - } - if (fixture.expectedRange?.maxInclusive !== undefined) { - expect(gain).toBeLessThanOrEqual(fixture.expectedRange.maxInclusive); - } - }); - - it.each( - localMediaReconnectFixtures as Array, - )('replays local media reconnect suppression fixture: $name', ({input, expected}) => { - expect(shouldSuppressVoiceEngineV2LocalTrackStateDuringReconnect(input)).toBe(expected); - }); - - it.each(e2eeFixtures as Array)('replays E2EE state normalization fixture %#', ({raw, expected}) => { - expect(normalizeVoiceEngineV2ParticipantE2eeState(raw)).toBe(expected); - }); - it.each( hardwareEncoderCapabilityFixtures as Array, )('replays hardware encoder capability fixture: $name', ({ diff --git a/packages/voice_engine_v2/src/protocol/events.ts b/packages/voice_engine_v2/src/protocol/events.ts index 1655531f2..ef78f4445 100644 --- a/packages/voice_engine_v2/src/protocol/events.ts +++ b/packages/voice_engine_v2/src/protocol/events.ts @@ -39,7 +39,6 @@ import type { VoiceEngineV2LiveKitRoomState, VoiceEngineV2LocalStreamSource, VoiceEngineV2MicrophoneOptions, - VoiceEngineV2NativeAudioDeviceModuleStatus, VoiceEngineV2NativeAudioTapOptions, VoiceEngineV2NativeCaptureFrame, VoiceEngineV2NativeCaptureOptions, @@ -198,11 +197,6 @@ export type VoiceEngineV2Event = | {type: 'devices.selectAudioInputRequested'; deviceId: string | null} | {type: 'devices.selectAudioOutputRequested'; deviceId: string | null} | {type: 'devices.selectCameraRequested'; deviceId: string | null} - | { - type: 'nativeAudioDeviceModule.statusChanged'; - status: VoiceEngineV2NativeAudioDeviceModuleStatus; - detail?: string | null; - } | {type: 'audioControls.changed'; controls: VoiceEngineV2AudioControlsPatch} | {type: 'nativeCapture.startRequested'; options: VoiceEngineV2NativeCaptureOptions} | {type: 'nativeCapture.updateRequested'; options: VoiceEngineV2NativeCaptureOptions} diff --git a/packages/voice_engine_v2/src/protocol/index.ts b/packages/voice_engine_v2/src/protocol/index.ts index 692160be9..c444febf3 100644 --- a/packages/voice_engine_v2/src/protocol/index.ts +++ b/packages/voice_engine_v2/src/protocol/index.ts @@ -59,8 +59,6 @@ export type { VoiceEngineV2MediaStatus, VoiceEngineV2MicrophoneOptions, VoiceEngineV2Model, - VoiceEngineV2NativeAudioDeviceModuleState, - VoiceEngineV2NativeAudioDeviceModuleStatus, VoiceEngineV2NativeAudioTapOptions, VoiceEngineV2NativeCaptureFrame, VoiceEngineV2NativeCaptureKind, diff --git a/packages/voice_engine_v2/src/protocol/types.ts b/packages/voice_engine_v2/src/protocol/types.ts index ea482a07d..1d972be58 100644 --- a/packages/voice_engine_v2/src/protocol/types.ts +++ b/packages/voice_engine_v2/src/protocol/types.ts @@ -11,7 +11,6 @@ export type VoiceEngineV2ConnectionStatus = | 'failed'; export type VoiceEngineV2MediaStatus = 'idle' | 'publishing' | 'published' | 'unpublishing' | 'failed'; -export type VoiceEngineV2NativeAudioDeviceModuleStatus = 'unknown' | 'unsupported' | 'warming' | 'ready' | 'failed'; export type VoiceEngineV2OperationId = number; @@ -175,11 +174,6 @@ export interface VoiceEngineV2LiveKitRoomState { serverRegion: string | null; } -export interface VoiceEngineV2NativeAudioDeviceModuleState { - status: VoiceEngineV2NativeAudioDeviceModuleStatus; - detail: string | null; -} - export interface VoiceEngineV2MicrophoneOptions { deviceId?: string; echoCancellation?: boolean; diff --git a/packages/voice_engine_v2/src/runtime/frameStatsBatching.bench.ts b/packages/voice_engine_v2/src/runtime/frameStatsBatching.bench.ts deleted file mode 100644 index 6d85c8442..000000000 --- a/packages/voice_engine_v2/src/runtime/frameStatsBatching.bench.ts +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {bench, describe} from 'vitest'; -import type {VoiceEngineV2InboundVideoFrame} from '../protocol/types'; -import {VoiceEngineV2FrameStatsAccumulator} from './frameStatsBatching'; - -const frame: VoiceEngineV2InboundVideoFrame = Object.freeze({ - participantSid: 'sid-remote', - participantIdentity: 'identity-remote', - trackSid: 'track-video-1', - width: 1280, - height: 720, - timestampUs: 1_000_000, - byteLength: 320_000, -}); - -const recordAccumulator = new VoiceEngineV2FrameStatsAccumulator(); -const flushAccumulator = new VoiceEngineV2FrameStatsAccumulator(); - -describe('voice engine v2 frame stats accumulator hot paths', () => { - bench('record per-frame update', () => { - recordAccumulator.record(frame); - }); - - bench('record then flush one dirty track', () => { - flushAccumulator.record(frame); - flushAccumulator.flushDirty(() => {}); - }); -}); diff --git a/packages/voice_engine_v2/src/runtime/frameStatsBatching.test.ts b/packages/voice_engine_v2/src/runtime/frameStatsBatching.test.ts deleted file mode 100644 index eda939917..000000000 --- a/packages/voice_engine_v2/src/runtime/frameStatsBatching.test.ts +++ /dev/null @@ -1,129 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import {describe, expect, it} from 'vitest'; -import type {VoiceEngineV2InboundVideoFrame} from '../protocol/types'; -import { - VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP, - VoiceEngineV2FrameStatsAccumulator, - type VoiceEngineV2FrameStatsEvent, -} from './frameStatsBatching'; - -function makeFrame(overrides: Partial = {}): VoiceEngineV2InboundVideoFrame { - return { - participantSid: 'PA_alice', - participantIdentity: 'alice', - trackSid: 'TR_screen', - width: 1280, - height: 720, - timestampUs: 1000, - byteLength: 1_382_400, - ...overrides, - }; -} - -function collectFlush(accumulator: VoiceEngineV2FrameStatsAccumulator): Array { - const events: Array = []; - accumulator.flushDirty((event) => events.push(event)); - return events; -} - -describe('voice engine v2 frame stats accumulator', () => { - it('accumulates a cumulative frame count and flushes one absolute event per dirty track', () => { - const accumulator = new VoiceEngineV2FrameStatsAccumulator(); - for (let index = 1; index <= 30; index += 1) { - expect(accumulator.record(makeFrame({timestampUs: index * 33_333}))).toBe(true); - } - expect(accumulator.trackedTracksCount).toBe(1); - expect(accumulator.dirtyTracksCount).toBe(1); - - const events = collectFlush(accumulator); - expect(events).toEqual([ - { - type: 'inboundVideo.frameStats', - stats: { - participantSid: 'PA_alice', - participantIdentity: 'alice', - trackSid: 'TR_screen', - width: 1280, - height: 720, - frameCount: 30, - lastFrameTimestampUs: 30 * 33_333, - lastFrameByteLength: 1_382_400, - }, - }, - ]); - expect(accumulator.dirtyTracksCount).toBe(0); - expect(collectFlush(accumulator)).toEqual([]); - }); - - it('keeps the cumulative count across flushes and reuses the same record without re-allocating slots', () => { - const accumulator = new VoiceEngineV2FrameStatsAccumulator(); - accumulator.record(makeFrame({timestampUs: 1000})); - collectFlush(accumulator); - accumulator.record(makeFrame({timestampUs: 2000, width: 640, height: 360, byteLength: undefined})); - - const events = collectFlush(accumulator); - expect(events).toHaveLength(1); - expect(events[0]?.stats).toMatchObject({ - frameCount: 2, - lastFrameTimestampUs: 2000, - lastFrameByteLength: null, - width: 640, - height: 360, - }); - expect(accumulator.trackedTracksCount).toBe(1); - }); - - it('omits participantIdentity when never observed for the track', () => { - const accumulator = new VoiceEngineV2FrameStatsAccumulator(); - accumulator.record(makeFrame({participantIdentity: undefined})); - - const events = collectFlush(accumulator); - expect(events).toHaveLength(1); - expect(events[0]?.stats).not.toHaveProperty('participantIdentity'); - }); - - it('drops updates for new tracks at the cap and counts the drops', () => { - const accumulator = new VoiceEngineV2FrameStatsAccumulator(); - for (let index = 0; index < VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP; index += 1) { - expect(accumulator.record(makeFrame({trackSid: `TR_${index}`}))).toBe(true); - } - expect(accumulator.trackedTracksCount).toBe(VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP); - - expect(accumulator.record(makeFrame({trackSid: 'TR_overflow'}))).toBe(false); - expect(accumulator.droppedUpdatesCount).toBe(1); - expect(accumulator.record(makeFrame({trackSid: 'TR_0'}))).toBe(true); - expect(accumulator.trackedTracksCount).toBe(VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP); - - accumulator.removeTrack('TR_1'); - expect(accumulator.record(makeFrame({trackSid: 'TR_overflow'}))).toBe(true); - expect(accumulator.trackedTracksCount).toBe(VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP); - }); - - it('removes dirty bookkeeping when a tracked track is removed before a flush', () => { - const accumulator = new VoiceEngineV2FrameStatsAccumulator(); - accumulator.record(makeFrame({trackSid: 'TR_a'})); - accumulator.record(makeFrame({trackSid: 'TR_b'})); - expect(accumulator.dirtyTracksCount).toBe(2); - - accumulator.removeTrack('TR_a'); - expect(accumulator.dirtyTracksCount).toBe(1); - const events = collectFlush(accumulator); - expect(events).toHaveLength(1); - expect(events[0]?.stats.trackSid).toBe('TR_b'); - }); - - it('clear resets records, dirty bookkeeping, and keeps the dropped counter', () => { - const accumulator = new VoiceEngineV2FrameStatsAccumulator(); - for (let index = 0; index <= VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP; index += 1) { - accumulator.record(makeFrame({trackSid: `TR_${index}`})); - } - expect(accumulator.droppedUpdatesCount).toBe(1); - - accumulator.clear(); - expect(accumulator.trackedTracksCount).toBe(0); - expect(accumulator.dirtyTracksCount).toBe(0); - expect(accumulator.droppedUpdatesCount).toBe(1); - expect(collectFlush(accumulator)).toEqual([]); - }); -}); diff --git a/packages/voice_engine_v2/src/runtime/frameStatsBatching.ts b/packages/voice_engine_v2/src/runtime/frameStatsBatching.ts deleted file mode 100644 index 551214fb1..000000000 --- a/packages/voice_engine_v2/src/runtime/frameStatsBatching.ts +++ /dev/null @@ -1,129 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-or-later - -import assert from 'node:assert/strict'; -import type {VoiceEngineV2Event} from '../protocol/events'; -import type {VoiceEngineV2InboundVideoFrame} from '../protocol/types'; -import {VOICE_ENGINE_V2_COALESCED_TRACKS_CAP} from './frameCoalescing'; - -export const VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP = VOICE_ENGINE_V2_COALESCED_TRACKS_CAP; -export const VOICE_ENGINE_V2_FRAME_STATS_FLUSH_INTERVAL_MS = 1000; - -export type VoiceEngineV2FrameStatsEvent = Extract; - -interface VoiceEngineV2FrameStatsRecord { - participantSid: string; - participantIdentity: string | null; - trackSid: string; - width: number; - height: number; - frameCount: number; - lastFrameTimestampUs: number; - lastFrameByteLength: number | null; - dirty: boolean; -} - -export class VoiceEngineV2FrameStatsAccumulator { - private readonly recordsByTrack = new Map(); - private droppedUpdatesCountValue = 0; - private dirtyTracksCountValue = 0; - - record(frame: VoiceEngineV2InboundVideoFrame): boolean { - assert.ok(frame != null, 'record requires a non-null frame'); - assert.equal(typeof frame.trackSid, 'string', 'record requires a string trackSid'); - let entry = this.recordsByTrack.get(frame.trackSid); - if (entry === undefined) { - if (this.recordsByTrack.size >= VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP) { - this.droppedUpdatesCountValue += 1; - assert.ok(this.droppedUpdatesCountValue >= 1, 'dropped updates count must stay positive after a drop'); - return false; - } - entry = { - participantSid: frame.participantSid, - participantIdentity: frame.participantIdentity ?? null, - trackSid: frame.trackSid, - width: frame.width, - height: frame.height, - frameCount: 0, - lastFrameTimestampUs: frame.timestampUs, - lastFrameByteLength: frame.byteLength ?? null, - dirty: false, - }; - this.recordsByTrack.set(frame.trackSid, entry); - assert.ok(this.recordsByTrack.size <= VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP, 'frame stats records exceeded cap'); - } - entry.participantSid = frame.participantSid; - entry.participantIdentity = frame.participantIdentity ?? entry.participantIdentity; - entry.width = frame.width; - entry.height = frame.height; - entry.frameCount += 1; - entry.lastFrameTimestampUs = frame.timestampUs; - entry.lastFrameByteLength = frame.byteLength ?? null; - if (!entry.dirty) { - entry.dirty = true; - this.dirtyTracksCountValue += 1; - } - assert.ok(this.dirtyTracksCountValue <= this.recordsByTrack.size, 'dirty count must not exceed tracked records'); - return true; - } - - flushDirty(emit: (event: VoiceEngineV2FrameStatsEvent) => void): number { - assert.equal(typeof emit, 'function', 'flushDirty requires a function emitter'); - assert.ok(this.dirtyTracksCountValue >= 0, 'dirty count must be non-negative before flush'); - let flushedCount = 0; - for (const entry of this.recordsByTrack.values()) { - if (!entry.dirty) continue; - entry.dirty = false; - flushedCount += 1; - emit({ - type: 'inboundVideo.frameStats', - stats: { - participantSid: entry.participantSid, - ...(entry.participantIdentity !== null ? {participantIdentity: entry.participantIdentity} : {}), - trackSid: entry.trackSid, - width: entry.width, - height: entry.height, - frameCount: entry.frameCount, - lastFrameTimestampUs: entry.lastFrameTimestampUs, - lastFrameByteLength: entry.lastFrameByteLength, - }, - }); - } - assert.equal(flushedCount, this.dirtyTracksCountValue, 'flush must visit every dirty record exactly once'); - this.dirtyTracksCountValue = 0; - return flushedCount; - } - - removeTrack(trackSid: string): void { - assert.equal(typeof trackSid, 'string', 'removeTrack requires a string trackSid'); - assert.ok(trackSid.length > 0, 'removeTrack requires a non-empty trackSid'); - const entry = this.recordsByTrack.get(trackSid); - if (entry === undefined) return; - if (entry.dirty) { - this.dirtyTracksCountValue -= 1; - assert.ok(this.dirtyTracksCountValue >= 0, 'dirty count must stay non-negative after removal'); - } - this.recordsByTrack.delete(trackSid); - } - - clear(): void { - this.recordsByTrack.clear(); - this.dirtyTracksCountValue = 0; - assert.equal(this.recordsByTrack.size, 0, 'clear must remove every tracked record'); - assert.equal(this.dirtyTracksCountValue, 0, 'clear must reset the dirty count'); - } - - get trackedTracksCount(): number { - assert.ok(this.recordsByTrack.size <= VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP, 'frame stats records exceeded cap'); - return this.recordsByTrack.size; - } - - get dirtyTracksCount(): number { - assert.ok(this.dirtyTracksCountValue >= 0, 'dirty count must be non-negative'); - return this.dirtyTracksCountValue; - } - - get droppedUpdatesCount(): number { - assert.ok(this.droppedUpdatesCountValue >= 0, 'dropped updates count must be non-negative'); - return this.droppedUpdatesCountValue; - } -} diff --git a/packages/voice_engine_v2/src/runtime/index.ts b/packages/voice_engine_v2/src/runtime/index.ts index 74903b50a..11e4d5a80 100644 --- a/packages/voice_engine_v2/src/runtime/index.ts +++ b/packages/voice_engine_v2/src/runtime/index.ts @@ -15,12 +15,6 @@ export { isVoiceEngineV2FrameReceivedEvent, VOICE_ENGINE_V2_COALESCED_TRACKS_CAP, } from './frameCoalescing'; -export type {VoiceEngineV2FrameStatsEvent} from './frameStatsBatching'; -export { - VOICE_ENGINE_V2_FRAME_STATS_FLUSH_INTERVAL_MS, - VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP, - VoiceEngineV2FrameStatsAccumulator, -} from './frameStatsBatching'; export type { VoiceEngineV2ClockPort, VoiceEngineV2EntropySource, diff --git a/packages/voice_engine_v2/src/testing/eventLogReplay.test.ts b/packages/voice_engine_v2/src/testing/eventLogReplay.test.ts index fd4265442..aa16ffec1 100644 --- a/packages/voice_engine_v2/src/testing/eventLogReplay.test.ts +++ b/packages/voice_engine_v2/src/testing/eventLogReplay.test.ts @@ -5,7 +5,6 @@ import appVoiceSessionFixtureJson from '../../fixtures/event_logs/app_voice_sess import type {VoiceEngineV2Event} from '../protocol/events'; import {createVoiceEngineV2MemoryEventLogSpillSink} from '../runtime/eventLogRing'; import {coalesceVoiceEngineV2EventSequence} from '../runtime/frameCoalescing'; -import {VoiceEngineV2FrameStatsAccumulator} from '../runtime/frameStatsBatching'; import {VoiceEngineV2Runtime} from '../runtime/VoiceEngineV2Runtime'; import { replayVoiceEngineV2EventLogFixture, @@ -127,45 +126,6 @@ describe('voice engine v2 event-log fixture replay', () => { expect(replay.finalSnapshot.inboundVideo.tracks.TR_screen?.frameCount).toBe(1); }); - it('replays a batched frame-stats log to the identical snapshot with a true cumulative frame count', () => { - const accumulator = new VoiceEngineV2FrameStatsAccumulator(); - const runtime = makeCoalescingRuntime(); - runtime.dispatch({ - type: 'inboundVideo.trackSubscribed', - track: {participantSid: 'PA_alice', participantIdentity: 'alice', trackSid: 'TR_screen', source: 'screen'}, - }); - const burst = 30; - for (let index = 1; index <= burst; index += 1) { - accumulator.record({ - participantSid: 'PA_alice', - participantIdentity: 'alice', - trackSid: 'TR_screen', - width: 1280, - height: 720, - timestampUs: index * 33_333, - byteLength: 1_382_400, - }); - } - const flushedCount = accumulator.flushDirty((event) => { - runtime.dispatch(event); - }); - - expect(flushedCount).toBe(1); - expect(runtime.eventLog).toHaveLength(2); - expect(runtime.snapshot.inboundVideo.tracks.TR_screen?.frameCount).toBe(burst); - expect(runtime.snapshot.inboundVideo.tracks.TR_screen?.lastFrameTimestampUs).toBe(burst * 33_333); - - const fixture: VoiceEngineV2EventLogFixture = { - version: VOICE_ENGINE_V2_EVENT_LOG_FIXTURE_VERSION, - name: 'batched_inbound_video_frame_stats', - steps: runtime.eventLog.map((entry) => ({event: entry.event, commands: entry.commands})), - expected: {finalSnapshot: runtime.snapshot}, - }; - const replay = replayVoiceEngineV2EventLogFixture(fixture); - - expect(replay.finalSnapshot).toEqual(runtime.snapshot); - }); - it('replays the same coalesced log deterministically across runtimes', () => { const rawEvents = inboundVideoBurstEvents(); const first = makeCoalescingRuntime();