refactor(voice): drop the native only surface from voice_engine_v2 and narrow the desktop bridge (#1788)

This commit is contained in:
Hampus
2026-08-20 17:44:37 +02:00
committed by GitHub
parent 9731bac40f
commit 4f21430880
43 changed files with 196 additions and 3682 deletions
@@ -1,93 +0,0 @@
{
"version": 18,
"methods": [
"isSupported",
"getCapabilities",
"prewarm",
"getHardwareEncoderCapabilities",
"connect",
"disconnect",
"isConnected",
"publishMicrophone",
"pushPcm",
"publishScreen",
"updateScreenShareEncoding",
"unpublishScreen",
"publishScreenAudio",
"pushScreenAudioPcm",
"pushScreenAudioFloat",
"unpublishScreenAudio",
"setMicEnabled",
"setSpeakingDetection",
"listAudioInputDevices",
"listAudioOutputDevices",
"setAudioOutputDevice",
"setParticipantVolume",
"setRemoteTrackSubscription",
"publishData",
"listCameraDevices",
"publishCamera",
"updateCameraCapture",
"publishNativeCameraSink",
"publishProcessedCamera",
"pushProcessedCameraFrame",
"pushCameraBackgroundFrame",
"clearCameraBackgroundFrame",
"publishDeviceScreenShare",
"unpublishCamera",
"isPublishingCamera",
"startCameraPreview",
"stopCameraPreview",
"getConnectionStats",
"getVoiceEngineReadiness",
"getAudioDeviceModuleState",
"onEvent",
"onVideoFrame"
],
"ipcChannels": {
"isSupported": "voice-engine-v2:is-supported",
"getCapabilities": "voice-engine-v2:get-capabilities",
"prewarm": "voice-engine-v2:prewarm",
"getHardwareEncoderCapabilities": "voice-engine-v2:get-hardware-encoder-capabilities",
"connect": "voice-engine-v2:connect",
"disconnect": "voice-engine-v2:disconnect",
"isConnected": "voice-engine-v2:is-connected",
"publishMicrophone": "voice-engine-v2:publish-microphone",
"pushPcm": "voice-engine-v2:push-pcm",
"publishScreen": "voice-engine-v2:publish-screen",
"updateScreenShareEncoding": "voice-engine-v2:update-screen-share-encoding",
"unpublishScreen": "voice-engine-v2:unpublish-screen",
"publishScreenAudio": "voice-engine-v2:publish-screen-audio",
"pushScreenAudioPcm": "voice-engine-v2:push-screen-audio-pcm",
"pushScreenAudioFloat": "voice-engine-v2:push-screen-audio-float",
"unpublishScreenAudio": "voice-engine-v2:unpublish-screen-audio",
"setMicEnabled": "voice-engine-v2:set-mic-enabled",
"setSpeakingDetection": "voice-engine-v2:set-speaking-detection",
"listAudioInputDevices": "voice-engine-v2:list-audio-input-devices",
"listAudioOutputDevices": "voice-engine-v2:list-audio-output-devices",
"setAudioOutputDevice": "voice-engine-v2:set-audio-output-device",
"setParticipantVolume": "voice-engine-v2:set-participant-volume",
"setRemoteTrackSubscription": "voice-engine-v2:set-remote-track-subscription",
"publishData": "voice-engine-v2:publish-data",
"listCameraDevices": "voice-engine-v2:list-camera-devices",
"publishCamera": "voice-engine-v2:publish-camera",
"updateCameraCapture": "voice-engine-v2:update-camera-capture",
"publishNativeCameraSink": "voice-engine-v2:publish-native-camera-sink",
"publishProcessedCamera": "voice-engine-v2:publish-processed-camera",
"pushProcessedCameraFrame": "voice-engine-v2:push-processed-camera-frame",
"pushCameraBackgroundFrame": "voice-engine-v2:push-camera-background-frame",
"clearCameraBackgroundFrame": "voice-engine-v2:clear-camera-background-frame",
"publishDeviceScreenShare": "voice-engine-v2:publish-device-screen-share",
"unpublishCamera": "voice-engine-v2:unpublish-camera",
"isPublishingCamera": "voice-engine-v2:is-publishing-camera",
"startCameraPreview": "voice-engine-v2:start-camera-preview",
"stopCameraPreview": "voice-engine-v2:stop-camera-preview",
"getConnectionStats": "voice-engine-v2:get-connection-stats",
"getVoiceEngineReadiness": "voice-engine-v2:get-readiness",
"getAudioDeviceModuleState": "voice-engine-v2:get-audio-device-module-state"
},
"eventChannels": {
"event": "voice-engine-v2:event",
"videoFrame": "voice-engine-v2:video-frame"
}
}
@@ -658,10 +658,6 @@
"serverRegion": "us-east",
"failure": null
},
"nativeAudioDeviceModule": {
"status": "unsupported",
"detail": null
},
"microphone": {
"status": "published",
"desired": {
@@ -1,14 +0,0 @@
[
{"raw": "encrypted", "expected": "encrypted"},
{"raw": "ok", "expected": "encrypted"},
{"raw": "decryption_failed", "expected": "decryption_failed"},
{"raw": "decryptionFailed", "expected": "decryption_failed"},
{"raw": "error", "expected": "decryption_failed"},
{"raw": "broken", "expected": "decryption_failed"},
{"raw": "unencrypted", "expected": "unencrypted"},
{"raw": "missing_key", "expected": "unencrypted"},
{"raw": "missingKey", "expected": "unencrypted"},
{"raw": "internal_error", "expected": "unencrypted"},
{"raw": "unexpected", "expected": "unknown"},
{"raw": null, "expected": "unknown"}
]
@@ -1,56 +0,0 @@
[
{
"name": "camera unpublish suppressed during reconnect when video restore is pending",
"input": {"source": "camera", "enabled": false, "reconnecting": true, "restoreVideo": true, "restoreStream": false},
"expected": true
},
{
"name": "screen share unpublish suppressed during reconnect when stream restore is pending",
"input": {
"source": "screen_share",
"enabled": false,
"reconnecting": true,
"restoreVideo": false,
"restoreStream": true
},
"expected": true
},
{
"name": "publish events are not suppressed",
"input": {
"source": "screen_share",
"enabled": true,
"reconnecting": true,
"restoreVideo": false,
"restoreStream": true
},
"expected": false
},
{
"name": "inactive reconnect is not suppressed",
"input": {
"source": "screen_share",
"enabled": false,
"reconnecting": false,
"restoreVideo": false,
"restoreStream": true
},
"expected": false
},
{
"name": "camera without video restore is not suppressed",
"input": {"source": "camera", "enabled": false, "reconnecting": true, "restoreVideo": false, "restoreStream": true},
"expected": false
},
{
"name": "microphone is not suppressed",
"input": {
"source": "microphone",
"enabled": false,
"reconnecting": true,
"restoreVideo": true,
"restoreStream": true
},
"expected": false
}
]
@@ -1,98 +0,0 @@
[
{
"name": "retry while accepted connection is still connecting before gateway echo",
"input": {
"connectionId": "connection-a",
"guildId": "guild-a",
"channelId": "channel-a",
"voiceState": null,
"connectionState": {
"connectionId": "connection-a",
"guildId": "guild-a",
"channelId": "channel-a",
"connecting": true,
"connected": false,
"reconnecting": false
}
},
"expected": true
},
{
"name": "retry when gateway voice state confirms channel",
"input": {
"connectionId": "connection-a",
"guildId": "guild-a",
"channelId": "channel-a",
"voiceState": {
"guild_id": "guild-a",
"channel_id": "channel-a"
},
"connectionState": {
"connectionId": "stale-connection",
"guildId": "guild-a",
"channelId": "channel-a",
"connecting": false,
"connected": false,
"reconnecting": false
}
},
"expected": true
},
{
"name": "reject stale local connection",
"input": {
"connectionId": "connection-a",
"guildId": "guild-a",
"channelId": "channel-a",
"voiceState": null,
"connectionState": {
"connectionId": "connection-b",
"guildId": "guild-a",
"channelId": "channel-a",
"connecting": true,
"connected": false,
"reconnecting": false
}
},
"expected": false
},
{
"name": "reject inactive accepted connection",
"input": {
"connectionId": "connection-a",
"guildId": "guild-a",
"channelId": "channel-a",
"voiceState": null,
"connectionState": {
"connectionId": "connection-a",
"guildId": "guild-a",
"channelId": "channel-a",
"connecting": false,
"connected": false,
"reconnecting": false
}
},
"expected": false
},
{
"name": "treat null guild ids as equivalent private voice scope",
"input": {
"connectionId": "connection-a",
"guildId": null,
"channelId": "dm-channel-a",
"voiceState": {
"guild_id": null,
"channel_id": "dm-channel-a"
},
"connectionState": {
"connectionId": "connection-b",
"guildId": null,
"channelId": "dm-channel-a",
"connecting": false,
"connected": false,
"reconnecting": false
}
},
"expected": true
}
]
@@ -1,37 +0,0 @@
[
{
"name": "local mute wins",
"input": {"userVolumePercent": 200, "outputVolumePercent": 100, "locallyMuted": true},
"expected": 0
},
{
"name": "unity gain",
"input": {"userVolumePercent": 100, "outputVolumePercent": 100, "locallyMuted": false},
"expected": 1
},
{
"name": "max user volume reaches ceiling",
"input": {"userVolumePercent": 200, "outputVolumePercent": 100, "locallyMuted": false},
"expected": 2
},
{
"name": "quiet participant below unity",
"input": {"userVolumePercent": 50, "outputVolumePercent": 100, "locallyMuted": false},
"expectedRange": {"minExclusive": 0, "maxExclusive": 1}
},
{
"name": "zero composed volume",
"input": {"userVolumePercent": 0, "outputVolumePercent": 100, "locallyMuted": false},
"expected": 0
},
{
"name": "non finite values fall back to unity inputs",
"input": {"userVolumePercent": null, "outputVolumePercent": null, "locallyMuted": false},
"expected": 1
},
{
"name": "oversized values clamp to native range",
"input": {"userVolumePercent": 999, "outputVolumePercent": 999, "locallyMuted": false},
"expectedRange": {"minInclusive": 0, "maxInclusive": 2}
}
]
@@ -1,104 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {describe, expect, it} from 'vitest';
import {
runVoiceEngineV2AdmWarmup,
VOICE_ENGINE_V2_ADM_WARMUP_POLL_ATTEMPTS_MAX,
VOICE_ENGINE_V2_ADM_WARMUP_POLL_INTERVAL_MS,
} from './admWarmup';
describe('runVoiceEngineV2AdmWarmup', () => {
it('resolves ready when the first probe succeeds', async () => {
let probeCalls = 0;
const state = await runVoiceEngineV2AdmWarmup({
probe: async () => {
probeCalls += 1;
return true;
},
delay: async () => {},
});
expect(state).toEqual({status: 'ready'});
expect(probeCalls).toBe(1);
});
it('polls at the fixed cadence until the probe reports ready', async () => {
let probeCalls = 0;
const delays: Array<number> = [];
const state = await runVoiceEngineV2AdmWarmup({
probe: async () => {
probeCalls += 1;
return probeCalls >= 5;
},
delay: async (durationMs) => {
delays.push(durationMs);
},
});
expect(state).toEqual({status: 'ready'});
expect(probeCalls).toBe(5);
expect(delays.every((durationMs) => durationMs === VOICE_ENGINE_V2_ADM_WARMUP_POLL_INTERVAL_MS)).toBe(true);
});
it('fails after the bounded attempt count when the probe never reports ready', async () => {
let probeCalls = 0;
const state = await runVoiceEngineV2AdmWarmup({
probe: async () => {
probeCalls += 1;
return false;
},
delay: async () => {},
});
expect(state.status).toBe('failed');
expect(state.detail).toContain('deadline');
expect(probeCalls).toBe(VOICE_ENGINE_V2_ADM_WARMUP_POLL_ATTEMPTS_MAX);
});
it('fails with the probe error detail when the probe throws', async () => {
let probeCalls = 0;
const state = await runVoiceEngineV2AdmWarmup({
probe: async () => {
probeCalls += 1;
throw new Error('addon exploded');
},
delay: async () => {},
});
expect(state.status).toBe('failed');
expect(state.detail).toContain('addon exploded');
expect(probeCalls).toBe(1);
});
it('fails at the deadline without stacking probes when the probe hangs forever', async () => {
let probeCalls = 0;
const state = await runVoiceEngineV2AdmWarmup({
probe: () => {
probeCalls += 1;
return new Promise<boolean>(() => {});
},
delay: async () => {},
});
expect(state.status).toBe('failed');
expect(state.detail).toContain('deadline');
expect(probeCalls).toBe(1);
});
it('resolves ready when a slow probe settles after several poll intervals', async () => {
let probeCalls = 0;
let resolveProbe: ((ready: boolean) => void) | null = null;
let delayCalls = 0;
const state = await runVoiceEngineV2AdmWarmup({
probe: () => {
probeCalls += 1;
return new Promise<boolean>((resolve) => {
resolveProbe = resolve;
});
},
delay: async () => {
delayCalls += 1;
if (delayCalls === 3) {
resolveProbe?.(true);
}
},
});
expect(state).toEqual({status: 'ready'});
expect(probeCalls).toBe(1);
});
});
@@ -1,62 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {VoiceEngineV2BridgeAudioDeviceModuleState} from './types';
export const VOICE_ENGINE_V2_ADM_WARMUP_POLL_INTERVAL_MS = 250;
export const VOICE_ENGINE_V2_ADM_WARMUP_DEADLINE_MS = 60_000;
export const VOICE_ENGINE_V2_ADM_WARMUP_POLL_ATTEMPTS_MAX = Math.floor(
VOICE_ENGINE_V2_ADM_WARMUP_DEADLINE_MS / VOICE_ENGINE_V2_ADM_WARMUP_POLL_INTERVAL_MS,
);
export interface VoiceEngineV2AdmWarmupPorts {
probe(): Promise<boolean>;
delay(durationMs: number): Promise<void>;
}
type AdmProbeOutcome = {kind: 'ready'; ready: boolean} | {kind: 'error'; detail: string};
const ADM_PROBE_STILL_PENDING = Symbol('voice-engine-v2-adm-probe-still-pending');
function startBoundedAdmProbe(ports: VoiceEngineV2AdmWarmupPorts): Promise<AdmProbeOutcome> {
return ports.probe().then(
(ready) => ({kind: 'ready', ready}),
(error) => {
const detail = error instanceof Error ? error.message : String(error);
return {kind: 'error', detail: `audio device module probe failed: ${detail}`};
},
);
}
export async function runVoiceEngineV2AdmWarmup(
ports: VoiceEngineV2AdmWarmupPorts,
): Promise<VoiceEngineV2BridgeAudioDeviceModuleState> {
if (VOICE_ENGINE_V2_ADM_WARMUP_POLL_ATTEMPTS_MAX < 1) {
throw new Error('ADM warmup poll attempt bound must be at least 1');
}
let pendingProbe: Promise<AdmProbeOutcome> | null = null;
for (let attempt = 1; attempt <= VOICE_ENGINE_V2_ADM_WARMUP_POLL_ATTEMPTS_MAX; attempt++) {
if (pendingProbe === null) {
pendingProbe = startBoundedAdmProbe(ports);
}
const attemptTimeout = ports.delay(VOICE_ENGINE_V2_ADM_WARMUP_POLL_INTERVAL_MS).then(() => ADM_PROBE_STILL_PENDING);
const outcome = await Promise.race([pendingProbe, attemptTimeout]);
if (outcome === ADM_PROBE_STILL_PENDING) {
continue;
}
pendingProbe = null;
const probeOutcome = outcome as AdmProbeOutcome;
if (probeOutcome.kind === 'error') {
return {status: 'failed', detail: probeOutcome.detail};
}
if (probeOutcome.ready) {
return {status: 'ready'};
}
if (attempt < VOICE_ENGINE_V2_ADM_WARMUP_POLL_ATTEMPTS_MAX) {
await ports.delay(VOICE_ENGINE_V2_ADM_WARMUP_POLL_INTERVAL_MS);
}
}
return {
status: 'failed',
detail: `audio device module warmup exceeded ${VOICE_ENGINE_V2_ADM_WARMUP_DEADLINE_MS}ms deadline`,
};
}
@@ -1,10 +1,8 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {beforeEach, describe, expect, it} from 'vitest';
import fixtureJson from '../../fixtures/bridge/bridge_contract.json';
import {describe, expect, it} from 'vitest';
import {
assertAudioFrameInvariants,
assertSchemaVersion,
assertVideoFrameInvariants,
VOICE_ENGINE_V2_AUDIO_FRAME_BYTES_MAX,
VOICE_ENGINE_V2_VIDEO_DIMENSION_MAX,
@@ -13,125 +11,15 @@ import {
} from './ffi_assertions';
import {
assertVoiceEngineV2BridgeAudioOptionsInvariants,
assertVoiceEngineV2BridgeEnvelopeSchema,
assertVoiceEngineV2BridgeVideoOptionsInvariants,
getVoiceEngineV2BridgeDroppedEventCounts,
getVoiceEngineV2BridgeProcessedCameraI420ByteLength,
isVoiceEngineV2BridgeFloatPcmFrame,
isVoiceEngineV2BridgeKnownEventType,
isVoiceEngineV2BridgeProcessedCameraFrame,
isVoiceEngineV2BridgePublishCameraOptions,
isVoiceEngineV2BridgePublishProcessedCameraOptions,
isVoiceEngineV2BridgeReadiness,
resetVoiceEngineV2BridgeDroppedEventCounts,
translateVoiceEngineV2BridgeEventToEvents,
translateVoiceEngineV2BridgeVideoFrameToEvent,
VOICE_ENGINE_V2_ADM_STATUS_EVENT_TYPE,
VOICE_ENGINE_V2_BRIDGE_METHODS,
VOICE_ENGINE_V2_BRIDGE_VERSION,
VOICE_ENGINE_V2_ENGINE_READY_EVENT_TYPE,
VOICE_ENGINE_V2_EVENT_CHANNELS,
VOICE_ENGINE_V2_IPC_CHANNELS,
normalizeVoiceEngineV2BridgeHardwareEncoderCapabilities,
unavailableVoiceEngineV2BridgeHardwareEncoderCapabilities,
VOICE_ENGINE_V2_HARDWARE_ENCODER_IPC_CHANNEL,
} from './index';
interface VoiceEngineV2BridgeContractFixture {
version: number;
methods: Array<string>;
ipcChannels: Record<string, string>;
eventChannels: Record<string, string>;
}
describe('voice engine v2 bridge contract', () => {
it('matches the shared bridge contract fixture', () => {
const fixture = fixtureJson as VoiceEngineV2BridgeContractFixture;
expect(VOICE_ENGINE_V2_BRIDGE_VERSION).toBe(fixture.version);
expect(VOICE_ENGINE_V2_BRIDGE_METHODS).toEqual(fixture.methods);
expect(VOICE_ENGINE_V2_IPC_CHANNELS).toEqual(fixture.ipcChannels);
expect(VOICE_ENGINE_V2_EVENT_CHANNELS).toEqual(fixture.eventChannels);
expect(new Set(VOICE_ENGINE_V2_BRIDGE_METHODS).size).toBe(VOICE_ENGINE_V2_BRIDGE_METHODS.length);
expect(new Set(Object.values(VOICE_ENGINE_V2_IPC_CHANNELS)).size).toBe(
Object.values(VOICE_ENGINE_V2_IPC_CHANNELS).length,
);
expect(new Set(Object.values(VOICE_ENGINE_V2_EVENT_CHANNELS)).size).toBe(
Object.values(VOICE_ENGINE_V2_EVENT_CHANNELS).length,
);
});
it('accepts the canonical schema version on the FFI boundary', () => {
expect(() => assertSchemaVersion(VOICE_ENGINE_V2_BRIDGE_VERSION)).not.toThrow();
});
it('rejects a schema version older than the bridge', () => {
try {
assertSchemaVersion(VOICE_ENGINE_V2_BRIDGE_VERSION - 1);
expect.fail('expected schema mismatch to throw');
} catch (error) {
expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError);
expect((error as VoiceEngineV2FfiAssertError).code).toBe('schemaVersionMismatch');
}
});
it('rejects a schema version newer than the bridge', () => {
try {
assertSchemaVersion(VOICE_ENGINE_V2_BRIDGE_VERSION + 1);
expect.fail('expected schema mismatch to throw');
} catch (error) {
expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError);
expect((error as VoiceEngineV2FfiAssertError).code).toBe('schemaVersionMismatch');
}
});
});
describe('voice engine v2 readiness contract', () => {
it('registers the readiness method and channel in the shared contract', () => {
expect(VOICE_ENGINE_V2_BRIDGE_METHODS).toContain('getVoiceEngineReadiness');
expect(VOICE_ENGINE_V2_IPC_CHANNELS.getVoiceEngineReadiness).toBe('voice-engine-v2:get-readiness');
});
it('classifies engineReady as a known bridge event type', () => {
expect(VOICE_ENGINE_V2_ENGINE_READY_EVENT_TYPE).toBe('engineReady');
expect(isVoiceEngineV2BridgeKnownEventType(VOICE_ENGINE_V2_ENGINE_READY_EVENT_TYPE)).toBe(true);
});
it('translates engineReady as a renderer-gating event with no reducer output', () => {
resetVoiceEngineV2BridgeDroppedEventCounts();
const events = translateVoiceEngineV2BridgeEventToEvents({
type: 'engineReady',
payload: {ready: true},
});
expect(events).toEqual([]);
const counts = getVoiceEngineV2BridgeDroppedEventCounts();
expect(counts.malformedPayloadCount).toBe(0);
expect(counts.unknownTypeCount).toBe(0);
});
it('translates audio device module status into a runtime event', () => {
resetVoiceEngineV2BridgeDroppedEventCounts();
const events = translateVoiceEngineV2BridgeEventToEvents({
type: VOICE_ENGINE_V2_ADM_STATUS_EVENT_TYPE,
payload: {status: 'warming'},
});
expect(events).toEqual([{type: 'nativeAudioDeviceModule.statusChanged', status: 'warming'}]);
const counts = getVoiceEngineV2BridgeDroppedEventCounts();
expect(counts.malformedPayloadCount).toBe(0);
expect(counts.unknownTypeCount).toBe(0);
});
it('accepts ready and not-ready readiness payloads', () => {
expect(isVoiceEngineV2BridgeReadiness({ready: true})).toBe(true);
expect(isVoiceEngineV2BridgeReadiness({ready: false})).toBe(true);
expect(isVoiceEngineV2BridgeReadiness({ready: false, reason: 'native voice engine prewarm pending'})).toBe(true);
});
it('rejects malformed readiness payloads', () => {
expect(isVoiceEngineV2BridgeReadiness(null)).toBe(false);
expect(isVoiceEngineV2BridgeReadiness({})).toBe(false);
expect(isVoiceEngineV2BridgeReadiness({ready: 'yes'})).toBe(false);
expect(isVoiceEngineV2BridgeReadiness({ready: false, reason: 42})).toBe(false);
expect(isVoiceEngineV2BridgeReadiness({ready: true, reason: 'ready engines carry no reason'})).toBe(false);
it('pins the hardware encoder ipc channel shared by preload and the main process', () => {
expect(VOICE_ENGINE_V2_HARDWARE_ENCODER_IPC_CHANNEL).toBe('voice-engine-v2:get-hardware-encoder-capabilities');
});
});
@@ -151,37 +39,37 @@ describe('voice engine v2 audio frame invariants', () => {
expect(() => assertAudioFrameInvariants(canonicalAudioFrame, 500)).not.toThrow();
});
it('rejects an audio frame with zero bytes', () => {
it('rejects an empty audio frame', () => {
try {
assertAudioFrameInvariants({...canonicalAudioFrame, frameBytes: 0});
expect.fail('expected size assertion to throw');
expect.fail('expected frame bytes assertion to throw');
} catch (error) {
expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError);
expect((error as VoiceEngineV2FfiAssertError).code).toBe('audioFrameBytesOutOfRange');
}
});
it('rejects an audio frame over the byte cap', () => {
it('rejects an audio frame past the byte cap', () => {
try {
assertAudioFrameInvariants({...canonicalAudioFrame, frameBytes: VOICE_ENGINE_V2_AUDIO_FRAME_BYTES_MAX + 1});
expect.fail('expected size assertion to throw');
expect.fail('expected frame bytes assertion to throw');
} catch (error) {
expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError);
expect((error as VoiceEngineV2FfiAssertError).code).toBe('audioFrameBytesOutOfRange');
}
});
it('rejects an audio frame with an unsupported sample rate', () => {
it('rejects an unsupported sample rate', () => {
try {
assertAudioFrameInvariants({...canonicalAudioFrame, sampleRateHz: 44100});
expect.fail('expected sample-rate assertion to throw');
expect.fail('expected sample rate assertion to throw');
} catch (error) {
expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError);
expect((error as VoiceEngineV2FfiAssertError).code).toBe('audioSampleRateInvalid');
}
});
it('rejects an audio frame with an unsupported channel count', () => {
it('rejects an unsupported channel count', () => {
try {
assertAudioFrameInvariants({...canonicalAudioFrame, numChannels: 3});
expect.fail('expected channels assertion to throw');
@@ -191,7 +79,7 @@ describe('voice engine v2 audio frame invariants', () => {
}
});
it('rejects an audio frame whose timestamp does not advance', () => {
it('rejects a repeated audio timestamp', () => {
try {
assertAudioFrameInvariants(canonicalAudioFrame, canonicalAudioFrame.timestampNs);
expect.fail('expected timestamp assertion to throw');
@@ -201,7 +89,7 @@ describe('voice engine v2 audio frame invariants', () => {
}
});
it('rejects an audio frame whose timestamp moves backwards', () => {
it('rejects a regressed audio timestamp', () => {
try {
assertAudioFrameInvariants(canonicalAudioFrame, canonicalAudioFrame.timestampNs + 1);
expect.fail('expected timestamp assertion to throw');
@@ -216,35 +104,35 @@ describe('voice engine v2 video frame invariants', () => {
const canonicalVideoFrame = {
widthPx: 1920,
heightPx: 1080,
frameBytes: (1920 * 1080 * 3) / 2,
frameBytes: 1920 * 1080 * 4,
timestampNs: 1000,
};
it('accepts a canonical video frame with no previous timestamp', () => {
it('accepts a canonical video frame', () => {
expect(() => assertVideoFrameInvariants(canonicalVideoFrame)).not.toThrow();
});
it('rejects a video frame with zero bytes', () => {
it('rejects an empty video frame', () => {
try {
assertVideoFrameInvariants({...canonicalVideoFrame, frameBytes: 0});
expect.fail('expected size assertion to throw');
expect.fail('expected frame bytes assertion to throw');
} catch (error) {
expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError);
expect((error as VoiceEngineV2FfiAssertError).code).toBe('videoFrameBytesOutOfRange');
}
});
it('rejects a video frame over the byte cap', () => {
it('rejects a video frame past the byte cap', () => {
try {
assertVideoFrameInvariants({...canonicalVideoFrame, frameBytes: VOICE_ENGINE_V2_VIDEO_FRAME_BYTES_MAX + 1});
expect.fail('expected size assertion to throw');
expect.fail('expected frame bytes assertion to throw');
} catch (error) {
expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError);
expect((error as VoiceEngineV2FfiAssertError).code).toBe('videoFrameBytesOutOfRange');
}
});
it('rejects a video frame with a zero dimension', () => {
it('rejects a zero video dimension', () => {
try {
assertVideoFrameInvariants({...canonicalVideoFrame, widthPx: 0});
expect.fail('expected dimension assertion to throw');
@@ -254,7 +142,7 @@ describe('voice engine v2 video frame invariants', () => {
}
});
it('rejects a video frame with an oversized dimension', () => {
it('rejects a video dimension past the cap', () => {
try {
assertVideoFrameInvariants({...canonicalVideoFrame, heightPx: VOICE_ENGINE_V2_VIDEO_DIMENSION_MAX + 1});
expect.fail('expected dimension assertion to throw');
@@ -264,7 +152,7 @@ describe('voice engine v2 video frame invariants', () => {
}
});
it('rejects a video frame whose timestamp regresses', () => {
it('rejects a repeated video timestamp', () => {
try {
assertVideoFrameInvariants(canonicalVideoFrame, canonicalVideoFrame.timestampNs);
expect.fail('expected timestamp assertion to throw');
@@ -273,183 +161,6 @@ describe('voice engine v2 video frame invariants', () => {
expect((error as VoiceEngineV2FfiAssertError).code).toBe('videoTimestampRegressed');
}
});
it('documents that inboundVideo.frameReceived will activate this guard once the port leaves shadow', () => {
const hotPathTypes = ['inboundVideo.frameReceived', 'nativeCapture.frame'];
expect(hotPathTypes.length).toBeGreaterThan(0);
for (const type of hotPathTypes) {
expect(typeof type).toBe('string');
}
});
});
describe('voice engine v2 bridge schema version literal', () => {
it('pins the TS bridge version to the literal 18 to pair with the native addon bridge', () => {
expect(VOICE_ENGINE_V2_BRIDGE_VERSION).toBe(18);
});
it('reuses the schema-version airlock helper from bridge exports', () => {
expect(() => assertVoiceEngineV2BridgeEnvelopeSchema(VOICE_ENGINE_V2_BRIDGE_VERSION)).not.toThrow();
});
it('rejects bridge schema mismatch via the exported airlock helper', () => {
try {
assertVoiceEngineV2BridgeEnvelopeSchema(VOICE_ENGINE_V2_BRIDGE_VERSION + 1);
expect.fail('expected schema mismatch to throw');
} catch (error) {
expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError);
expect((error as VoiceEngineV2FfiAssertError).code).toBe('schemaVersionMismatch');
}
});
});
describe('voice engine v2 float pcm bridge frames', () => {
it('accepts Float32Array payloads for native-side PCM conversion', () => {
expect(
isVoiceEngineV2BridgeFloatPcmFrame({
sampleRate: 48_000,
numChannels: 1,
samples: new Float32Array([0, 0.5, -0.5]),
}),
).toBe(true);
});
it('rejects non-float payloads on the float PCM bridge', () => {
expect(
isVoiceEngineV2BridgeFloatPcmFrame({
sampleRate: 48_000,
numChannels: 1,
samples: new Uint8Array([0, 1, 2, 3]).buffer,
}),
).toBe(false);
});
});
describe('voice engine v2 bridge translator airlock', () => {
function makeFrame(width: number, height: number, byteLength: number, timestampUs: number) {
return {
meta: {participantSid: 'PA', trackSid: 'TR', width, height, timestampUs},
data: new ArrayBuffer(byteLength),
};
}
it('translates a canonical video frame received from Rust', () => {
const event = translateVoiceEngineV2BridgeVideoFrameToEvent(makeFrame(1920, 1080, 4096, 1));
expect(event.type).toBe('inboundVideo.frameReceived');
});
it('panics through the airlock when an inbound video frame has zero dimensions', () => {
try {
translateVoiceEngineV2BridgeVideoFrameToEvent(makeFrame(0, 1080, 4096, 1));
expect.fail('expected dimension assertion to throw');
} catch (error) {
expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError);
expect((error as VoiceEngineV2FfiAssertError).code).toBe('videoDimensionOutOfRange');
}
});
it('panics through the airlock when an inbound video frame has zero bytes', () => {
try {
translateVoiceEngineV2BridgeVideoFrameToEvent(makeFrame(1920, 1080, 0, 1));
expect.fail('expected size assertion to throw');
} catch (error) {
expect(error).toBeInstanceOf(VoiceEngineV2FfiAssertError);
expect((error as VoiceEngineV2FfiAssertError).code).toBe('videoFrameBytesOutOfRange');
}
});
});
describe('voice engine v2 processed camera bridge airlock', () => {
it('accepts native camera background modes', () => {
expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundMode: 'none'})).toBe(true);
expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundMode: 'non'})).toBe(true);
expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundMode: 'blur'})).toBe(true);
expect(
isVoiceEngineV2BridgePublishCameraOptions({
backgroundMode: 'custom',
backgroundCustomMediaPath: '/tmp/background.webm',
backgroundCustomMediaKind: 'video',
}),
).toBe(true);
});
it('registers the camera capture update channel in the shared contract', () => {
expect(VOICE_ENGINE_V2_BRIDGE_METHODS).toContain('updateCameraCapture');
expect(VOICE_ENGINE_V2_IPC_CHANNELS.updateCameraCapture).toBe('voice-engine-v2:update-camera-capture');
});
it('accepts native camera mirroring as a typed boolean only', () => {
expect(isVoiceEngineV2BridgePublishCameraOptions({mirror: true})).toBe(true);
expect(isVoiceEngineV2BridgePublishCameraOptions({mirror: false})).toBe(true);
expect(isVoiceEngineV2BridgePublishCameraOptions({mirror: 'true'})).toBe(false);
});
it('accepts camera effect strengths across the full integer range', () => {
expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundBlurStrength: 0})).toBe(true);
expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundBlurStrength: 100})).toBe(true);
expect(
isVoiceEngineV2BridgePublishCameraOptions({
backgroundMode: 'blur',
backgroundBlurStrength: 75,
}),
).toBe(true);
});
it('rejects out-of-range or non-integer camera effect strengths', () => {
expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundBlurStrength: -1})).toBe(false);
expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundBlurStrength: 101})).toBe(false);
expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundBlurStrength: 49.5})).toBe(false);
expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundBlurStrength: '50'})).toBe(false);
expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundBlurStrength: Number.NaN})).toBe(false);
});
it('rejects invalid native camera background modes', () => {
expect(isVoiceEngineV2BridgePublishCameraOptions({backgroundMode: 'sepia'})).toBe(false);
expect(
isVoiceEngineV2BridgePublishCameraOptions({
backgroundMode: 'custom',
backgroundCustomMediaPath: '/tmp/background.webm',
backgroundCustomMediaKind: 'audio',
}),
).toBe(false);
});
it('accepts canonical processed camera publish options', () => {
expect(isVoiceEngineV2BridgePublishProcessedCameraOptions({width: 1280, height: 720, frameRate: 30})).toBe(true);
});
it('rejects processed camera publish options with odd dimensions', () => {
expect(isVoiceEngineV2BridgePublishProcessedCameraOptions({width: 1279, height: 720, frameRate: 30})).toBe(false);
});
it('calculates tight i420 payload byte length', () => {
expect(getVoiceEngineV2BridgeProcessedCameraI420ByteLength(4, 2)).toBe(12);
});
it('accepts tight i420 processed camera frames', () => {
const data = new Uint8Array(getVoiceEngineV2BridgeProcessedCameraI420ByteLength(4, 2));
expect(
isVoiceEngineV2BridgeProcessedCameraFrame({
format: 'i420',
width: 4,
height: 2,
timestampUs: 1,
data,
}),
).toBe(true);
});
it('rejects processed camera frames with loose byte lengths', () => {
expect(
isVoiceEngineV2BridgeProcessedCameraFrame({
format: 'i420',
width: 4,
height: 2,
timestampUs: 1,
data: new ArrayBuffer(11),
}),
).toBe(false);
});
});
describe('voice engine v2 bridge option airlock helpers', () => {
@@ -502,61 +213,64 @@ describe('voice engine v2 bridge option airlock helpers', () => {
});
});
describe('voice engine v2 bridge event translation drop accounting', () => {
beforeEach(() => {
resetVoiceEngineV2BridgeDroppedEventCounts();
});
it('returns no events for a known-but-ignored type without counting a drop', () => {
const events = translateVoiceEngineV2BridgeEventToEvents({
type: 'stats',
payload: {rttMs: 12, outbound: [], inbound: []},
describe('voice engine v2 bridge hardware encoder capabilities', () => {
it('reports an unavailable encoder with the supplied reason and detail', () => {
expect(unavailableVoiceEngineV2BridgeHardwareEncoderCapabilities('load-failed', 'addon missing')).toEqual({
available: false,
backend: 'none',
compiled: false,
runtime: false,
codecs: [],
zeroCopy: false,
nativeInputs: [],
reason: 'load-failed',
detail: 'addon missing',
});
expect(events).toEqual([]);
const counts = getVoiceEngineV2BridgeDroppedEventCounts();
expect(counts.malformedPayloadCount).toBe(0);
expect(counts.unknownTypeCount).toBe(0);
});
it('counts a malformed payload for a translated known type', () => {
const events = translateVoiceEngineV2BridgeEventToEvents({
type: 'trackMuted',
payload: {},
it('omits the detail field when the caller supplies none', () => {
expect(unavailableVoiceEngineV2BridgeHardwareEncoderCapabilities('unsupported-addon-version')).not.toHaveProperty(
'detail',
);
});
it('normalizes a well-formed addon payload', () => {
expect(
normalizeVoiceEngineV2BridgeHardwareEncoderCapabilities({
available: true,
backend: 'nvenc',
compiled: true,
runtime: true,
codecs: ['h264', 42, 'av1'],
zeroCopy: true,
nativeInputs: ['d3d11Texture', null],
}),
).toEqual({
available: true,
backend: 'nvenc',
compiled: true,
runtime: true,
codecs: ['h264', 'av1'],
zeroCopy: true,
nativeInputs: ['d3d11Texture'],
});
expect(events).toEqual([]);
const counts = getVoiceEngineV2BridgeDroppedEventCounts();
expect(counts.malformedPayloadCount).toBe(1);
expect(counts.unknownTypeCount).toBe(0);
});
it('counts an unknown event type as a dropped event', () => {
const events = translateVoiceEngineV2BridgeEventToEvents({
type: 'someFutureEvent',
payload: {},
it('falls back to an unavailable encoder when the addon returns a non-object', () => {
expect(normalizeVoiceEngineV2BridgeHardwareEncoderCapabilities(null)).toEqual({
available: false,
backend: 'none',
compiled: false,
runtime: false,
codecs: [],
zeroCopy: false,
nativeInputs: [],
reason: 'query-failed',
detail: 'Native addon returned an invalid result',
});
expect(events).toEqual([]);
const counts = getVoiceEngineV2BridgeDroppedEventCounts();
expect(counts.malformedPayloadCount).toBe(0);
expect(counts.unknownTypeCount).toBe(1);
});
it('classifies every payload key as a known bridge event type', () => {
expect(isVoiceEngineV2BridgeKnownEventType('trackSubscribed')).toBe(true);
expect(isVoiceEngineV2BridgeKnownEventType('audioPlaybackUnavailable')).toBe(true);
expect(isVoiceEngineV2BridgeKnownEventType('someFutureEvent')).toBe(false);
});
it('still translates a well-formed known event after counting drops', () => {
translateVoiceEngineV2BridgeEventToEvents({type: 'trackMuted', payload: {}});
const events = translateVoiceEngineV2BridgeEventToEvents({
type: 'trackMuted',
payload: {trackSid: 'TR_1'},
});
expect(events).toEqual([{type: 'room.trackMuted', trackSid: 'TR_1'}]);
expect(getVoiceEngineV2BridgeDroppedEventCounts().malformedPayloadCount).toBe(1);
it('falls back to the none backend when the addon omits one', () => {
expect(normalizeVoiceEngineV2BridgeHardwareEncoderCapabilities({available: true}).backend).toBe('none');
});
});
@@ -1,7 +1,5 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {VOICE_ENGINE_V2_BRIDGE_VERSION} from './index';
export const VOICE_ENGINE_V2_AUDIO_FRAME_BYTES_MAX = 1 << 20;
export const VOICE_ENGINE_V2_VIDEO_FRAME_BYTES_MAX = 64 << 20;
export const VOICE_ENGINE_V2_VIDEO_DIMENSION_MAX = 16384;
@@ -24,7 +22,6 @@ interface VoiceEngineV2VideoFrameInvariants {
export class VoiceEngineV2FfiAssertError extends Error {
readonly code:
| 'schemaVersionMismatch'
| 'audioFrameBytesOutOfRange'
| 'audioSampleRateInvalid'
| 'audioChannelsInvalid'
@@ -44,21 +41,6 @@ function isFiniteInteger(value: number): boolean {
return Number.isFinite(value) && Number.isInteger(value);
}
export function assertSchemaVersion(received: number): void {
if (!isFiniteInteger(received)) {
throw new VoiceEngineV2FfiAssertError(
'schemaVersionMismatch',
`schema version must be a finite integer (received ${received})`,
);
}
if (received !== VOICE_ENGINE_V2_BRIDGE_VERSION) {
throw new VoiceEngineV2FfiAssertError(
'schemaVersionMismatch',
`schema version mismatch: expected ${VOICE_ENGINE_V2_BRIDGE_VERSION}, received ${received}`,
);
}
}
function assertAudioFrameBytes(bytes: number): void {
if (!isFiniteInteger(bytes)) {
throw new VoiceEngineV2FfiAssertError(
@@ -1,482 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {assertAudioFrameInvariants, assertSchemaVersion, assertVideoFrameInvariants} from './ffi_assertions';
import type {
VoiceEngineV2BridgeCameraBackgroundCustomMediaKind,
VoiceEngineV2BridgeCameraBackgroundMode,
VoiceEngineV2BridgeCameraPreviewInfo,
VoiceEngineV2BridgeCapabilities,
VoiceEngineV2BridgeConnectOptions,
VoiceEngineV2BridgeFloatPcmFrame,
VoiceEngineV2BridgeHardwareEncoderCapabilities,
VoiceEngineV2BridgeParticipantVolumeOptions,
VoiceEngineV2BridgePcmFrame,
VoiceEngineV2BridgeProcessedCameraFrame,
VoiceEngineV2BridgePublishCameraOptions,
VoiceEngineV2BridgePublishDataOptions,
VoiceEngineV2BridgePublishDeviceScreenShareOptions,
VoiceEngineV2BridgePublishMicrophoneOptions,
VoiceEngineV2BridgePublishProcessedCameraOptions,
VoiceEngineV2BridgePublishProcessedCameraResult,
VoiceEngineV2BridgePublishScreenAudioOptions,
VoiceEngineV2BridgePublishScreenOptions,
VoiceEngineV2BridgeReadiness,
VoiceEngineV2BridgeRemoteTrackSubscriptionOptions,
VoiceEngineV2BridgeRemoteTrackSubscriptionQuality,
VoiceEngineV2BridgeSpeakingDetectionOptions,
VoiceEngineV2BridgeUpdateScreenShareEncodingOptions,
} from './types';
import {VOICE_ENGINE_V2_CAMERA_EFFECT_STRENGTH_MAX, VOICE_ENGINE_V2_CAMERA_EFFECT_STRENGTH_MIN} from './types';
export function assertVoiceEngineV2BridgeAudioOptionsInvariants(options: {
sampleRate: number;
numChannels: number;
}): void {
assertAudioFrameInvariants({
sampleRateHz: options.sampleRate,
numChannels: options.numChannels,
frameBytes: 1,
timestampNs: 1,
});
}
export function assertVoiceEngineV2BridgeVideoOptionsInvariants(options: {width: number; height: number}): void {
assertVideoFrameInvariants({
widthPx: options.width,
heightPx: options.height,
frameBytes: 1,
timestampNs: 1,
});
}
export function assertVoiceEngineV2BridgeEnvelopeSchema(received: number): void {
assertSchemaVersion(received);
}
export function unavailableVoiceEngineV2BridgeCapabilities(): VoiceEngineV2BridgeCapabilities {
return {
microphoneCapture: false,
syntheticMicrophonePcm: false,
cameraCapture: false,
nativeCameraBackgrounds: false,
screenShare: false,
screenShareEncodingUpdate: false,
screenShareAudio: false,
deviceLists: false,
outputDeviceSelection: false,
participantVolume: false,
remoteTrackSubscription: false,
dataChannel: false,
connectionStats: false,
nativeVideoFrames: false,
hardwareEncoderCapabilities: false,
};
}
export function normalizeVoiceEngineV2BridgeCapabilities(value: unknown): VoiceEngineV2BridgeCapabilities {
if (typeof value !== 'object' || value === null) {
return unavailableVoiceEngineV2BridgeCapabilities();
}
const candidate = value as Partial<VoiceEngineV2BridgeCapabilities> & {
microphone?: boolean;
camera?: boolean;
cameraBackgrounds?: boolean;
screen?: boolean;
screenAudio?: boolean;
outputDevice?: boolean;
dataChannel?: boolean;
stats?: boolean;
hardwareEncoding?: boolean;
};
return {
microphoneCapture: candidate.microphoneCapture === true || candidate.microphone === true,
syntheticMicrophonePcm: candidate.syntheticMicrophonePcm === true,
cameraCapture: candidate.cameraCapture === true || candidate.camera === true,
nativeCameraBackgrounds: candidate.nativeCameraBackgrounds === true || candidate.cameraBackgrounds === true,
screenShare: candidate.screenShare === true || candidate.screen === true,
screenShareEncodingUpdate: candidate.screenShareEncodingUpdate === true,
screenShareAudio: candidate.screenShareAudio === true || candidate.screenAudio === true,
deviceLists: candidate.deviceLists === true,
outputDeviceSelection: candidate.outputDeviceSelection === true || candidate.outputDevice === true,
participantVolume: candidate.participantVolume === true,
remoteTrackSubscription: candidate.remoteTrackSubscription === true,
dataChannel: candidate.dataChannel === true,
connectionStats: candidate.connectionStats === true || candidate.stats === true,
nativeVideoFrames: candidate.nativeVideoFrames === true,
hardwareEncoderCapabilities: candidate.hardwareEncoderCapabilities === true || candidate.hardwareEncoding === true,
};
}
export function unavailableVoiceEngineV2BridgeHardwareEncoderCapabilities(
reason: NonNullable<VoiceEngineV2BridgeHardwareEncoderCapabilities['reason']>,
detail?: string,
): VoiceEngineV2BridgeHardwareEncoderCapabilities {
return {
available: false,
backend: 'none',
compiled: false,
runtime: false,
codecs: [],
zeroCopy: false,
nativeInputs: [],
reason,
...(detail ? {detail} : {}),
};
}
export function normalizeVoiceEngineV2BridgeHardwareEncoderCapabilities(
value: unknown,
): VoiceEngineV2BridgeHardwareEncoderCapabilities {
if (typeof value !== 'object' || value === null) {
return unavailableVoiceEngineV2BridgeHardwareEncoderCapabilities(
'query-failed',
'Native addon returned an invalid result',
);
}
const candidate = value as Partial<VoiceEngineV2BridgeHardwareEncoderCapabilities>;
const backend = typeof candidate.backend === 'string' && candidate.backend.length > 0 ? candidate.backend : 'none';
return {
available: candidate.available === true,
backend,
compiled: candidate.compiled === true,
runtime: candidate.runtime === true,
codecs: Array.isArray(candidate.codecs) ? candidate.codecs.filter((codec) => typeof codec === 'string') : [],
zeroCopy: candidate.zeroCopy === true,
nativeInputs: Array.isArray(candidate.nativeInputs)
? candidate.nativeInputs.filter((input) => typeof input === 'string')
: [],
...(typeof candidate.reason === 'string' ? {reason: candidate.reason} : {}),
...(typeof candidate.detail === 'string' ? {detail: candidate.detail} : {}),
};
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null;
}
function isFiniteNumber(value: unknown): value is number {
return typeof value === 'number' && Number.isFinite(value);
}
function isPositiveFiniteNumber(value: unknown): value is number {
return isFiniteNumber(value) && value > 0;
}
function isOptionalPositiveFiniteNumber(value: unknown): value is number | undefined {
return value === undefined || isPositiveFiniteNumber(value);
}
function isOptionalDeepFilterNoiseReductionLevel(value: unknown): value is number | undefined {
if (value === undefined) return true;
return isFiniteNumber(value) && value >= 0 && value <= 100;
}
function isOptionalBoolean(value: unknown): value is boolean | undefined {
return value === undefined || typeof value === 'boolean';
}
function isOptionalString(value: unknown): value is string | undefined {
return value === undefined || typeof value === 'string';
}
function isOptionalCameraBackgroundMode(value: unknown): value is VoiceEngineV2BridgeCameraBackgroundMode | undefined {
return value === undefined || value === 'none' || value === 'non' || value === 'blur' || value === 'custom';
}
export function isVoiceEngineV2BridgeCameraEffectStrength(value: unknown): value is number {
return (
typeof value === 'number' &&
Number.isInteger(value) &&
value >= VOICE_ENGINE_V2_CAMERA_EFFECT_STRENGTH_MIN &&
value <= VOICE_ENGINE_V2_CAMERA_EFFECT_STRENGTH_MAX
);
}
function isOptionalCameraEffectStrength(value: unknown): value is number | undefined {
return value === undefined || isVoiceEngineV2BridgeCameraEffectStrength(value);
}
function isOptionalCameraBackgroundCustomMediaKind(
value: unknown,
): value is VoiceEngineV2BridgeCameraBackgroundCustomMediaKind | undefined {
return value === undefined || value === 'static' || value === 'animated' || value === 'video';
}
function isStringArray(value: unknown): value is Array<string> {
return Array.isArray(value) && value.every((item) => typeof item === 'string');
}
function isOptionalStringArray(value: unknown): value is Array<string> | undefined {
return value === undefined || isStringArray(value);
}
function isArrayBuffer(value: unknown): value is ArrayBuffer {
return value instanceof ArrayBuffer || Object.prototype.toString.call(value) === '[object ArrayBuffer]';
}
function isFloat32Array(value: unknown): value is Float32Array {
return value instanceof Float32Array || Object.prototype.toString.call(value) === '[object Float32Array]';
}
export function isVoiceEngineV2BridgeBinaryPayload(value: unknown): value is ArrayBuffer | ArrayBufferView {
return isArrayBuffer(value) || ArrayBuffer.isView(value);
}
export function getVoiceEngineV2BridgeProcessedCameraI420ByteLength(width: number, height: number): number {
return (width * height * 3) / 2;
}
function isPositiveFiniteInteger(value: unknown): value is number {
return isPositiveFiniteNumber(value) && Number.isInteger(value);
}
function isPositiveEvenFiniteInteger(value: unknown): value is number {
return isPositiveFiniteInteger(value) && value % 2 === 0;
}
function getVoiceEngineV2BridgeBinaryPayloadByteLength(value: ArrayBuffer | ArrayBufferView): number {
return value.byteLength;
}
export function isVoiceEngineV2BridgeRemoteTrackSubscriptionQuality(
value: unknown,
): value is VoiceEngineV2BridgeRemoteTrackSubscriptionQuality {
return value === 'low' || value === 'medium' || value === 'high';
}
export function isVoiceEngineV2BridgeConnectOptions(value: unknown): value is VoiceEngineV2BridgeConnectOptions {
if (!isRecord(value)) return false;
return (
typeof value.url === 'string' &&
value.url.length > 0 &&
typeof value.token === 'string' &&
value.token.length > 0 &&
(value.e2eeKey === undefined || value.e2eeKey === null || isArrayBuffer(value.e2eeKey))
);
}
const VOICE_ENGINE_V2_SCREEN_CODECS = new Set(['', 'vp8', 'vp9', 'h264', 'h265', 'av1']);
const VOICE_ENGINE_V2_SCREEN_PACING_MODES = new Set(['sender', 'source']);
export function isVoiceEngineV2BridgePublishScreenOptions(
value: unknown,
): value is VoiceEngineV2BridgePublishScreenOptions {
if (!isRecord(value)) return false;
return (
typeof value.captureId === 'string' &&
value.captureId.length > 0 &&
isPositiveFiniteNumber(value.width) &&
isPositiveFiniteNumber(value.height) &&
(value.codec === undefined ||
(typeof value.codec === 'string' && VOICE_ENGINE_V2_SCREEN_CODECS.has(value.codec))) &&
isOptionalBoolean(value.hardwareEncoding) &&
isOptionalBoolean(value.zeroCopyRequired) &&
isOptionalPositiveFiniteNumber(value.maxBitrateBps) &&
isOptionalPositiveFiniteNumber(value.maxFramerate) &&
isOptionalBoolean(value.adaptiveSend) &&
isOptionalPositiveFiniteNumber(value.minVideoFps) &&
isOptionalPositiveFiniteNumber(value.maxAudioBufferMs) &&
(value.trackName === undefined || (typeof value.trackName === 'string' && value.trackName.length > 0)) &&
(value.pacing === undefined ||
(typeof value.pacing === 'string' && VOICE_ENGINE_V2_SCREEN_PACING_MODES.has(value.pacing)))
);
}
export function isVoiceEngineV2BridgeUpdateScreenShareEncodingOptions(
value: unknown,
): value is VoiceEngineV2BridgeUpdateScreenShareEncodingOptions {
if (!isRecord(value)) return false;
return (
typeof value.captureId === 'string' &&
value.captureId.length > 0 &&
isPositiveFiniteNumber(value.width) &&
isPositiveFiniteNumber(value.height) &&
isOptionalPositiveFiniteNumber(value.frameRate) &&
isOptionalPositiveFiniteNumber(value.maxBitrateBps) &&
(value.codec === undefined ||
(typeof value.codec === 'string' && VOICE_ENGINE_V2_SCREEN_CODECS.has(value.codec))) &&
isOptionalBoolean(value.hardwareEncoding) &&
isOptionalBoolean(value.zeroCopyRequired)
);
}
export function isVoiceEngineV2BridgePublishMicrophoneOptions(
value: unknown,
): value is VoiceEngineV2BridgePublishMicrophoneOptions {
if (!isRecord(value)) return false;
const mode = value.mode ?? 'device';
const pcmTestValuesAreValid =
mode !== 'pcm-test' || (isPositiveFiniteNumber(value.sampleRate) && isPositiveFiniteNumber(value.numChannels));
return (
(mode === 'device' || mode === 'pcm-test') &&
isOptionalString(value.deviceId) &&
isOptionalBoolean(value.echoCancellation) &&
isOptionalBoolean(value.noiseSuppression) &&
isOptionalBoolean(value.autoGainControl) &&
isOptionalBoolean(value.deepFilter) &&
isOptionalDeepFilterNoiseReductionLevel(value.deepFilterNoiseReductionLevel) &&
isOptionalPositiveFiniteNumber(value.sampleRate) &&
isOptionalPositiveFiniteNumber(value.numChannels) &&
isOptionalPositiveFiniteNumber(value.maxBitrateBps) &&
pcmTestValuesAreValid
);
}
export function isVoiceEngineV2BridgePcmFrame(value: unknown): value is VoiceEngineV2BridgePcmFrame {
if (!isRecord(value)) return false;
return (
isPositiveFiniteNumber(value.sampleRate) &&
isPositiveFiniteNumber(value.numChannels) &&
isArrayBuffer(value.samples)
);
}
export function isVoiceEngineV2BridgeFloatPcmFrame(value: unknown): value is VoiceEngineV2BridgeFloatPcmFrame {
if (!isRecord(value)) return false;
return (
isPositiveFiniteNumber(value.sampleRate) &&
isPositiveFiniteNumber(value.numChannels) &&
isFloat32Array(value.samples)
);
}
export function isVoiceEngineV2BridgePublishScreenAudioOptions(
value: unknown,
): value is VoiceEngineV2BridgePublishScreenAudioOptions {
if (!isRecord(value)) return false;
return (
isPositiveFiniteNumber(value.sampleRate) &&
isPositiveFiniteNumber(value.numChannels) &&
(value.route === undefined || value.route === 'browser' || value.route === 'native') &&
isOptionalString(value.captureId) &&
isOptionalString(value.tapId)
);
}
export function isVoiceEngineV2ParticipantVolumeOptions(
value: unknown,
): value is VoiceEngineV2BridgeParticipantVolumeOptions {
if (!isRecord(value)) return false;
return typeof value.participantSid === 'string' && value.participantSid.length > 0 && isFiniteNumber(value.volume);
}
export function clampVoiceEngineV2ParticipantVolume(volume: number): number {
if (!Number.isFinite(volume)) return 1;
return Math.max(0, Math.min(2, volume));
}
export function isVoiceEngineV2BridgeSpeakingDetectionOptions(
value: unknown,
): value is VoiceEngineV2BridgeSpeakingDetectionOptions {
if (!isRecord(value)) return false;
return isPositiveFiniteNumber(value.localThresholdRms) && isPositiveFiniteNumber(value.remoteThresholdRms);
}
export function isVoiceEngineV2BridgeRemoteTrackSubscriptionOptions(
value: unknown,
): value is VoiceEngineV2BridgeRemoteTrackSubscriptionOptions {
if (!isRecord(value)) return false;
return (
typeof value.participantIdentity === 'string' &&
value.participantIdentity.length > 0 &&
typeof value.source === 'string' &&
value.source.length > 0 &&
typeof value.subscribed === 'boolean' &&
isOptionalBoolean(value.enabled) &&
(value.quality === undefined || isVoiceEngineV2BridgeRemoteTrackSubscriptionQuality(value.quality))
);
}
export function isVoiceEngineV2BridgePublishDataOptions(
value: unknown,
): value is VoiceEngineV2BridgePublishDataOptions {
if (!isRecord(value)) return false;
return (
isVoiceEngineV2BridgeBinaryPayload(value.payload) &&
isOptionalBoolean(value.reliable) &&
isOptionalString(value.topic) &&
isOptionalStringArray(value.destinationIdentities)
);
}
export function isVoiceEngineV2BridgePublishCameraOptions(
value: unknown,
): value is VoiceEngineV2BridgePublishCameraOptions {
if (!isRecord(value)) return false;
return (
isOptionalString(value.deviceId) &&
isOptionalPositiveFiniteNumber(value.width) &&
isOptionalPositiveFiniteNumber(value.height) &&
isOptionalPositiveFiniteNumber(value.frameRate) &&
isOptionalBoolean(value.mirror) &&
isOptionalCameraBackgroundMode(value.backgroundMode) &&
isOptionalString(value.backgroundCustomMediaPath) &&
isOptionalCameraBackgroundCustomMediaKind(value.backgroundCustomMediaKind) &&
isOptionalCameraEffectStrength(value.backgroundBlurStrength)
);
}
export function isVoiceEngineV2BridgePublishProcessedCameraOptions(
value: unknown,
): value is VoiceEngineV2BridgePublishProcessedCameraOptions {
if (!isRecord(value)) return false;
return (
isPositiveEvenFiniteInteger(value.width) &&
isPositiveEvenFiniteInteger(value.height) &&
isPositiveFiniteInteger(value.frameRate)
);
}
export function isVoiceEngineV2BridgePublishProcessedCameraResult(
value: unknown,
): value is VoiceEngineV2BridgePublishProcessedCameraResult {
if (!isRecord(value)) return false;
return typeof value.trackSid === 'string' && value.trackSid.length > 0;
}
export function isVoiceEngineV2BridgeCameraPreviewInfo(value: unknown): value is VoiceEngineV2BridgeCameraPreviewInfo {
if (!isRecord(value)) return false;
return (
typeof value.trackSid === 'string' &&
value.trackSid.length > 0 &&
isPositiveEvenFiniteInteger(value.width) &&
isPositiveEvenFiniteInteger(value.height) &&
isPositiveFiniteInteger(value.frameRate)
);
}
export function isVoiceEngineV2BridgeProcessedCameraFrame(
value: unknown,
): value is VoiceEngineV2BridgeProcessedCameraFrame {
if (!isRecord(value)) return false;
if (value.format !== 'i420') return false;
if (!isPositiveEvenFiniteInteger(value.width)) return false;
if (!isPositiveEvenFiniteInteger(value.height)) return false;
if (!isPositiveFiniteInteger(value.timestampUs)) return false;
if (!isVoiceEngineV2BridgeBinaryPayload(value.data)) return false;
const expectedBytes = getVoiceEngineV2BridgeProcessedCameraI420ByteLength(value.width, value.height);
return getVoiceEngineV2BridgeBinaryPayloadByteLength(value.data) === expectedBytes;
}
export function isVoiceEngineV2BridgeReadiness(value: unknown): value is VoiceEngineV2BridgeReadiness {
if (!isRecord(value)) return false;
if (typeof value.ready !== 'boolean') return false;
if (value.ready) return value.reason === undefined;
return isOptionalString(value.reason);
}
export function isVoiceEngineV2BridgePublishDeviceScreenShareOptions(
value: unknown,
): value is VoiceEngineV2BridgePublishDeviceScreenShareOptions {
if (!isRecord(value)) return false;
return (
isOptionalString(value.deviceId) &&
isOptionalPositiveFiniteNumber(value.width) &&
isOptionalPositiveFiniteNumber(value.height) &&
isOptionalPositiveFiniteNumber(value.frameRate) &&
(value.codec === undefined ||
(typeof value.codec === 'string' && VOICE_ENGINE_V2_SCREEN_CODECS.has(value.codec))) &&
isOptionalPositiveFiniteNumber(value.maxBitrateBps) &&
isOptionalPositiveFiniteNumber(value.maxFramerate)
);
}
+74 -5
View File
@@ -1,7 +1,76 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
export * from './admWarmup';
export * from './guards';
export * from './stats';
export * from './translate';
export * from './types';
import type {VoiceEngineV2HardwareEncoderCapabilities} from '../protocol';
import {assertAudioFrameInvariants, assertVideoFrameInvariants} from './ffi_assertions';
export const VOICE_ENGINE_V2_HARDWARE_ENCODER_IPC_CHANNEL = 'voice-engine-v2:get-hardware-encoder-capabilities';
export type VoiceEngineV2BridgeHardwareEncoderCapabilities = VoiceEngineV2HardwareEncoderCapabilities;
export interface VoiceEngineV2BridgeHardwareEncoderApi {
getHardwareEncoderCapabilities(): Promise<VoiceEngineV2BridgeHardwareEncoderCapabilities>;
}
export function assertVoiceEngineV2BridgeAudioOptionsInvariants(options: {
sampleRate: number;
numChannels: number;
}): void {
assertAudioFrameInvariants({
sampleRateHz: options.sampleRate,
numChannels: options.numChannels,
frameBytes: 1,
timestampNs: 1,
});
}
export function assertVoiceEngineV2BridgeVideoOptionsInvariants(options: {width: number; height: number}): void {
assertVideoFrameInvariants({
widthPx: options.width,
heightPx: options.height,
frameBytes: 1,
timestampNs: 1,
});
}
export function unavailableVoiceEngineV2BridgeHardwareEncoderCapabilities(
reason: NonNullable<VoiceEngineV2BridgeHardwareEncoderCapabilities['reason']>,
detail?: string,
): VoiceEngineV2BridgeHardwareEncoderCapabilities {
return {
available: false,
backend: 'none',
compiled: false,
runtime: false,
codecs: [],
zeroCopy: false,
nativeInputs: [],
reason,
...(detail ? {detail} : {}),
};
}
export function normalizeVoiceEngineV2BridgeHardwareEncoderCapabilities(
value: unknown,
): VoiceEngineV2BridgeHardwareEncoderCapabilities {
if (typeof value !== 'object' || value === null) {
return unavailableVoiceEngineV2BridgeHardwareEncoderCapabilities(
'query-failed',
'Native addon returned an invalid result',
);
}
const candidate = value as Partial<VoiceEngineV2BridgeHardwareEncoderCapabilities>;
const backend = typeof candidate.backend === 'string' && candidate.backend.length > 0 ? candidate.backend : 'none';
return {
available: candidate.available === true,
backend,
compiled: candidate.compiled === true,
runtime: candidate.runtime === true,
codecs: Array.isArray(candidate.codecs) ? candidate.codecs.filter((codec) => typeof codec === 'string') : [],
zeroCopy: candidate.zeroCopy === true,
nativeInputs: Array.isArray(candidate.nativeInputs)
? candidate.nativeInputs.filter((input) => typeof input === 'string')
: [],
...(typeof candidate.reason === 'string' ? {reason: candidate.reason} : {}),
...(typeof candidate.detail === 'string' ? {detail: candidate.detail} : {}),
};
}
@@ -1,141 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {
VoiceEngineV2BridgeInboundStat,
VoiceEngineV2BridgeOutboundStat,
VoiceEngineV2BridgeSendStats,
VoiceEngineV2BridgeStats,
VoiceEngineV2BridgeTrackKind,
} from './types';
function asTrackKind(value: unknown): VoiceEngineV2BridgeTrackKind {
return value === 'video' ? 'video' : 'audio';
}
function asNumber(value: unknown): number {
return typeof value === 'number' && Number.isFinite(value) ? value : 0;
}
function asNonNegativeNumber(value: unknown): number {
return Math.max(0, asNumber(value));
}
function asOptionalNumber(value: unknown): number | undefined {
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
}
function asOptionalCount(value: unknown): number | undefined {
const number = asOptionalNumber(value);
return number == null ? undefined : Math.max(0, Math.trunc(number));
}
function asString(value: unknown): string | undefined {
return typeof value === 'string' && value.length > 0 ? value : undefined;
}
function coerceVoiceEngineV2BridgeSendStats(value: unknown): VoiceEngineV2BridgeSendStats | null | undefined {
if (value == null || typeof value !== 'object') return value === null ? null : undefined;
const payload = value as Record<string, unknown>;
return {
outgoingVideoQueueDepth: asNonNegativeNumber(payload.outgoingVideoQueueDepth),
outgoingVideoQueueCapacity: asNonNegativeNumber(payload.outgoingVideoQueueCapacity),
outgoingVideoMaxQueueDepth: asNonNegativeNumber(payload.outgoingVideoMaxQueueDepth),
outgoingVideoFramesProduced: asNonNegativeNumber(payload.outgoingVideoFramesProduced),
outgoingVideoFramesAccepted: asNonNegativeNumber(payload.outgoingVideoFramesAccepted),
outgoingVideoFramesDropped: asNonNegativeNumber(payload.outgoingVideoFramesDropped),
outgoingVideoFramesCoalesced: asNonNegativeNumber(payload.outgoingVideoFramesCoalesced),
outgoingVideoFramesCaptured: asNonNegativeNumber(payload.outgoingVideoFramesCaptured),
outgoingVideoCaptureFailures: asNonNegativeNumber(payload.outgoingVideoCaptureFailures),
outgoingVideoEffectiveFps: asNonNegativeNumber(payload.outgoingVideoEffectiveFps),
outgoingVideoTargetFps: asNonNegativeNumber(payload.outgoingVideoTargetFps),
outgoingVideoPacingTargetFps: asNonNegativeNumber(payload.outgoingVideoPacingTargetFps),
outgoingVideoMaxQueueAgeMs: asNonNegativeNumber(payload.outgoingVideoMaxQueueAgeMs),
outgoingVideoMaxPushLatencyMs: asNonNegativeNumber(payload.outgoingVideoMaxPushLatencyMs),
outgoingVideoPacingMode: asString(payload.outgoingVideoPacingMode) ?? 'unknown',
outgoingVideoBusActive: payload.outgoingVideoBusActive === true,
outgoingAudioBufferTargetMs: asNonNegativeNumber(payload.outgoingAudioBufferTargetMs),
outgoingAudioBufferMaxMs: asNonNegativeNumber(payload.outgoingAudioBufferMaxMs),
outgoingAudioUnderruns: asNonNegativeNumber(payload.outgoingAudioUnderruns),
outgoingAudioRebuffers: asNonNegativeNumber(payload.outgoingAudioRebuffers),
outgoingAudioMaxFrameGapMs: asNonNegativeNumber(payload.outgoingAudioMaxFrameGapMs),
adaptiveSendTier: asString(payload.adaptiveSendTier) ?? 'unknown',
adaptiveSendReason: asString(payload.adaptiveSendReason) ?? 'unknown',
};
}
function coerceVoiceEngineV2BridgeOutboundStat(entry: unknown): Array<VoiceEngineV2BridgeOutboundStat> {
if (entry == null || typeof entry !== 'object') return [];
const e = entry as Record<string, unknown>;
const zeroCopy = e.zeroCopy === true ? true : undefined;
return [
{
trackSid: asString(e.trackSid) ?? '',
source: asString(e.source) ?? '',
kind: asTrackKind(e.kind),
codec: asString(e.codec),
bitrateKbps: asNumber(e.bitrateKbps),
packetsLost: asNumber(e.packetsLost),
fps: asOptionalNumber(e.fps),
audioLevel: asOptionalNumber(e.audioLevel),
width: asOptionalCount(e.width),
height: asOptionalCount(e.height),
sourceWidth: asOptionalCount(e.sourceWidth),
sourceHeight: asOptionalCount(e.sourceHeight),
targetBitrateKbps: asOptionalNumber(e.targetBitrateKbps),
configuredFps: asOptionalNumber(e.configuredFps),
targetFps: asOptionalNumber(e.targetFps),
effectiveFps: asOptionalNumber(e.effectiveFps),
framesProduced: asOptionalCount(e.framesProduced),
framesAccepted: asOptionalCount(e.framesAccepted),
framesDropped: asOptionalCount(e.framesDropped),
framesCoalesced: asOptionalCount(e.framesCoalesced),
framesCaptured: asOptionalCount(e.framesCaptured),
captureFailures: asOptionalCount(e.captureFailures),
maxQueueAgeMs: asOptionalNumber(e.maxQueueAgeMs),
maxPushLatencyMs: asOptionalNumber(e.maxPushLatencyMs),
adaptiveSendTier: asString(e.adaptiveSendTier),
adaptiveSendReason: asString(e.adaptiveSendReason),
...(zeroCopy === true ? {zeroCopy} : {}),
},
];
}
function coerceVoiceEngineV2BridgeInboundStat(entry: unknown): Array<VoiceEngineV2BridgeInboundStat> {
if (entry == null || typeof entry !== 'object') return [];
const e = entry as Record<string, unknown>;
const participantIdentity = asString(e.participantIdentity);
return [
{
...(participantIdentity ? {participantIdentity} : {}),
participantSid: asString(e.participantSid) ?? '',
trackSid: asString(e.trackSid) ?? '',
source: asString(e.source),
kind: asTrackKind(e.kind),
codec: asString(e.codec),
bitrateKbps: asNumber(e.bitrateKbps),
packetsLost: asNumber(e.packetsLost),
jitterMs: asOptionalNumber(e.jitterMs),
audioLevel: asOptionalNumber(e.audioLevel),
width: asOptionalCount(e.width),
height: asOptionalCount(e.height),
fps: asOptionalNumber(e.fps),
sourceWidth: asOptionalCount(e.sourceWidth),
sourceHeight: asOptionalCount(e.sourceHeight),
},
];
}
export function coerceVoiceEngineV2BridgeStats(payload: Record<string, unknown>): VoiceEngineV2BridgeStats {
const rttRaw = payload.rttMs;
const rttMs = typeof rttRaw === 'number' && Number.isFinite(rttRaw) ? rttRaw : null;
const droppedNativeVideoFrames = asOptionalCount(payload.droppedNativeVideoFrames);
const droppedVideoFrameCallbacks = asOptionalCount(payload.droppedVideoFrameCallbacks);
const outbound = Array.isArray(payload.outbound)
? payload.outbound.flatMap((entry) => coerceVoiceEngineV2BridgeOutboundStat(entry))
: [];
const inbound = Array.isArray(payload.inbound)
? payload.inbound.flatMap((entry) => coerceVoiceEngineV2BridgeInboundStat(entry))
: [];
const send = coerceVoiceEngineV2BridgeSendStats(payload.send);
return {rttMs, outbound, inbound, droppedNativeVideoFrames, droppedVideoFrameCallbacks, send};
}
@@ -1,310 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {
VoiceEngineV2Event,
VoiceEngineV2NativeAudioDeviceModuleStatus,
VoiceEngineV2TrackKind,
VoiceEngineV2TrackSource,
} from '../protocol';
import {assertVideoFrameInvariants} from './ffi_assertions';
import type {VoiceEngineV2BridgeEvent, VoiceEngineV2BridgeKnownEventType, VoiceEngineV2BridgeVideoFrame} from './types';
type VoiceEngineV2RoomTrackPublishedEvent = Extract<VoiceEngineV2Event, {type: 'room.trackPublished'}>;
type VoiceEngineV2BridgeRecord = Record<string, unknown>;
function bridgeString(value: unknown): string | null {
return typeof value === 'string' && value.length > 0 ? value : null;
}
function bridgeBoolean(value: unknown): boolean {
return value === true;
}
function bridgeAudioDeviceModuleStatus(value: unknown): VoiceEngineV2NativeAudioDeviceModuleStatus | null {
if (value === 'warming' || value === 'ready' || value === 'failed') return value;
return null;
}
function normalizeVoiceEngineV2BridgeTrackSource(value: unknown): VoiceEngineV2TrackSource | string {
switch (value) {
case 'microphone':
return 'microphone';
case 'camera':
return 'camera';
case 'screen':
case 'screen_share':
case 'screenshare':
return 'screen';
case 'screenAudio':
case 'screen_share_audio':
case 'screenshareAudio':
case 'screenshare_audio':
case 'screen_audio':
case 'system_audio':
return 'screenAudio';
default:
return 'unknown';
}
}
function normalizeVoiceEngineV2BridgeTrackKind(value: unknown): VoiceEngineV2TrackKind | null {
return value === 'audio' || value === 'video' ? value : null;
}
function participantEventFromBridgePayload(payload: VoiceEngineV2BridgeRecord): VoiceEngineV2Event | null {
const identity = bridgeString(payload.identity);
const sid = bridgeString(payload.sid);
if (!identity || !sid) return null;
return {
type: 'room.participantJoined',
participant: {
identity,
sid,
name: bridgeString(payload.name) ?? identity,
},
};
}
function trackEventFromBridgePayload(payload: VoiceEngineV2BridgeRecord): VoiceEngineV2RoomTrackPublishedEvent | null {
const participantIdentity = bridgeString(payload.identity) ?? bridgeString(payload.participantIdentity);
const participantSid = bridgeString(payload.participantSid);
const trackSid = bridgeString(payload.trackSid);
const kind = normalizeVoiceEngineV2BridgeTrackKind(payload.kind);
if (!participantIdentity || !participantSid || !trackSid || !kind) return null;
const source = normalizeVoiceEngineV2BridgeTrackSource(payload.source);
return {
type: 'room.trackPublished',
track: {
participantIdentity,
participantSid,
trackSid,
trackName: bridgeString(payload.trackName) ?? String(source),
kind,
source,
muted: bridgeBoolean(payload.muted),
},
};
}
function nativeAudioDeviceModuleStatusEventFromBridgePayload(
payload: VoiceEngineV2BridgeRecord,
): VoiceEngineV2Event | null {
const status = bridgeAudioDeviceModuleStatus(payload.status);
if (!status) return null;
const detail = bridgeString(payload.detail);
return {
type: 'nativeAudioDeviceModule.statusChanged',
status,
...(detail ? {detail} : {}),
};
}
const VOICE_ENGINE_V2_BRIDGE_KNOWN_EVENT_TYPE_FLAGS: {[Type in VoiceEngineV2BridgeKnownEventType]: true} = {
connected: true,
connectionState: true,
disconnected: true,
participantJoined: true,
participantLeft: true,
participantNameChanged: true,
participantMetadataChanged: true,
participantAttributesChanged: true,
trackPublished: true,
trackUnpublished: true,
trackSubscribed: true,
trackUnsubscribed: true,
trackSubscriptionFailed: true,
trackMuted: true,
trackUnmuted: true,
localTrackPublished: true,
localTrackUnpublished: true,
localTrackRepublished: true,
activeSpeakers: true,
speakingChanged: true,
connectionQuality: true,
dataReceived: true,
e2eeState: true,
stats: true,
audioPlaybackUnavailable: true,
engineReady: true,
audioDeviceModuleStatus: true,
};
const VOICE_ENGINE_V2_BRIDGE_KNOWN_EVENT_TYPE_SET: ReadonlySet<string> = new Set(
Object.keys(VOICE_ENGINE_V2_BRIDGE_KNOWN_EVENT_TYPE_FLAGS),
);
export function isVoiceEngineV2BridgeKnownEventType(value: string): value is VoiceEngineV2BridgeKnownEventType {
return VOICE_ENGINE_V2_BRIDGE_KNOWN_EVENT_TYPE_SET.has(value);
}
export const VOICE_ENGINE_V2_BRIDGE_DROPPED_EVENTS_COUNT_MAX = 1_000_000_000;
export interface VoiceEngineV2BridgeDroppedEventCounts {
malformedPayloadCount: number;
unknownTypeCount: number;
}
let bridgeMalformedPayloadCount = 0;
let bridgeUnknownTypeCount = 0;
function saturatingDroppedEventCount(count: number): number {
return Math.min(count + 1, VOICE_ENGINE_V2_BRIDGE_DROPPED_EVENTS_COUNT_MAX);
}
export function getVoiceEngineV2BridgeDroppedEventCounts(): VoiceEngineV2BridgeDroppedEventCounts {
return {
malformedPayloadCount: bridgeMalformedPayloadCount,
unknownTypeCount: bridgeUnknownTypeCount,
};
}
export function resetVoiceEngineV2BridgeDroppedEventCounts(): void {
bridgeMalformedPayloadCount = 0;
bridgeUnknownTypeCount = 0;
}
type VoiceEngineV2BridgeTranslationResult = Array<VoiceEngineV2Event> | 'ignored' | 'malformed';
export function translateVoiceEngineV2BridgeEventToEvents(event: VoiceEngineV2BridgeEvent): Array<VoiceEngineV2Event> {
if (!isVoiceEngineV2BridgeKnownEventType(event.type)) {
bridgeUnknownTypeCount = saturatingDroppedEventCount(bridgeUnknownTypeCount);
return [];
}
const payload = event.payload as VoiceEngineV2BridgeRecord;
const translated = translateKnownVoiceEngineV2BridgeEvent(event.type, payload);
if (translated === 'ignored') return [];
if (translated === 'malformed') {
bridgeMalformedPayloadCount = saturatingDroppedEventCount(bridgeMalformedPayloadCount);
return [];
}
return translated;
}
function translateKnownVoiceEngineV2BridgeEvent(
type: VoiceEngineV2BridgeKnownEventType,
payload: VoiceEngineV2BridgeRecord,
): VoiceEngineV2BridgeTranslationResult {
switch (type) {
case 'participantJoined':
case 'participantNameChanged':
case 'participantMetadataChanged':
case 'participantAttributesChanged': {
const participantEvent = participantEventFromBridgePayload(payload);
return participantEvent ? [participantEvent] : 'malformed';
}
case 'participantLeft':
return participantLeftEventsFromBridgePayload(payload);
case 'trackPublished':
case 'localTrackPublished': {
const trackEvent = trackEventFromBridgePayload(payload);
return trackEvent ? [trackEvent] : 'malformed';
}
case 'trackSubscribed':
return trackSubscribedEventsFromBridgePayload(payload);
case 'trackUnpublished':
case 'localTrackUnpublished': {
const trackSid = bridgeString(payload.trackSid);
return trackSid
? [
{type: 'room.trackUnpublished', trackSid},
{type: 'inboundVideo.trackUnsubscribed', trackSid},
]
: 'malformed';
}
case 'trackUnsubscribed':
case 'trackSubscriptionFailed': {
const trackSid = bridgeString(payload.trackSid);
return trackSid ? [{type: 'inboundVideo.trackUnsubscribed', trackSid}] : 'malformed';
}
case 'trackMuted': {
const trackSid = bridgeString(payload.trackSid);
return trackSid ? [{type: 'room.trackMuted', trackSid}] : 'malformed';
}
case 'trackUnmuted': {
const trackSid = bridgeString(payload.trackSid);
return trackSid ? [{type: 'room.trackUnmuted', trackSid}] : 'malformed';
}
case 'activeSpeakers':
return Array.isArray(payload.participants)
? payload.participants.flatMap((participant) =>
typeof participant === 'object' && participant !== null
? (participantEventFromBridgePayload(participant as VoiceEngineV2BridgeRecord) ?? [])
: [],
)
: 'malformed';
case 'connected':
case 'connectionState':
case 'disconnected':
case 'localTrackRepublished':
case 'speakingChanged':
case 'connectionQuality':
case 'dataReceived':
case 'e2eeState':
case 'stats':
case 'audioPlaybackUnavailable':
case 'engineReady':
return 'ignored';
case 'audioDeviceModuleStatus': {
const event = nativeAudioDeviceModuleStatusEventFromBridgePayload(payload);
return event ? [event] : 'malformed';
}
}
}
function participantLeftEventsFromBridgePayload(
payload: VoiceEngineV2BridgeRecord,
): VoiceEngineV2BridgeTranslationResult {
const participantIdentity = bridgeString(payload.identity);
const participantSid = bridgeString(payload.sid);
if (!participantIdentity && !participantSid) return 'malformed';
return [
{
type: 'room.participantLeft',
...(participantIdentity ? {participantIdentity} : {}),
...(participantSid ? {participantSid} : {}),
},
];
}
function trackSubscribedEventsFromBridgePayload(
payload: VoiceEngineV2BridgeRecord,
): VoiceEngineV2BridgeTranslationResult {
const trackEvent = trackEventFromBridgePayload(payload);
if (!trackEvent) return 'malformed';
if (trackEvent.track.kind !== 'video') return [trackEvent];
return [
trackEvent,
{
type: 'inboundVideo.trackSubscribed',
track: {
participantSid: trackEvent.track.participantSid,
participantIdentity: trackEvent.track.participantIdentity,
trackSid: trackEvent.track.trackSid,
source: trackEvent.track.source,
},
},
];
}
export function translateVoiceEngineV2BridgeVideoFrameToEvent(
frame: VoiceEngineV2BridgeVideoFrame,
): VoiceEngineV2Event {
assertVideoFrameInvariants({
widthPx: frame.meta.width,
heightPx: frame.meta.height,
frameBytes: frame.data.byteLength,
timestampNs: frame.meta.timestampUs * 1000,
});
return {
type: 'inboundVideo.frameReceived',
frame: {
participantSid: frame.meta.participantSid,
trackSid: frame.meta.trackSid,
width: frame.meta.width,
height: frame.meta.height,
timestampUs: frame.meta.timestampUs,
byteLength: frame.data.byteLength,
},
};
}
@@ -1,680 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
// biome-ignore-all lint/suspicious/noConfusingVoidType: IPC result metadata mirrors void-returning bridge methods.
import type {
VoiceEngineV2CameraOptions,
VoiceEngineV2ConnectOptions,
VoiceEngineV2DataOptions,
VoiceEngineV2HardwareEncoderCapabilities,
VoiceEngineV2PcmFrame,
VoiceEngineV2RemoteTrackQuality,
VoiceEngineV2RemoteTrackSubscriptionOptions,
VoiceEngineV2ScreenAudioOptions,
VoiceEngineV2ScreenOptions,
VoiceEngineV2SendStats,
VoiceEngineV2TrackKind,
VoiceEngineV2TrackSource,
VoiceEngineV2VideoCodec,
} from '../protocol';
export const VOICE_ENGINE_V2_BRIDGE_VERSION = 18;
export type VoiceEngineV2BridgeMethodName =
| 'isSupported'
| 'getCapabilities'
| 'prewarm'
| 'getHardwareEncoderCapabilities'
| 'connect'
| 'disconnect'
| 'isConnected'
| 'publishMicrophone'
| 'pushPcm'
| 'publishScreen'
| 'updateScreenShareEncoding'
| 'unpublishScreen'
| 'publishScreenAudio'
| 'pushScreenAudioPcm'
| 'pushScreenAudioFloat'
| 'unpublishScreenAudio'
| 'setMicEnabled'
| 'setSpeakingDetection'
| 'listAudioInputDevices'
| 'listAudioOutputDevices'
| 'setAudioOutputDevice'
| 'setParticipantVolume'
| 'setRemoteTrackSubscription'
| 'publishData'
| 'listCameraDevices'
| 'publishCamera'
| 'updateCameraCapture'
| 'publishNativeCameraSink'
| 'publishProcessedCamera'
| 'pushProcessedCameraFrame'
| 'pushCameraBackgroundFrame'
| 'clearCameraBackgroundFrame'
| 'publishDeviceScreenShare'
| 'unpublishCamera'
| 'isPublishingCamera'
| 'startCameraPreview'
| 'stopCameraPreview'
| 'getConnectionStats'
| 'getVoiceEngineReadiness'
| 'getAudioDeviceModuleState'
| 'onEvent'
| 'onVideoFrame';
export type VoiceEngineV2InvokeMethodName = Exclude<VoiceEngineV2BridgeMethodName, 'onEvent' | 'onVideoFrame'>;
export type VoiceEngineV2BridgeConnectOptions = VoiceEngineV2ConnectOptions;
export type VoiceEngineV2BridgeScreenPacing = 'sender' | 'source';
export interface VoiceEngineV2BridgePublishScreenOptions extends VoiceEngineV2ScreenOptions {
codec?: VoiceEngineV2VideoCodec;
pacing?: VoiceEngineV2BridgeScreenPacing;
trackName?: string;
}
export interface VoiceEngineV2BridgeUpdateScreenShareEncodingOptions {
captureId: string;
width: number;
height: number;
frameRate?: number;
maxBitrateBps?: number;
codec?: VoiceEngineV2VideoCodec;
hardwareEncoding?: boolean;
zeroCopyRequired?: boolean;
}
export interface VoiceEngineV2BridgePublishMicrophoneOptions {
mode?: 'device' | 'pcm-test';
deviceId?: string;
echoCancellation?: boolean;
noiseSuppression?: boolean;
autoGainControl?: boolean;
deepFilter?: boolean;
deepFilterNoiseReductionLevel?: number;
sampleRate?: number;
numChannels?: number;
maxBitrateBps?: number;
}
export type VoiceEngineV2BridgePcmFrame = VoiceEngineV2PcmFrame;
export interface VoiceEngineV2BridgeFloatPcmFrame {
sampleRate: number;
numChannels: number;
samples: Float32Array;
}
export type VoiceEngineV2BridgePublishScreenAudioOptions = VoiceEngineV2ScreenAudioOptions;
export type VoiceEngineV2BridgePublishDataOptions = VoiceEngineV2DataOptions;
export type VoiceEngineV2BridgeTrackKind = VoiceEngineV2TrackKind;
export type VoiceEngineV2BridgeTrackSource =
| VoiceEngineV2TrackSource
| 'screen_share'
| 'screen_share_audio'
| 'screenshare'
| 'screenshareAudio';
export type VoiceEngineV2BridgeSubscriptionStatus = 'desired' | 'subscribed' | 'unsubscribed';
export type VoiceEngineV2BridgeConnectionQuality = 'excellent' | 'good' | 'poor' | 'lost';
export interface VoiceEngineV2BridgeParticipantEventPayload {
sid: string;
identity: string;
name: string;
}
export interface VoiceEngineV2BridgeTrackEventPayload {
participantSid: string;
identity: string;
participantName: string;
trackSid: string;
trackName: string;
kind: VoiceEngineV2BridgeTrackKind;
source: VoiceEngineV2BridgeTrackSource;
muted: boolean;
}
export interface VoiceEngineV2BridgeSubscribedTrackEventPayload extends VoiceEngineV2BridgeTrackEventPayload {
subscribed: boolean;
subscriptionStatus: VoiceEngineV2BridgeSubscriptionStatus;
}
export interface VoiceEngineV2BridgeTrackSubscriptionFailedEventPayload {
participantSid: string;
identity: string;
participantName: string;
trackSid: string;
error: string;
trackName?: string;
kind?: VoiceEngineV2BridgeTrackKind;
source?: VoiceEngineV2BridgeTrackSource;
muted?: boolean;
subscribed?: boolean;
subscriptionStatus?: VoiceEngineV2BridgeSubscriptionStatus;
}
export interface VoiceEngineV2BridgeLocalTrackRepublishedEventPayload extends VoiceEngineV2BridgeTrackEventPayload {
previousTrackSid: string;
}
export interface VoiceEngineV2BridgeEventPayloads {
connected: Record<string, never>;
connectionState: {state: string};
disconnected: {reason: string};
participantJoined: VoiceEngineV2BridgeParticipantEventPayload;
participantLeft: VoiceEngineV2BridgeParticipantEventPayload;
participantNameChanged: {sid: string; identity: string; oldName: string; name: string};
participantMetadataChanged: {
sid: string;
identity: string;
name: string;
oldMetadata: string;
metadata: string;
attributes: Record<string, string>;
};
participantAttributesChanged: {
sid: string;
identity: string;
name: string;
attributes: Record<string, string>;
changedAttributes: Record<string, string>;
};
trackPublished: VoiceEngineV2BridgeSubscribedTrackEventPayload;
trackUnpublished: VoiceEngineV2BridgeSubscribedTrackEventPayload;
trackSubscribed: VoiceEngineV2BridgeSubscribedTrackEventPayload;
trackUnsubscribed: VoiceEngineV2BridgeSubscribedTrackEventPayload;
trackSubscriptionFailed: VoiceEngineV2BridgeTrackSubscriptionFailedEventPayload;
trackMuted: VoiceEngineV2BridgeTrackEventPayload;
trackUnmuted: VoiceEngineV2BridgeTrackEventPayload;
localTrackPublished: VoiceEngineV2BridgeTrackEventPayload;
localTrackUnpublished: VoiceEngineV2BridgeTrackEventPayload;
localTrackRepublished: VoiceEngineV2BridgeLocalTrackRepublishedEventPayload;
activeSpeakers: {
sids: Array<string>;
participants: Array<VoiceEngineV2BridgeParticipantEventPayload>;
};
speakingChanged: {
participantSid: string;
identity: string;
trackSid: string;
source: VoiceEngineV2BridgeTrackSource;
isLocal: boolean;
speaking: boolean;
};
connectionQuality: {
sid: string;
identity: string;
name: string;
quality: VoiceEngineV2BridgeConnectionQuality;
};
dataReceived: {
payloadBytes: Array<number>;
payloadText?: string;
topic?: string;
reliable: boolean;
kind: 'reliable' | 'lossy';
participantSid?: string;
identity?: string;
participantName?: string;
};
e2eeState: {sid: string; identity: string; name: string; state: string};
stats: VoiceEngineV2BridgeStats;
audioPlaybackUnavailable: {message: string};
engineReady: {ready: true};
audioDeviceModuleStatus: VoiceEngineV2BridgeAudioDeviceModuleState;
}
export const VOICE_ENGINE_V2_ENGINE_READY_EVENT_TYPE = 'engineReady';
export const VOICE_ENGINE_V2_ADM_STATUS_EVENT_TYPE = 'audioDeviceModuleStatus';
export interface VoiceEngineV2BridgeReadiness {
ready: boolean;
reason?: string;
}
export type VoiceEngineV2BridgeAudioDeviceModuleStatus = 'warming' | 'ready' | 'failed';
export interface VoiceEngineV2BridgeAudioDeviceModuleState {
status: VoiceEngineV2BridgeAudioDeviceModuleStatus;
detail?: string;
}
export type VoiceEngineV2BridgeKnownEventType = keyof VoiceEngineV2BridgeEventPayloads;
export type VoiceEngineV2BridgeEventType = VoiceEngineV2BridgeKnownEventType | (string & {});
export type VoiceEngineV2BridgeKnownEvent = {
[Type in VoiceEngineV2BridgeKnownEventType]: {type: Type; payload: VoiceEngineV2BridgeEventPayloads[Type]};
}[VoiceEngineV2BridgeKnownEventType];
export type VoiceEngineV2BridgeEvent =
| VoiceEngineV2BridgeKnownEvent
| {type: string & {}; payload: Record<string, unknown>};
export interface VoiceEngineV2BridgeVideoFrameMeta {
bridgeVersion?: number;
participantSid: string;
participantIdentity?: string;
trackSid: string;
trackName?: string;
source?: VoiceEngineV2BridgeTrackSource;
width: number;
height: number;
timestampUs: number;
}
export interface VoiceEngineV2BridgeVideoFrame {
meta: VoiceEngineV2BridgeVideoFrameMeta;
data: ArrayBuffer;
}
export type VoiceEngineV2BridgeAudioDeviceRole = 'default' | 'communications' | 'endpoint';
export interface VoiceEngineV2BridgeAudioOutputDevice {
deviceId: string;
label: string;
isDefault: boolean;
role?: VoiceEngineV2BridgeAudioDeviceRole;
endpointLabel?: string;
isDefaultRoute?: boolean;
}
export interface VoiceEngineV2BridgeAudioInputDevice {
deviceId: string;
label: string;
isDefault: boolean;
role?: VoiceEngineV2BridgeAudioDeviceRole;
endpointLabel?: string;
isDefaultRoute?: boolean;
}
export interface VoiceEngineV2BridgeCameraDevice {
deviceId: string;
label: string;
description: string;
index?: number | null;
deviceIdAliases?: Array<string>;
}
export const VOICE_ENGINE_V2_CAMERA_EFFECT_STRENGTH_MIN = 0;
export const VOICE_ENGINE_V2_CAMERA_EFFECT_STRENGTH_MAX = 100;
export const VOICE_ENGINE_V2_CAMERA_EFFECT_STRENGTH_DEFAULT = 50;
export interface VoiceEngineV2BridgePublishCameraOptions extends VoiceEngineV2CameraOptions {
mirror?: boolean;
backgroundBlurStrength?: number;
}
export type VoiceEngineV2BridgeUpdateCameraCaptureOptions = VoiceEngineV2BridgePublishCameraOptions;
export type VoiceEngineV2BridgeCameraBackgroundMode = 'none' | 'non' | 'blur' | 'custom';
export type VoiceEngineV2BridgeCameraBackgroundCustomMediaKind = 'static' | 'animated' | 'video';
export interface VoiceEngineV2BridgePublishProcessedCameraOptions {
width: number;
height: number;
frameRate: number;
}
export interface VoiceEngineV2BridgePublishProcessedCameraResult {
trackSid: string;
}
export type VoiceEngineV2BridgePublishNativeCameraSinkResult = VoiceEngineV2BridgePublishProcessedCameraResult;
export type VoiceEngineV2BridgeStartCameraPreviewOptions = VoiceEngineV2BridgePublishCameraOptions;
export interface VoiceEngineV2BridgeCameraPreviewInfo {
trackSid: string;
width: number;
height: number;
frameRate: number;
}
export type VoiceEngineV2BridgeProcessedCameraFrameFormat = 'i420';
export interface VoiceEngineV2BridgeProcessedCameraFrame {
format: VoiceEngineV2BridgeProcessedCameraFrameFormat;
width: number;
height: number;
timestampUs: number;
data: ArrayBuffer | ArrayBufferView;
}
export interface VoiceEngineV2BridgePublishDeviceScreenShareOptions {
deviceId?: string;
width?: number;
height?: number;
frameRate?: number;
codec?: VoiceEngineV2VideoCodec;
maxBitrateBps?: number;
maxFramerate?: number;
}
export interface VoiceEngineV2BridgeOutboundStat {
trackSid: string;
source: string;
kind: VoiceEngineV2BridgeTrackKind;
codec?: string;
bitrateKbps: number;
packetsLost: number;
fps?: number;
audioLevel?: number;
width?: number;
height?: number;
sourceWidth?: number;
sourceHeight?: number;
targetBitrateKbps?: number;
configuredFps?: number;
targetFps?: number;
effectiveFps?: number;
framesProduced?: number;
framesAccepted?: number;
framesDropped?: number;
framesCoalesced?: number;
framesCaptured?: number;
captureFailures?: number;
maxQueueAgeMs?: number;
maxPushLatencyMs?: number;
adaptiveSendTier?: string;
adaptiveSendReason?: string;
zeroCopy?: boolean;
}
export interface VoiceEngineV2BridgeInboundStat {
participantSid: string;
participantIdentity?: string;
trackSid: string;
source?: string;
kind: VoiceEngineV2BridgeTrackKind;
codec?: string;
bitrateKbps: number;
packetsLost: number;
jitterMs?: number;
audioLevel?: number;
width?: number;
height?: number;
fps?: number;
sourceWidth?: number;
sourceHeight?: number;
}
export type VoiceEngineV2BridgeSendStats = VoiceEngineV2SendStats;
export interface VoiceEngineV2BridgeStats {
rttMs: number | null;
outbound: Array<VoiceEngineV2BridgeOutboundStat>;
inbound: Array<VoiceEngineV2BridgeInboundStat>;
droppedNativeVideoFrames?: number;
droppedVideoFrameCallbacks?: number;
send?: VoiceEngineV2BridgeSendStats | null;
}
export type VoiceEngineV2BridgeRemoteTrackSubscriptionQuality = VoiceEngineV2RemoteTrackQuality;
export type VoiceEngineV2BridgeRemoteTrackSubscriptionOptions = VoiceEngineV2RemoteTrackSubscriptionOptions;
export type VoiceEngineV2BridgeHardwareEncoderBackend = 'nvenc' | 'videotoolbox' | 'none' | string;
export type VoiceEngineV2BridgeHardwareEncoderCapabilities = VoiceEngineV2HardwareEncoderCapabilities;
export interface VoiceEngineV2BridgeCapabilities {
microphoneCapture: boolean;
syntheticMicrophonePcm: boolean;
cameraCapture: boolean;
nativeCameraBackgrounds: boolean;
screenShare: boolean;
screenShareEncodingUpdate: boolean;
screenShareAudio: boolean;
deviceLists: boolean;
outputDeviceSelection: boolean;
participantVolume: boolean;
remoteTrackSubscription: boolean;
dataChannel: boolean;
connectionStats: boolean;
nativeVideoFrames: boolean;
hardwareEncoderCapabilities: boolean;
}
export type VoiceEngineV2BridgeOperationErrorCode =
| 'unsupported-capability'
| 'not-connected'
| 'invalid-args'
| 'native-error';
export interface VoiceEngineV2BridgeOperationError {
code: VoiceEngineV2BridgeOperationErrorCode;
message: string;
capability?: keyof VoiceEngineV2BridgeCapabilities | string;
}
export interface VoiceEngineV2BridgeOperationSuccess {
ok: true;
}
export interface VoiceEngineV2BridgeOperationFailure {
ok: false;
error: VoiceEngineV2BridgeOperationError;
}
export type VoiceEngineV2BridgeOperationResult =
| VoiceEngineV2BridgeOperationSuccess
| VoiceEngineV2BridgeOperationFailure;
export const VOICE_ENGINE_V2_OPERATION_SUCCESS: VoiceEngineV2BridgeOperationSuccess = {ok: true};
export function createVoiceEngineV2OperationFailure(
error: VoiceEngineV2BridgeOperationError,
): VoiceEngineV2BridgeOperationFailure {
return {ok: false, error};
}
export interface VoiceEngineV2BridgeParticipantVolumeOptions {
participantSid: string;
volume: number;
}
export interface VoiceEngineV2BridgeSpeakingDetectionOptions {
localThresholdRms: number;
remoteThresholdRms: number;
}
export interface VoiceEngineV2BridgeIpcMethods {
isSupported: {args: []; result: boolean};
getCapabilities: {args: []; result: VoiceEngineV2BridgeCapabilities};
prewarm: {args: []; result: void};
getHardwareEncoderCapabilities: {args: []; result: VoiceEngineV2BridgeHardwareEncoderCapabilities};
connect: {args: [VoiceEngineV2BridgeConnectOptions]; result: void};
disconnect: {args: []; result: void};
isConnected: {args: []; result: boolean};
publishMicrophone: {args: [VoiceEngineV2BridgePublishMicrophoneOptions]; result: VoiceEngineV2BridgeOperationResult};
pushPcm: {args: [VoiceEngineV2BridgePcmFrame]; result: boolean};
publishScreen: {args: [VoiceEngineV2BridgePublishScreenOptions]; result: void};
updateScreenShareEncoding: {args: [VoiceEngineV2BridgeUpdateScreenShareEncodingOptions]; result: void};
unpublishScreen: {args: []; result: void};
publishScreenAudio: {args: [VoiceEngineV2BridgePublishScreenAudioOptions]; result: void};
pushScreenAudioPcm: {args: [VoiceEngineV2BridgePcmFrame]; result: boolean};
pushScreenAudioFloat: {args: [VoiceEngineV2BridgeFloatPcmFrame]; result: boolean};
unpublishScreenAudio: {args: []; result: void};
setMicEnabled: {args: [boolean]; result: VoiceEngineV2BridgeOperationResult};
setSpeakingDetection: {args: [VoiceEngineV2BridgeSpeakingDetectionOptions]; result: void};
listAudioInputDevices: {args: []; result: Array<VoiceEngineV2BridgeAudioInputDevice>};
listAudioOutputDevices: {args: []; result: Array<VoiceEngineV2BridgeAudioOutputDevice>};
setAudioOutputDevice: {args: [string]; result: void};
setParticipantVolume: {args: [VoiceEngineV2BridgeParticipantVolumeOptions]; result: void};
setRemoteTrackSubscription: {args: [VoiceEngineV2BridgeRemoteTrackSubscriptionOptions]; result: void};
publishData: {args: [VoiceEngineV2BridgePublishDataOptions]; result: void};
listCameraDevices: {args: []; result: Array<VoiceEngineV2BridgeCameraDevice>};
publishCamera: {args: [VoiceEngineV2BridgePublishCameraOptions]; result: void};
updateCameraCapture: {args: [VoiceEngineV2BridgeUpdateCameraCaptureOptions]; result: void};
publishNativeCameraSink: {
args: [VoiceEngineV2BridgePublishCameraOptions];
result: VoiceEngineV2BridgePublishNativeCameraSinkResult;
};
publishProcessedCamera: {
args: [VoiceEngineV2BridgePublishProcessedCameraOptions];
result: VoiceEngineV2BridgePublishProcessedCameraResult;
};
pushProcessedCameraFrame: {args: [VoiceEngineV2BridgeProcessedCameraFrame]; result: boolean};
pushCameraBackgroundFrame: {args: [VoiceEngineV2BridgeProcessedCameraFrame]; result: boolean};
clearCameraBackgroundFrame: {args: []; result: void};
publishDeviceScreenShare: {args: [VoiceEngineV2BridgePublishDeviceScreenShareOptions]; result: void};
unpublishCamera: {args: []; result: void};
isPublishingCamera: {args: []; result: boolean};
startCameraPreview: {
args: [VoiceEngineV2BridgeStartCameraPreviewOptions];
result: VoiceEngineV2BridgeCameraPreviewInfo;
};
stopCameraPreview: {args: []; result: void};
getConnectionStats: {args: []; result: VoiceEngineV2BridgeStats};
getVoiceEngineReadiness: {args: []; result: VoiceEngineV2BridgeReadiness};
getAudioDeviceModuleState: {args: []; result: VoiceEngineV2BridgeAudioDeviceModuleState};
}
export const VOICE_ENGINE_V2_BRIDGE_METHODS = [
'isSupported',
'getCapabilities',
'prewarm',
'getHardwareEncoderCapabilities',
'connect',
'disconnect',
'isConnected',
'publishMicrophone',
'pushPcm',
'publishScreen',
'updateScreenShareEncoding',
'unpublishScreen',
'publishScreenAudio',
'pushScreenAudioPcm',
'pushScreenAudioFloat',
'unpublishScreenAudio',
'setMicEnabled',
'setSpeakingDetection',
'listAudioInputDevices',
'listAudioOutputDevices',
'setAudioOutputDevice',
'setParticipantVolume',
'setRemoteTrackSubscription',
'publishData',
'listCameraDevices',
'publishCamera',
'updateCameraCapture',
'publishNativeCameraSink',
'publishProcessedCamera',
'pushProcessedCameraFrame',
'pushCameraBackgroundFrame',
'clearCameraBackgroundFrame',
'publishDeviceScreenShare',
'unpublishCamera',
'isPublishingCamera',
'startCameraPreview',
'stopCameraPreview',
'getConnectionStats',
'getVoiceEngineReadiness',
'getAudioDeviceModuleState',
'onEvent',
'onVideoFrame',
] as const satisfies ReadonlyArray<VoiceEngineV2BridgeMethodName>;
export const VOICE_ENGINE_V2_IPC_CHANNELS = {
isSupported: 'voice-engine-v2:is-supported',
getCapabilities: 'voice-engine-v2:get-capabilities',
prewarm: 'voice-engine-v2:prewarm',
getHardwareEncoderCapabilities: 'voice-engine-v2:get-hardware-encoder-capabilities',
connect: 'voice-engine-v2:connect',
disconnect: 'voice-engine-v2:disconnect',
isConnected: 'voice-engine-v2:is-connected',
publishMicrophone: 'voice-engine-v2:publish-microphone',
pushPcm: 'voice-engine-v2:push-pcm',
publishScreen: 'voice-engine-v2:publish-screen',
updateScreenShareEncoding: 'voice-engine-v2:update-screen-share-encoding',
unpublishScreen: 'voice-engine-v2:unpublish-screen',
publishScreenAudio: 'voice-engine-v2:publish-screen-audio',
pushScreenAudioPcm: 'voice-engine-v2:push-screen-audio-pcm',
pushScreenAudioFloat: 'voice-engine-v2:push-screen-audio-float',
unpublishScreenAudio: 'voice-engine-v2:unpublish-screen-audio',
setMicEnabled: 'voice-engine-v2:set-mic-enabled',
setSpeakingDetection: 'voice-engine-v2:set-speaking-detection',
listAudioInputDevices: 'voice-engine-v2:list-audio-input-devices',
listAudioOutputDevices: 'voice-engine-v2:list-audio-output-devices',
setAudioOutputDevice: 'voice-engine-v2:set-audio-output-device',
setParticipantVolume: 'voice-engine-v2:set-participant-volume',
setRemoteTrackSubscription: 'voice-engine-v2:set-remote-track-subscription',
publishData: 'voice-engine-v2:publish-data',
listCameraDevices: 'voice-engine-v2:list-camera-devices',
publishCamera: 'voice-engine-v2:publish-camera',
updateCameraCapture: 'voice-engine-v2:update-camera-capture',
publishNativeCameraSink: 'voice-engine-v2:publish-native-camera-sink',
publishProcessedCamera: 'voice-engine-v2:publish-processed-camera',
pushProcessedCameraFrame: 'voice-engine-v2:push-processed-camera-frame',
pushCameraBackgroundFrame: 'voice-engine-v2:push-camera-background-frame',
clearCameraBackgroundFrame: 'voice-engine-v2:clear-camera-background-frame',
publishDeviceScreenShare: 'voice-engine-v2:publish-device-screen-share',
unpublishCamera: 'voice-engine-v2:unpublish-camera',
isPublishingCamera: 'voice-engine-v2:is-publishing-camera',
startCameraPreview: 'voice-engine-v2:start-camera-preview',
stopCameraPreview: 'voice-engine-v2:stop-camera-preview',
getConnectionStats: 'voice-engine-v2:get-connection-stats',
getVoiceEngineReadiness: 'voice-engine-v2:get-readiness',
getAudioDeviceModuleState: 'voice-engine-v2:get-audio-device-module-state',
} as const satisfies {[Method in VoiceEngineV2InvokeMethodName]: string};
export const VOICE_ENGINE_V2_EVENT_CHANNELS = {
event: 'voice-engine-v2:event',
videoFrame: 'voice-engine-v2:video-frame',
} as const satisfies {
event: string;
videoFrame: string;
};
export interface VoiceEngineV2BridgePushedEvents {
event: VoiceEngineV2BridgeEvent;
videoFrame: VoiceEngineV2BridgeVideoFrame;
}
export interface VoiceEngineV2BridgeApi {
bridgeVersion: number;
isSupported(): Promise<boolean>;
getCapabilities(): Promise<VoiceEngineV2BridgeCapabilities>;
prewarm(): Promise<void>;
getHardwareEncoderCapabilities(): Promise<VoiceEngineV2BridgeHardwareEncoderCapabilities>;
connect(options: VoiceEngineV2BridgeConnectOptions): Promise<void>;
disconnect(): Promise<void>;
isConnected(): Promise<boolean>;
publishMicrophone(options: VoiceEngineV2BridgePublishMicrophoneOptions): Promise<VoiceEngineV2BridgeOperationResult>;
pushPcm(frame: VoiceEngineV2BridgePcmFrame): Promise<boolean>;
publishScreen(options: VoiceEngineV2BridgePublishScreenOptions): Promise<void>;
updateScreenShareEncoding(options: VoiceEngineV2BridgeUpdateScreenShareEncodingOptions): Promise<void>;
unpublishScreen(): Promise<void>;
publishScreenAudio(options: VoiceEngineV2BridgePublishScreenAudioOptions): Promise<void>;
pushScreenAudioPcm(frame: VoiceEngineV2BridgePcmFrame): Promise<boolean>;
pushScreenAudioFloat(frame: VoiceEngineV2BridgeFloatPcmFrame): Promise<boolean>;
unpublishScreenAudio(): Promise<void>;
setMicEnabled(enabled: boolean): Promise<VoiceEngineV2BridgeOperationResult>;
setSpeakingDetection(options: VoiceEngineV2BridgeSpeakingDetectionOptions): Promise<void>;
listAudioInputDevices(): Promise<Array<VoiceEngineV2BridgeAudioInputDevice>>;
listAudioOutputDevices(): Promise<Array<VoiceEngineV2BridgeAudioOutputDevice>>;
setAudioOutputDevice(deviceId: string): Promise<void>;
setParticipantVolume(participantSid: string, volume: number): Promise<void>;
setRemoteTrackSubscription(options: VoiceEngineV2BridgeRemoteTrackSubscriptionOptions): Promise<void>;
publishData(options: VoiceEngineV2BridgePublishDataOptions): Promise<void>;
listCameraDevices(): Promise<Array<VoiceEngineV2BridgeCameraDevice>>;
publishCamera(options: VoiceEngineV2BridgePublishCameraOptions): Promise<void>;
updateCameraCapture(options: VoiceEngineV2BridgeUpdateCameraCaptureOptions): Promise<void>;
publishNativeCameraSink(
options: VoiceEngineV2BridgePublishCameraOptions,
): Promise<VoiceEngineV2BridgePublishNativeCameraSinkResult>;
publishProcessedCamera(
options: VoiceEngineV2BridgePublishProcessedCameraOptions,
): Promise<VoiceEngineV2BridgePublishProcessedCameraResult>;
pushProcessedCameraFrame(frame: VoiceEngineV2BridgeProcessedCameraFrame): Promise<boolean>;
pushCameraBackgroundFrame(frame: VoiceEngineV2BridgeProcessedCameraFrame): Promise<boolean>;
clearCameraBackgroundFrame(): Promise<void>;
publishDeviceScreenShare(options: VoiceEngineV2BridgePublishDeviceScreenShareOptions): Promise<void>;
unpublishCamera(): Promise<void>;
isPublishingCamera(): Promise<boolean>;
startCameraPreview(
options: VoiceEngineV2BridgeStartCameraPreviewOptions,
): Promise<VoiceEngineV2BridgeCameraPreviewInfo>;
stopCameraPreview(): Promise<void>;
getConnectionStats(): Promise<VoiceEngineV2BridgeStats>;
getVoiceEngineReadiness(): Promise<VoiceEngineV2BridgeReadiness>;
getAudioDeviceModuleState?(): Promise<VoiceEngineV2BridgeAudioDeviceModuleState>;
onEvent(callback: (event: VoiceEngineV2BridgeEvent) => void): () => void;
onVideoFrame(callback: (frame: VoiceEngineV2BridgeVideoFrame) => void): () => void;
}
@@ -6,7 +6,6 @@ import inboundVideoFramesFixtureJson from '../../fixtures/event_logs/inbound_vid
import participantsTracksFixtureJson from '../../fixtures/event_logs/participants_tracks.json';
import subscriptionQualityFixtureJson from '../../fixtures/event_logs/subscription_quality.json';
import watchedStreamsFixtureJson from '../../fixtures/event_logs/watched_streams.json';
import {translateVoiceEngineV2BridgeEventToEvents, translateVoiceEngineV2BridgeVideoFrameToEvent} from '../bridge';
import type {VoiceEngineV2Command} from '../protocol/commands';
import type {VoiceEngineV2Event} from '../protocol/events';
import type {
@@ -136,39 +135,45 @@ describe('voice engine v2 participants, subscriptions, and inbound video fixture
assertFixture(fixture);
});
it('translates native bridge participant, track, and frame events into v2 projections', () => {
let snapshot = createVoiceEngineV2InitialSnapshot(availableVoiceEngineV2Capabilities());
const bridgeEvents = translateVoiceEngineV2BridgeEventToEvents({
type: 'trackSubscribed',
payload: {
participantSid: 'PA_alice',
identity: 'alice',
participantName: 'Alice',
trackSid: 'TR_screen',
trackName: 'screen_share',
kind: 'video',
source: 'screen_share',
muted: false,
subscribed: true,
subscriptionStatus: 'subscribed',
it('projects a subscribed video track and its first frame into room and inbound-video state', () => {
const events: Array<VoiceEngineV2Event> = [
{
type: 'room.trackPublished',
track: {
participantIdentity: 'alice',
participantSid: 'PA_alice',
trackSid: 'TR_screen',
trackName: 'screen_share',
kind: 'video',
source: 'screen',
muted: false,
},
},
});
for (const event of bridgeEvents) {
snapshot = transitionVoiceEngineV2(snapshot, event).snapshot;
}
snapshot = transitionVoiceEngineV2(
snapshot,
translateVoiceEngineV2BridgeVideoFrameToEvent({
meta: {
{
type: 'inboundVideo.trackSubscribed',
track: {
participantSid: 'PA_alice',
participantIdentity: 'alice',
trackSid: 'TR_screen',
source: 'screen',
},
},
{
type: 'inboundVideo.frameReceived',
frame: {
participantSid: 'PA_alice',
trackSid: 'TR_screen',
width: 320,
height: 180,
timestampUs: 100,
byteLength: 86_400,
},
data: new ArrayBuffer(86_400),
}),
).snapshot;
},
];
let snapshot = createVoiceEngineV2InitialSnapshot(availableVoiceEngineV2Capabilities());
for (const event of events) {
snapshot = transitionVoiceEngineV2(snapshot, event).snapshot;
}
expect(snapshot.room.tracks.TR_screen).toMatchObject({
participantIdentity: 'alice',
@@ -121,94 +121,6 @@ describe('transitionVoiceEngineV2', () => {
]);
});
it('defers native microphone publish until the audio device module is ready', () => {
const queued = applyEvents(initialSnapshot(), [
{type: 'nativeAudioDeviceModule.statusChanged', status: 'warming'},
{type: 'microphone.publishRequested', options: {deviceId: 'default'}},
]);
const established = transitionVoiceEngineV2(queued, {
type: 'connection.externallyEstablished',
options: {url: 'wss://voice', token: ''},
});
expect(established.snapshot.connection.status).toBe('connected');
expect(established.snapshot.microphone.status).toBe('idle');
expect(established.snapshot.microphone.desired).toEqual({deviceId: 'default'});
expect(established.commands).toEqual([]);
const ready = transitionVoiceEngineV2(established.snapshot, {
type: 'nativeAudioDeviceModule.statusChanged',
status: 'ready',
});
expect(ready.snapshot.microphone.status).toBe('publishing');
expect(ready.commands).toEqual([{type: 'microphone.publish', operationId: 1, options: {deviceId: 'default'}}]);
});
it('fails deferred native microphone publish when the audio device module fails', () => {
const established = applyEvents(initialSnapshot(), [
{type: 'nativeAudioDeviceModule.statusChanged', status: 'warming'},
{type: 'microphone.publishRequested', options: {deviceId: 'default'}},
{type: 'connection.externallyEstablished', options: {url: 'wss://voice', token: ''}},
]);
const failed = transitionVoiceEngineV2(established, {
type: 'nativeAudioDeviceModule.statusChanged',
status: 'failed',
detail: 'ADM probe failed',
});
expect(failed.snapshot.microphone.status).toBe('failed');
expect(failed.snapshot.microphone.failure).toEqual({
code: 'deviceUnavailable',
capability: 'microphone',
message: 'ADM probe failed',
});
expect(failed.commands).toEqual([]);
});
it('does not republish an already published microphone on repeated ADM ready events', () => {
const established = applyEvents(initialSnapshot(), [
{type: 'nativeAudioDeviceModule.statusChanged', status: 'warming'},
{type: 'microphone.publishRequested', options: {deviceId: 'default'}},
{type: 'connection.externallyEstablished', options: {url: 'wss://voice', token: ''}},
]);
const ready = transitionVoiceEngineV2(established, {
type: 'nativeAudioDeviceModule.statusChanged',
status: 'ready',
});
const operationId = ready.commands[0]?.operationId ?? -1;
const published = transitionVoiceEngineV2(ready.snapshot, {
type: 'microphone.publishSucceeded',
operationId,
}).snapshot;
const repeated = transitionVoiceEngineV2(published, {
type: 'nativeAudioDeviceModule.statusChanged',
status: 'ready',
});
expect(repeated.snapshot.microphone.status).toBe('published');
expect(repeated.commands).toEqual([]);
});
it('does not let set-microphone-enabled implicitly publish while native ADM is warming', () => {
const established = applyEvents(initialSnapshot(), [
{type: 'nativeAudioDeviceModule.statusChanged', status: 'warming'},
{type: 'connection.externallyEstablished', options: {url: 'wss://voice', token: ''}},
]);
const enabled = transitionVoiceEngineV2(established, {
type: 'microphone.setEnabledRequested',
enabled: true,
});
expect(enabled.snapshot.microphone.enabled).toBe(true);
expect(enabled.snapshot.microphone.status).toBe('idle');
expect(enabled.commands).toEqual([]);
});
it('treats a repeated connection.externallyEstablished for the same endpoint as a no-op', () => {
const established = applyEvents(initialSnapshot(), [
{type: 'connection.externallyEstablished', options: {url: 'wss://voice', token: ''}},
@@ -17,7 +17,6 @@ import {transitionImplementation} from './implementation';
import {transitionInboundVideo} from './inboundVideo';
import {transitionLifecycle} from './lifecycle';
import {transitionMicrophone} from './microphone';
import {transitionNativeAudioDeviceModule} from './nativeAudioDeviceModule';
import {transitionNativeAudioTap} from './nativeAudioTap';
import {transitionNativeCapture} from './nativeCapture';
import {transitionNativeFrameSink} from './nativeFrameSink';
@@ -179,8 +178,6 @@ export function dispatchPlatformEvent(
case 'devices.selectAudioOutputRequested':
case 'devices.selectCameraRequested':
return transitionDevices(snapshot, event);
case 'nativeAudioDeviceModule.statusChanged':
return transitionNativeAudioDeviceModule(snapshot, event);
case 'nativeCapture.startRequested':
case 'nativeCapture.updateRequested':
case 'nativeCapture.stopRequested':
@@ -203,46 +203,6 @@ function microphoneOptionsWithAudioInputDevice(
return {...options, deviceId};
}
function nativeAudioDeviceModuleNotReadyError(snapshot: VoiceEngineV2Snapshot): VoiceEngineV2Error {
assert.ok(snapshot != null, 'nativeAudioDeviceModuleNotReadyError snapshot must not be null');
assert.ok(
snapshot.nativeAudioDeviceModule.status === 'failed',
'nativeAudioDeviceModuleNotReadyError requires failed ADM status',
);
return {
code: 'deviceUnavailable',
capability: 'microphone',
message: snapshot.nativeAudioDeviceModule.detail ?? 'Native audio device module failed to become ready',
};
}
function planNativeAudioDeviceModuleGatedMicrophone(snapshot: VoiceEngineV2Snapshot): VoiceEngineV2Transition | null {
assert.ok(snapshot != null, 'planNativeAudioDeviceModuleGatedMicrophone snapshot must not be null');
assert.ok(snapshot.nativeAudioDeviceModule != null, 'snapshot.nativeAudioDeviceModule must not be null');
switch (snapshot.nativeAudioDeviceModule.status) {
case 'unsupported':
case 'ready':
return null;
case 'unknown':
case 'warming':
return {
snapshot: {...snapshot, microphone: {...snapshot.microphone, status: 'idle', failure: null}},
commands: [],
};
case 'failed': {
const error = nativeAudioDeviceModuleNotReadyError(snapshot);
return {
snapshot: {
...snapshot,
microphone: {...snapshot.microphone, status: 'failed', published: null, operationId: null, failure: error},
lastFailure: error,
},
commands: [],
};
}
}
}
export function planMicrophoneDeviceChange(
snapshot: VoiceEngineV2Snapshot,
deviceId: string | null,
@@ -327,8 +287,6 @@ export function beginMicrophonePublish(snapshot: VoiceEngineV2Snapshot): VoiceEn
};
}
if (!isConnected(snapshot)) return {snapshot, commands: []};
const gated = planNativeAudioDeviceModuleGatedMicrophone(snapshot);
if (gated) return gated;
if (snapshot.microphone.status === 'publishing') return {snapshot, commands: []};
if (snapshot.microphone.status === 'published' && sameMicrophoneOptions(snapshot.microphone.published, desired)) {
return {snapshot, commands: []};
@@ -1,29 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import assert from 'node:assert/strict';
import type {VoiceEngineV2Event} from '../../protocol/events';
import type {VoiceEngineV2Snapshot, VoiceEngineV2Transition} from '../state';
import {beginMicrophonePublish} from './microphone';
type VoiceEngineV2NativeAudioDeviceModuleEvent = Extract<
VoiceEngineV2Event,
{type: 'nativeAudioDeviceModule.statusChanged'}
>;
export function transitionNativeAudioDeviceModule(
snapshot: VoiceEngineV2Snapshot,
event: VoiceEngineV2NativeAudioDeviceModuleEvent,
): VoiceEngineV2Transition {
assert.ok(snapshot != null, 'transitionNativeAudioDeviceModule snapshot must not be null');
assert.ok(event != null, 'transitionNativeAudioDeviceModule event must not be null');
assert.equal(event.type, 'nativeAudioDeviceModule.statusChanged', 'unexpected ADM event type');
assert.equal(typeof event.status, 'string', 'ADM status must be a string');
const next: VoiceEngineV2Snapshot = {
...snapshot,
nativeAudioDeviceModule: {
status: event.status,
detail: event.detail ?? null,
},
};
return beginMicrophonePublish(next);
}
@@ -27,7 +27,6 @@ import type {
VoiceEngineV2LiveKitRoomState,
VoiceEngineV2MediaStatus,
VoiceEngineV2MicrophoneOptions,
VoiceEngineV2NativeAudioDeviceModuleState,
VoiceEngineV2NativeAudioTapOptions,
VoiceEngineV2NativeCaptureOptions,
VoiceEngineV2NativeFrameSinkOptions,
@@ -162,7 +161,6 @@ export interface VoiceEngineV2Snapshot {
connection: VoiceEngineV2ConnectionState;
gateway: VoiceEngineV2GatewayState;
liveKit: VoiceEngineV2LiveKitState;
nativeAudioDeviceModule: VoiceEngineV2NativeAudioDeviceModuleState;
microphone: VoiceEngineV2MicrophoneState;
camera: VoiceEngineV2LocalMediaState<VoiceEngineV2CameraOptions>;
screen: VoiceEngineV2LocalMediaState<VoiceEngineV2ScreenOptions>;
@@ -286,10 +284,6 @@ export function createVoiceEngineV2InitialSnapshot(
serverRegion: null,
failure: null,
},
nativeAudioDeviceModule: {
status: 'unsupported',
detail: null,
},
microphone: {
status: 'idle',
desired: null,
-110
View File
@@ -1,95 +1,5 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
export type {
VoiceEngineV2BridgeApi,
VoiceEngineV2BridgeAudioDeviceRole,
VoiceEngineV2BridgeAudioInputDevice,
VoiceEngineV2BridgeAudioOutputDevice,
VoiceEngineV2BridgeCameraDevice,
VoiceEngineV2BridgeCapabilities,
VoiceEngineV2BridgeConnectionQuality,
VoiceEngineV2BridgeConnectOptions,
VoiceEngineV2BridgeDroppedEventCounts,
VoiceEngineV2BridgeEvent,
VoiceEngineV2BridgeEventPayloads,
VoiceEngineV2BridgeEventType,
VoiceEngineV2BridgeFloatPcmFrame,
VoiceEngineV2BridgeHardwareEncoderBackend,
VoiceEngineV2BridgeHardwareEncoderCapabilities,
VoiceEngineV2BridgeInboundStat,
VoiceEngineV2BridgeIpcMethods,
VoiceEngineV2BridgeKnownEvent,
VoiceEngineV2BridgeKnownEventType,
VoiceEngineV2BridgeLocalTrackRepublishedEventPayload,
VoiceEngineV2BridgeMethodName,
VoiceEngineV2BridgeOperationError,
VoiceEngineV2BridgeOperationErrorCode,
VoiceEngineV2BridgeOperationFailure,
VoiceEngineV2BridgeOperationResult,
VoiceEngineV2BridgeOperationSuccess,
VoiceEngineV2BridgeOutboundStat,
VoiceEngineV2BridgeParticipantEventPayload,
VoiceEngineV2BridgeParticipantVolumeOptions,
VoiceEngineV2BridgePcmFrame,
VoiceEngineV2BridgePublishCameraOptions,
VoiceEngineV2BridgePublishDataOptions,
VoiceEngineV2BridgePublishDeviceScreenShareOptions,
VoiceEngineV2BridgePublishMicrophoneOptions,
VoiceEngineV2BridgePublishScreenAudioOptions,
VoiceEngineV2BridgePublishScreenOptions,
VoiceEngineV2BridgePushedEvents,
VoiceEngineV2BridgeRemoteTrackSubscriptionOptions,
VoiceEngineV2BridgeRemoteTrackSubscriptionQuality,
VoiceEngineV2BridgeScreenPacing,
VoiceEngineV2BridgeSendStats,
VoiceEngineV2BridgeStats,
VoiceEngineV2BridgeSubscribedTrackEventPayload,
VoiceEngineV2BridgeSubscriptionStatus,
VoiceEngineV2BridgeTrackEventPayload,
VoiceEngineV2BridgeTrackKind,
VoiceEngineV2BridgeTrackSource,
VoiceEngineV2BridgeTrackSubscriptionFailedEventPayload,
VoiceEngineV2BridgeUpdateScreenShareEncodingOptions,
VoiceEngineV2BridgeVideoFrame,
VoiceEngineV2BridgeVideoFrameMeta,
VoiceEngineV2InvokeMethodName,
} from './bridge';
export {
assertVoiceEngineV2BridgeAudioOptionsInvariants,
assertVoiceEngineV2BridgeVideoOptionsInvariants,
clampVoiceEngineV2ParticipantVolume,
coerceVoiceEngineV2BridgeStats,
createVoiceEngineV2OperationFailure,
getVoiceEngineV2BridgeDroppedEventCounts,
isVoiceEngineV2BridgeBinaryPayload,
isVoiceEngineV2BridgeConnectOptions,
isVoiceEngineV2BridgeFloatPcmFrame,
isVoiceEngineV2BridgeKnownEventType,
isVoiceEngineV2BridgePcmFrame,
isVoiceEngineV2BridgePublishCameraOptions,
isVoiceEngineV2BridgePublishDataOptions,
isVoiceEngineV2BridgePublishDeviceScreenShareOptions,
isVoiceEngineV2BridgePublishMicrophoneOptions,
isVoiceEngineV2BridgePublishScreenAudioOptions,
isVoiceEngineV2BridgePublishScreenOptions,
isVoiceEngineV2BridgeRemoteTrackSubscriptionOptions,
isVoiceEngineV2BridgeRemoteTrackSubscriptionQuality,
isVoiceEngineV2BridgeUpdateScreenShareEncodingOptions,
isVoiceEngineV2ParticipantVolumeOptions,
normalizeVoiceEngineV2BridgeCapabilities,
normalizeVoiceEngineV2BridgeHardwareEncoderCapabilities,
resetVoiceEngineV2BridgeDroppedEventCounts,
translateVoiceEngineV2BridgeEventToEvents,
translateVoiceEngineV2BridgeVideoFrameToEvent,
unavailableVoiceEngineV2BridgeCapabilities,
unavailableVoiceEngineV2BridgeHardwareEncoderCapabilities,
VOICE_ENGINE_V2_BRIDGE_DROPPED_EVENTS_COUNT_MAX,
VOICE_ENGINE_V2_BRIDGE_METHODS,
VOICE_ENGINE_V2_BRIDGE_VERSION,
VOICE_ENGINE_V2_EVENT_CHANNELS,
VOICE_ENGINE_V2_IPC_CHANNELS,
VOICE_ENGINE_V2_OPERATION_SUCCESS,
} from './bridge';
export type {
VoiceEngineV2CapabilitiesProjection,
VoiceEngineV2ConnectionState,
@@ -161,17 +71,10 @@ export type {
VoiceEngineV2CameraEncodingPlanAction,
VoiceEngineV2CameraEncodingPlanInput,
VoiceEngineV2CameraEncodingPlanReason,
VoiceEngineV2LocalTrackReconnectState,
VoiceEngineV2MicrophoneFailureContext,
VoiceEngineV2MicrophoneOperationFailureAction,
VoiceEngineV2NativeConnectRetryPolicyInput,
VoiceEngineV2NativeParticipantVolumeInput,
VoiceEngineV2OperationFailureLike,
VoiceEngineV2OperationResultLike,
VoiceEngineV2ParticipantE2eeState,
VoiceEngineV2ReconnectTrackSource,
VoiceEngineV2RetryConnectionState,
VoiceEngineV2RetryVoiceState,
VoiceEngineV2ScreenEncodingPlan,
VoiceEngineV2ScreenEncodingPlanAction,
VoiceEngineV2ScreenEncodingPlanInput,
@@ -195,12 +98,9 @@ export {
applyVoiceEngineV2CameraEncodingOptions,
applyVoiceEngineV2ScreenEncodingOptions,
asVoiceEngineV2StatsTrackSource,
boostedVoiceEngineV2VolumePercentToTrackVolume,
clampVoiceEngineV2VolumePercent,
classifyVoiceEngineV2TrackStats,
coalesceVoiceEngineV2OutboundStats,
coerceVoiceEngineV2Stats,
computeVoiceEngineV2NativeParticipantVolume,
getVoiceEngineV2MicrophoneOperationFailureAction,
hasVoiceEngineV2NativeHardwareEncoder,
hasVoiceEngineV2NativeNvencEncoder,
@@ -208,17 +108,13 @@ export {
isMoreEfficientVoiceEngineV2VideoCodec,
maxDecodableVoiceEngineV2VideoCodec,
normalizeVoiceEngineV2HardwareEncoderCapabilities,
normalizeVoiceEngineV2ParticipantE2eeState,
planVoiceEngineV2CameraEncodingChange,
planVoiceEngineV2NegotiatedVideoCodec,
planVoiceEngineV2ScreenEncodingChange,
shouldRetryVoiceEngineV2NativeConnectTimeout,
shouldSuppressVoiceEngineV2LocalTrackStateDuringReconnect,
summarizeVoiceEngineV2Stats,
unavailableVoiceEngineV2HardwareEncoderCapabilities,
VOICE_ENGINE_V2_VIDEO_CODEC_FLOOR,
VOICE_ENGINE_V2_VIDEO_CODEC_PREFERENCE,
VOICE_ENGINE_V2_VOLUME_MAX_PERCENT,
VoiceEngineV2StatsTrackSource,
voiceEngineV2VideoCodecRank,
worseVoiceEngineV2VideoCodec,
@@ -292,8 +188,6 @@ export type {
VoiceEngineV2MediaStatus,
VoiceEngineV2MicrophoneOptions,
VoiceEngineV2Model,
VoiceEngineV2NativeAudioDeviceModuleState,
VoiceEngineV2NativeAudioDeviceModuleStatus,
VoiceEngineV2NativeAudioTapOptions,
VoiceEngineV2NativeCaptureFrame,
VoiceEngineV2NativeCaptureKind,
@@ -344,7 +238,6 @@ export {
export type {
VoiceEngineV2EventLogEntry,
VoiceEngineV2EventLogSpillSink,
VoiceEngineV2FrameStatsEvent,
VoiceEngineV2MemoryEventLogSpillSink,
VoiceEngineV2RuntimeClock,
VoiceEngineV2RuntimeListener,
@@ -354,10 +247,7 @@ export type {
export {
commandResultToEvent,
createVoiceEngineV2MemoryEventLogSpillSink,
VOICE_ENGINE_V2_FRAME_STATS_FLUSH_INTERVAL_MS,
VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP,
VoiceEngineV2Controller,
VoiceEngineV2FrameStatsAccumulator,
VoiceEngineV2Runtime,
} from './runtime';
export type {
@@ -1,23 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
export type VoiceEngineV2ParticipantE2eeState = 'encrypted' | 'decryption_failed' | 'unencrypted' | 'unknown';
export function normalizeVoiceEngineV2ParticipantE2eeState(raw: unknown): VoiceEngineV2ParticipantE2eeState {
switch (raw) {
case 'encrypted':
case 'ok':
return 'encrypted';
case 'decryption_failed':
case 'decryptionFailed':
case 'error':
case 'broken':
return 'decryption_failed';
case 'unencrypted':
case 'missing_key':
case 'missingKey':
case 'internal_error':
return 'unencrypted';
default:
return 'unknown';
}
}
@@ -21,8 +21,6 @@ export {
voiceEngineV2VideoCodecRank,
worseVoiceEngineV2VideoCodec,
} from './codecNegotiation';
export type {VoiceEngineV2ParticipantE2eeState} from './e2eeState';
export {normalizeVoiceEngineV2ParticipantE2eeState} from './e2eeState';
export {
hasVoiceEngineV2NativeHardwareEncoder,
hasVoiceEngineV2NativeNvencEncoder,
@@ -30,11 +28,6 @@ export {
normalizeVoiceEngineV2HardwareEncoderCapabilities,
unavailableVoiceEngineV2HardwareEncoderCapabilities,
} from './hardwareEncoderCapabilities';
export type {
VoiceEngineV2LocalTrackReconnectState,
VoiceEngineV2ReconnectTrackSource,
} from './localMediaReconnectSuppression';
export {shouldSuppressVoiceEngineV2LocalTrackStateDuringReconnect} from './localMediaReconnectSuppression';
export type {
VoiceEngineV2MicrophoneFailureContext,
VoiceEngineV2MicrophoneOperationFailureAction,
@@ -42,19 +35,6 @@ export type {
VoiceEngineV2OperationResultLike,
} from './microphoneFailureAction';
export {getVoiceEngineV2MicrophoneOperationFailureAction} from './microphoneFailureAction';
export type {
VoiceEngineV2NativeConnectRetryPolicyInput,
VoiceEngineV2RetryConnectionState,
VoiceEngineV2RetryVoiceState,
} from './nativeConnectRetry';
export {shouldRetryVoiceEngineV2NativeConnectTimeout} from './nativeConnectRetry';
export type {VoiceEngineV2NativeParticipantVolumeInput} from './nativeParticipantVolume';
export {
boostedVoiceEngineV2VolumePercentToTrackVolume,
clampVoiceEngineV2VolumePercent,
computeVoiceEngineV2NativeParticipantVolume,
VOICE_ENGINE_V2_VOLUME_MAX_PERCENT,
} from './nativeParticipantVolume';
export type {
VoiceEngineV2ScreenEncodingPlan,
VoiceEngineV2ScreenEncodingPlanAction,
@@ -1,30 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
export type VoiceEngineV2ReconnectTrackSource =
| 'microphone'
| 'camera'
| 'screen_share'
| 'screen'
| 'unknown'
| string;
export interface VoiceEngineV2LocalTrackReconnectState {
source: VoiceEngineV2ReconnectTrackSource;
enabled: boolean;
reconnecting: boolean;
restoreVideo: boolean;
restoreStream: boolean;
}
export function shouldSuppressVoiceEngineV2LocalTrackStateDuringReconnect({
source,
enabled,
reconnecting,
restoreVideo,
restoreStream,
}: VoiceEngineV2LocalTrackReconnectState): boolean {
if (enabled || !reconnecting) return false;
if (source === 'camera') return restoreVideo;
if (source === 'screen_share' || source === 'screen') return restoreStream;
return false;
}
@@ -1,60 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
const PRIVATE_VOICE_SCOPE_ID = '@me';
export interface VoiceEngineV2RetryVoiceState {
guild_id: string | null;
channel_id: string | null;
}
export interface VoiceEngineV2RetryConnectionState {
connectionId: string | null;
guildId: string | null;
channelId: string | null;
connecting: boolean;
connected: boolean;
reconnecting: boolean;
}
export interface VoiceEngineV2NativeConnectRetryPolicyInput {
connectionId: string;
guildId: string | null;
channelId: string;
voiceState: VoiceEngineV2RetryVoiceState | null;
connectionState: VoiceEngineV2RetryConnectionState;
}
function normalizeVoiceScope(guildId: string | null | undefined): string {
return guildId ?? PRIVATE_VOICE_SCOPE_ID;
}
function isGatewayVoiceStateActiveForConnection({
guildId,
channelId,
voiceState,
}: VoiceEngineV2NativeConnectRetryPolicyInput): boolean {
if (!voiceState?.channel_id) return false;
return (
voiceState.channel_id === channelId && normalizeVoiceScope(voiceState.guild_id) === normalizeVoiceScope(guildId)
);
}
function isAcceptedVoiceConnectionStillActive({
connectionId,
guildId,
channelId,
connectionState,
}: VoiceEngineV2NativeConnectRetryPolicyInput): boolean {
return (
connectionState.connectionId === connectionId &&
connectionState.channelId === channelId &&
normalizeVoiceScope(connectionState.guildId) === normalizeVoiceScope(guildId) &&
(connectionState.connecting || connectionState.connected || connectionState.reconnecting)
);
}
export function shouldRetryVoiceEngineV2NativeConnectTimeout(
input: VoiceEngineV2NativeConnectRetryPolicyInput,
): boolean {
return isGatewayVoiceStateActiveForConnection(input) || isAcceptedVoiceConnectionStillActive(input);
}
@@ -1,195 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {describe, expect, it} from 'vitest';
import {
computeVoiceEngineV2NativeParticipantVolume,
VOICE_ENGINE_V2_VOLUME_MAX_PERCENT,
} from './nativeParticipantVolume';
const USER_VOLUME_PERCENTS = [0, 25, 50, 100, 150, VOICE_ENGINE_V2_VOLUME_MAX_PERCENT];
describe('computeVoiceEngineV2NativeParticipantVolume deafen handling', () => {
it.each(USER_VOLUME_PERCENTS)('forces volume 0 when effectiveDeaf regardless of user volume %d', (percent) => {
const volume = computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: percent,
outputVolumePercent: 100,
locallyMuted: false,
effectiveDeaf: true,
});
expect(volume).toBe(0);
});
it('forces volume 0 when effectiveDeaf and locallyMuted', () => {
const volume = computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: 100,
outputVolumePercent: 100,
locallyMuted: true,
effectiveDeaf: true,
});
expect(volume).toBe(0);
});
it('forces volume 0 when effectiveDeaf with boosted output volume', () => {
const volume = computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: VOICE_ENGINE_V2_VOLUME_MAX_PERCENT,
outputVolumePercent: VOICE_ENGINE_V2_VOLUME_MAX_PERCENT,
locallyMuted: false,
effectiveDeaf: true,
});
expect(volume).toBe(0);
});
it('returns 0 when locallyMuted and not deafened', () => {
const volume = computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: 100,
outputVolumePercent: 100,
locallyMuted: true,
effectiveDeaf: false,
});
expect(volume).toBe(0);
});
it('returns unity gain at 100/100 when not deafened and not muted', () => {
const volume = computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: 100,
outputVolumePercent: 100,
locallyMuted: false,
effectiveDeaf: false,
});
expect(volume).toBeCloseTo(1, 5);
});
it('treats omitted effectiveDeaf as not deafened', () => {
const volume = computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: 100,
outputVolumePercent: 100,
locallyMuted: false,
});
expect(volume).toBeCloseTo(1, 5);
});
it.each(USER_VOLUME_PERCENTS)('stays within [0, 2] when undeafened at user volume %d', (percent) => {
const volume = computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: percent,
outputVolumePercent: VOICE_ENGINE_V2_VOLUME_MAX_PERCENT,
locallyMuted: false,
effectiveDeaf: false,
});
expect(volume).toBeGreaterThanOrEqual(0);
expect(volume).toBeLessThanOrEqual(2);
});
it('restores a positive volume after undeafen for an audible participant', () => {
const deafened = computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: 150,
outputVolumePercent: 100,
locallyMuted: false,
effectiveDeaf: true,
});
const restored = computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: 150,
outputVolumePercent: 100,
locallyMuted: false,
effectiveDeaf: false,
});
expect(deafened).toBe(0);
expect(restored).toBeGreaterThan(1);
});
it('forces volume 0 when streamMuted regardless of stream volume', () => {
const volume = computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: 100,
outputVolumePercent: 100,
locallyMuted: false,
effectiveDeaf: false,
streamVolumePercent: 100,
streamMuted: true,
});
expect(volume).toBe(0);
});
it('returns 0 when stream volume is 0', () => {
const volume = computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: 100,
outputVolumePercent: 100,
locallyMuted: false,
effectiveDeaf: false,
streamVolumePercent: 0,
});
expect(volume).toBe(0);
});
it('composes a reduced stream volume below unity gain', () => {
const volume = computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: 100,
outputVolumePercent: 100,
locallyMuted: false,
effectiveDeaf: false,
streamVolumePercent: 50,
});
expect(volume).toBeGreaterThan(0);
expect(volume).toBeLessThan(1);
});
it('treats omitted stream inputs as full stream volume', () => {
const withStream = computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: 100,
outputVolumePercent: 100,
locallyMuted: false,
effectiveDeaf: false,
streamVolumePercent: 100,
streamMuted: false,
});
const withoutStream = computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: 100,
outputVolumePercent: 100,
locallyMuted: false,
effectiveDeaf: false,
});
expect(withStream).toBeCloseTo(withoutStream, 5);
});
it('forces volume 0 when deafened even if stream is unmuted at full volume', () => {
const volume = computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: 100,
outputVolumePercent: 100,
locallyMuted: false,
effectiveDeaf: true,
streamVolumePercent: 100,
streamMuted: false,
});
expect(volume).toBe(0);
});
it('throws on non-boolean streamMuted', () => {
expect(() =>
computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: 100,
outputVolumePercent: 100,
locallyMuted: false,
streamMuted: 'no' as unknown as boolean,
}),
).toThrow();
});
it('throws on non-boolean locallyMuted', () => {
expect(() =>
computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: 100,
outputVolumePercent: 100,
locallyMuted: 'yes' as unknown as boolean,
}),
).toThrow();
});
it('throws on non-boolean effectiveDeaf', () => {
expect(() =>
computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: 100,
outputVolumePercent: 100,
locallyMuted: false,
effectiveDeaf: 1 as unknown as boolean,
}),
).toThrow();
});
});
@@ -1,58 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import assert from 'node:assert/strict';
export const VOICE_ENGINE_V2_VOLUME_MAX_PERCENT = 200;
const UNITY_GAIN_PERCENT = 100;
const QUIET_RANGE_EXPONENT = 1.2;
const PARTICIPANT_GAIN_MAX = 2;
export interface VoiceEngineV2NativeParticipantVolumeInput {
userVolumePercent: number;
outputVolumePercent: number;
locallyMuted: boolean;
effectiveDeaf?: boolean;
streamVolumePercent?: number;
streamMuted?: boolean;
}
export function clampVoiceEngineV2VolumePercent(value: number): number {
if (!Number.isFinite(value)) return UNITY_GAIN_PERCENT;
return Math.max(0, Math.min(VOICE_ENGINE_V2_VOLUME_MAX_PERCENT, value));
}
export function boostedVoiceEngineV2VolumePercentToTrackVolume(value: number): number {
const clamped = clampVoiceEngineV2VolumePercent(value);
if (clamped === 0) return 0;
if (clamped <= UNITY_GAIN_PERCENT) {
return 10 ** (((clamped - UNITY_GAIN_PERCENT) / UNITY_GAIN_PERCENT) * QUIET_RANGE_EXPONENT);
}
return 2 ** ((clamped - UNITY_GAIN_PERCENT) / (VOICE_ENGINE_V2_VOLUME_MAX_PERCENT - UNITY_GAIN_PERCENT));
}
export function computeVoiceEngineV2NativeParticipantVolume(input: VoiceEngineV2NativeParticipantVolumeInput): number {
assert.equal(typeof input.locallyMuted, 'boolean', 'locallyMuted must be a boolean');
if (input.effectiveDeaf !== undefined) {
assert.equal(typeof input.effectiveDeaf, 'boolean', 'effectiveDeaf must be a boolean when provided');
}
if (input.streamMuted !== undefined) {
assert.equal(typeof input.streamMuted, 'boolean', 'streamMuted must be a boolean when provided');
}
if (input.effectiveDeaf === true) return 0;
if (input.locallyMuted) return 0;
if (input.streamMuted === true) return 0;
const streamVolumePercent =
input.streamVolumePercent === undefined
? UNITY_GAIN_PERCENT
: clampVoiceEngineV2VolumePercent(input.streamVolumePercent);
const composedPercent =
(clampVoiceEngineV2VolumePercent(input.userVolumePercent) / UNITY_GAIN_PERCENT) *
(streamVolumePercent / UNITY_GAIN_PERCENT) *
clampVoiceEngineV2VolumePercent(input.outputVolumePercent);
const gain = boostedVoiceEngineV2VolumePercentToTrackVolume(clampVoiceEngineV2VolumePercent(composedPercent));
const volume = Math.max(0, Math.min(PARTICIPANT_GAIN_MAX, gain));
assert.ok(volume >= 0, 'participant volume must be non-negative');
assert.ok(volume <= PARTICIPANT_GAIN_MAX, 'participant volume must not exceed gain cap');
return volume;
}
@@ -2,12 +2,8 @@
import {describe, expect, it} from 'vitest';
import cameraShareEncodingPlanFixtures from '../../fixtures/policies/camera_share_encoding_plan.json';
import e2eeFixtures from '../../fixtures/policies/e2ee_state_normalization.json';
import hardwareEncoderCapabilityFixtures from '../../fixtures/policies/hardware_encoder_capabilities.json';
import localMediaReconnectFixtures from '../../fixtures/policies/local_media_reconnect_suppression.json';
import microphoneFailureFixtures from '../../fixtures/policies/microphone_failure_action.json';
import connectRetryFixtures from '../../fixtures/policies/native_connect_retry.json';
import volumeFixtures from '../../fixtures/policies/native_participant_volume.json';
import screenShareEncodingPlanFixtures from '../../fixtures/policies/screen_share_encoding_plan.json';
import voiceStatsCoercionFixtures from '../../fixtures/policies/voice_stats_coercion.json';
import voiceStatsSummaryFixtures from '../../fixtures/policies/voice_stats_summary.json';
@@ -22,31 +18,19 @@ import type {
import {
classifyVoiceEngineV2TrackStats,
coerceVoiceEngineV2Stats,
computeVoiceEngineV2NativeParticipantVolume,
getVoiceEngineV2MicrophoneOperationFailureAction,
hasVoiceEngineV2NativeNvencEncoder,
normalizeVoiceEngineV2HardwareEncoderCapabilities,
normalizeVoiceEngineV2ParticipantE2eeState,
planVoiceEngineV2CameraEncodingChange,
planVoiceEngineV2ScreenEncodingChange,
shouldRetryVoiceEngineV2NativeConnectTimeout,
shouldSuppressVoiceEngineV2LocalTrackStateDuringReconnect,
summarizeVoiceEngineV2Stats,
type VoiceEngineV2LocalTrackReconnectState,
type VoiceEngineV2MicrophoneFailureContext,
type VoiceEngineV2NativeConnectRetryPolicyInput,
type VoiceEngineV2OperationResultLike,
type VoiceEngineV2StatsSummary,
type VoiceEngineV2StatsTrackClassificationInput,
type VoiceEngineV2StatsTrackRoleSelection,
} from './index';
interface ConnectRetryFixture {
name: string;
input: VoiceEngineV2NativeConnectRetryPolicyInput;
expected: boolean;
}
interface MicrophoneFailureFixture {
name: string;
result: VoiceEngineV2OperationResultLike;
@@ -55,33 +39,6 @@ interface MicrophoneFailureFixture {
expected: string;
}
interface VolumeFixture {
name: string;
input: {
userVolumePercent: number | null;
outputVolumePercent: number | null;
locallyMuted: boolean;
};
expected?: number;
expectedRange?: {
minExclusive?: number;
maxExclusive?: number;
minInclusive?: number;
maxInclusive?: number;
};
}
interface LocalMediaReconnectFixture {
name: string;
input: VoiceEngineV2LocalTrackReconnectState;
expected: boolean;
}
interface E2eeFixture {
raw: unknown;
expected: string;
}
interface HardwareEncoderCapabilityFixture {
name: string;
input: unknown;
@@ -141,52 +98,12 @@ interface VoiceTrackClassificationFixture {
}
describe('voice engine v2 policies', () => {
it.each(connectRetryFixtures as Array<ConnectRetryFixture>)('replays native connect retry fixture: $name', ({
input,
expected,
}) => {
expect(shouldRetryVoiceEngineV2NativeConnectTimeout(input)).toBe(expected);
});
it.each(
microphoneFailureFixtures as Array<MicrophoneFailureFixture>,
)('replays microphone failure action fixture: $name', ({result, requestedEnabled, context, expected}) => {
expect(getVoiceEngineV2MicrophoneOperationFailureAction(result, requestedEnabled, context)).toBe(expected);
});
it.each(volumeFixtures as Array<VolumeFixture>)('replays native volume fixture: $name', (fixture) => {
const gain = computeVoiceEngineV2NativeParticipantVolume({
userVolumePercent: fixture.input.userVolumePercent ?? Number.NaN,
outputVolumePercent: fixture.input.outputVolumePercent ?? Number.NaN,
locallyMuted: fixture.input.locallyMuted,
});
if (fixture.expected !== undefined) {
expect(gain).toBeCloseTo(fixture.expected, 5);
}
if (fixture.expectedRange?.minExclusive !== undefined) {
expect(gain).toBeGreaterThan(fixture.expectedRange.minExclusive);
}
if (fixture.expectedRange?.maxExclusive !== undefined) {
expect(gain).toBeLessThan(fixture.expectedRange.maxExclusive);
}
if (fixture.expectedRange?.minInclusive !== undefined) {
expect(gain).toBeGreaterThanOrEqual(fixture.expectedRange.minInclusive);
}
if (fixture.expectedRange?.maxInclusive !== undefined) {
expect(gain).toBeLessThanOrEqual(fixture.expectedRange.maxInclusive);
}
});
it.each(
localMediaReconnectFixtures as Array<LocalMediaReconnectFixture>,
)('replays local media reconnect suppression fixture: $name', ({input, expected}) => {
expect(shouldSuppressVoiceEngineV2LocalTrackStateDuringReconnect(input)).toBe(expected);
});
it.each(e2eeFixtures as Array<E2eeFixture>)('replays E2EE state normalization fixture %#', ({raw, expected}) => {
expect(normalizeVoiceEngineV2ParticipantE2eeState(raw)).toBe(expected);
});
it.each(
hardwareEncoderCapabilityFixtures as Array<HardwareEncoderCapabilityFixture>,
)('replays hardware encoder capability fixture: $name', ({
@@ -39,7 +39,6 @@ import type {
VoiceEngineV2LiveKitRoomState,
VoiceEngineV2LocalStreamSource,
VoiceEngineV2MicrophoneOptions,
VoiceEngineV2NativeAudioDeviceModuleStatus,
VoiceEngineV2NativeAudioTapOptions,
VoiceEngineV2NativeCaptureFrame,
VoiceEngineV2NativeCaptureOptions,
@@ -198,11 +197,6 @@ export type VoiceEngineV2Event =
| {type: 'devices.selectAudioInputRequested'; deviceId: string | null}
| {type: 'devices.selectAudioOutputRequested'; deviceId: string | null}
| {type: 'devices.selectCameraRequested'; deviceId: string | null}
| {
type: 'nativeAudioDeviceModule.statusChanged';
status: VoiceEngineV2NativeAudioDeviceModuleStatus;
detail?: string | null;
}
| {type: 'audioControls.changed'; controls: VoiceEngineV2AudioControlsPatch}
| {type: 'nativeCapture.startRequested'; options: VoiceEngineV2NativeCaptureOptions}
| {type: 'nativeCapture.updateRequested'; options: VoiceEngineV2NativeCaptureOptions}
@@ -59,8 +59,6 @@ export type {
VoiceEngineV2MediaStatus,
VoiceEngineV2MicrophoneOptions,
VoiceEngineV2Model,
VoiceEngineV2NativeAudioDeviceModuleState,
VoiceEngineV2NativeAudioDeviceModuleStatus,
VoiceEngineV2NativeAudioTapOptions,
VoiceEngineV2NativeCaptureFrame,
VoiceEngineV2NativeCaptureKind,
@@ -11,7 +11,6 @@ export type VoiceEngineV2ConnectionStatus =
| 'failed';
export type VoiceEngineV2MediaStatus = 'idle' | 'publishing' | 'published' | 'unpublishing' | 'failed';
export type VoiceEngineV2NativeAudioDeviceModuleStatus = 'unknown' | 'unsupported' | 'warming' | 'ready' | 'failed';
export type VoiceEngineV2OperationId = number;
@@ -175,11 +174,6 @@ export interface VoiceEngineV2LiveKitRoomState {
serverRegion: string | null;
}
export interface VoiceEngineV2NativeAudioDeviceModuleState {
status: VoiceEngineV2NativeAudioDeviceModuleStatus;
detail: string | null;
}
export interface VoiceEngineV2MicrophoneOptions {
deviceId?: string;
echoCancellation?: boolean;
@@ -1,29 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {bench, describe} from 'vitest';
import type {VoiceEngineV2InboundVideoFrame} from '../protocol/types';
import {VoiceEngineV2FrameStatsAccumulator} from './frameStatsBatching';
const frame: VoiceEngineV2InboundVideoFrame = Object.freeze({
participantSid: 'sid-remote',
participantIdentity: 'identity-remote',
trackSid: 'track-video-1',
width: 1280,
height: 720,
timestampUs: 1_000_000,
byteLength: 320_000,
});
const recordAccumulator = new VoiceEngineV2FrameStatsAccumulator();
const flushAccumulator = new VoiceEngineV2FrameStatsAccumulator();
describe('voice engine v2 frame stats accumulator hot paths', () => {
bench('record per-frame update', () => {
recordAccumulator.record(frame);
});
bench('record then flush one dirty track', () => {
flushAccumulator.record(frame);
flushAccumulator.flushDirty(() => {});
});
});
@@ -1,129 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {describe, expect, it} from 'vitest';
import type {VoiceEngineV2InboundVideoFrame} from '../protocol/types';
import {
VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP,
VoiceEngineV2FrameStatsAccumulator,
type VoiceEngineV2FrameStatsEvent,
} from './frameStatsBatching';
function makeFrame(overrides: Partial<VoiceEngineV2InboundVideoFrame> = {}): VoiceEngineV2InboundVideoFrame {
return {
participantSid: 'PA_alice',
participantIdentity: 'alice',
trackSid: 'TR_screen',
width: 1280,
height: 720,
timestampUs: 1000,
byteLength: 1_382_400,
...overrides,
};
}
function collectFlush(accumulator: VoiceEngineV2FrameStatsAccumulator): Array<VoiceEngineV2FrameStatsEvent> {
const events: Array<VoiceEngineV2FrameStatsEvent> = [];
accumulator.flushDirty((event) => events.push(event));
return events;
}
describe('voice engine v2 frame stats accumulator', () => {
it('accumulates a cumulative frame count and flushes one absolute event per dirty track', () => {
const accumulator = new VoiceEngineV2FrameStatsAccumulator();
for (let index = 1; index <= 30; index += 1) {
expect(accumulator.record(makeFrame({timestampUs: index * 33_333}))).toBe(true);
}
expect(accumulator.trackedTracksCount).toBe(1);
expect(accumulator.dirtyTracksCount).toBe(1);
const events = collectFlush(accumulator);
expect(events).toEqual([
{
type: 'inboundVideo.frameStats',
stats: {
participantSid: 'PA_alice',
participantIdentity: 'alice',
trackSid: 'TR_screen',
width: 1280,
height: 720,
frameCount: 30,
lastFrameTimestampUs: 30 * 33_333,
lastFrameByteLength: 1_382_400,
},
},
]);
expect(accumulator.dirtyTracksCount).toBe(0);
expect(collectFlush(accumulator)).toEqual([]);
});
it('keeps the cumulative count across flushes and reuses the same record without re-allocating slots', () => {
const accumulator = new VoiceEngineV2FrameStatsAccumulator();
accumulator.record(makeFrame({timestampUs: 1000}));
collectFlush(accumulator);
accumulator.record(makeFrame({timestampUs: 2000, width: 640, height: 360, byteLength: undefined}));
const events = collectFlush(accumulator);
expect(events).toHaveLength(1);
expect(events[0]?.stats).toMatchObject({
frameCount: 2,
lastFrameTimestampUs: 2000,
lastFrameByteLength: null,
width: 640,
height: 360,
});
expect(accumulator.trackedTracksCount).toBe(1);
});
it('omits participantIdentity when never observed for the track', () => {
const accumulator = new VoiceEngineV2FrameStatsAccumulator();
accumulator.record(makeFrame({participantIdentity: undefined}));
const events = collectFlush(accumulator);
expect(events).toHaveLength(1);
expect(events[0]?.stats).not.toHaveProperty('participantIdentity');
});
it('drops updates for new tracks at the cap and counts the drops', () => {
const accumulator = new VoiceEngineV2FrameStatsAccumulator();
for (let index = 0; index < VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP; index += 1) {
expect(accumulator.record(makeFrame({trackSid: `TR_${index}`}))).toBe(true);
}
expect(accumulator.trackedTracksCount).toBe(VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP);
expect(accumulator.record(makeFrame({trackSid: 'TR_overflow'}))).toBe(false);
expect(accumulator.droppedUpdatesCount).toBe(1);
expect(accumulator.record(makeFrame({trackSid: 'TR_0'}))).toBe(true);
expect(accumulator.trackedTracksCount).toBe(VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP);
accumulator.removeTrack('TR_1');
expect(accumulator.record(makeFrame({trackSid: 'TR_overflow'}))).toBe(true);
expect(accumulator.trackedTracksCount).toBe(VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP);
});
it('removes dirty bookkeeping when a tracked track is removed before a flush', () => {
const accumulator = new VoiceEngineV2FrameStatsAccumulator();
accumulator.record(makeFrame({trackSid: 'TR_a'}));
accumulator.record(makeFrame({trackSid: 'TR_b'}));
expect(accumulator.dirtyTracksCount).toBe(2);
accumulator.removeTrack('TR_a');
expect(accumulator.dirtyTracksCount).toBe(1);
const events = collectFlush(accumulator);
expect(events).toHaveLength(1);
expect(events[0]?.stats.trackSid).toBe('TR_b');
});
it('clear resets records, dirty bookkeeping, and keeps the dropped counter', () => {
const accumulator = new VoiceEngineV2FrameStatsAccumulator();
for (let index = 0; index <= VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP; index += 1) {
accumulator.record(makeFrame({trackSid: `TR_${index}`}));
}
expect(accumulator.droppedUpdatesCount).toBe(1);
accumulator.clear();
expect(accumulator.trackedTracksCount).toBe(0);
expect(accumulator.dirtyTracksCount).toBe(0);
expect(accumulator.droppedUpdatesCount).toBe(1);
expect(collectFlush(accumulator)).toEqual([]);
});
});
@@ -1,129 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import assert from 'node:assert/strict';
import type {VoiceEngineV2Event} from '../protocol/events';
import type {VoiceEngineV2InboundVideoFrame} from '../protocol/types';
import {VOICE_ENGINE_V2_COALESCED_TRACKS_CAP} from './frameCoalescing';
export const VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP = VOICE_ENGINE_V2_COALESCED_TRACKS_CAP;
export const VOICE_ENGINE_V2_FRAME_STATS_FLUSH_INTERVAL_MS = 1000;
export type VoiceEngineV2FrameStatsEvent = Extract<VoiceEngineV2Event, {type: 'inboundVideo.frameStats'}>;
interface VoiceEngineV2FrameStatsRecord {
participantSid: string;
participantIdentity: string | null;
trackSid: string;
width: number;
height: number;
frameCount: number;
lastFrameTimestampUs: number;
lastFrameByteLength: number | null;
dirty: boolean;
}
export class VoiceEngineV2FrameStatsAccumulator {
private readonly recordsByTrack = new Map<string, VoiceEngineV2FrameStatsRecord>();
private droppedUpdatesCountValue = 0;
private dirtyTracksCountValue = 0;
record(frame: VoiceEngineV2InboundVideoFrame): boolean {
assert.ok(frame != null, 'record requires a non-null frame');
assert.equal(typeof frame.trackSid, 'string', 'record requires a string trackSid');
let entry = this.recordsByTrack.get(frame.trackSid);
if (entry === undefined) {
if (this.recordsByTrack.size >= VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP) {
this.droppedUpdatesCountValue += 1;
assert.ok(this.droppedUpdatesCountValue >= 1, 'dropped updates count must stay positive after a drop');
return false;
}
entry = {
participantSid: frame.participantSid,
participantIdentity: frame.participantIdentity ?? null,
trackSid: frame.trackSid,
width: frame.width,
height: frame.height,
frameCount: 0,
lastFrameTimestampUs: frame.timestampUs,
lastFrameByteLength: frame.byteLength ?? null,
dirty: false,
};
this.recordsByTrack.set(frame.trackSid, entry);
assert.ok(this.recordsByTrack.size <= VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP, 'frame stats records exceeded cap');
}
entry.participantSid = frame.participantSid;
entry.participantIdentity = frame.participantIdentity ?? entry.participantIdentity;
entry.width = frame.width;
entry.height = frame.height;
entry.frameCount += 1;
entry.lastFrameTimestampUs = frame.timestampUs;
entry.lastFrameByteLength = frame.byteLength ?? null;
if (!entry.dirty) {
entry.dirty = true;
this.dirtyTracksCountValue += 1;
}
assert.ok(this.dirtyTracksCountValue <= this.recordsByTrack.size, 'dirty count must not exceed tracked records');
return true;
}
flushDirty(emit: (event: VoiceEngineV2FrameStatsEvent) => void): number {
assert.equal(typeof emit, 'function', 'flushDirty requires a function emitter');
assert.ok(this.dirtyTracksCountValue >= 0, 'dirty count must be non-negative before flush');
let flushedCount = 0;
for (const entry of this.recordsByTrack.values()) {
if (!entry.dirty) continue;
entry.dirty = false;
flushedCount += 1;
emit({
type: 'inboundVideo.frameStats',
stats: {
participantSid: entry.participantSid,
...(entry.participantIdentity !== null ? {participantIdentity: entry.participantIdentity} : {}),
trackSid: entry.trackSid,
width: entry.width,
height: entry.height,
frameCount: entry.frameCount,
lastFrameTimestampUs: entry.lastFrameTimestampUs,
lastFrameByteLength: entry.lastFrameByteLength,
},
});
}
assert.equal(flushedCount, this.dirtyTracksCountValue, 'flush must visit every dirty record exactly once');
this.dirtyTracksCountValue = 0;
return flushedCount;
}
removeTrack(trackSid: string): void {
assert.equal(typeof trackSid, 'string', 'removeTrack requires a string trackSid');
assert.ok(trackSid.length > 0, 'removeTrack requires a non-empty trackSid');
const entry = this.recordsByTrack.get(trackSid);
if (entry === undefined) return;
if (entry.dirty) {
this.dirtyTracksCountValue -= 1;
assert.ok(this.dirtyTracksCountValue >= 0, 'dirty count must stay non-negative after removal');
}
this.recordsByTrack.delete(trackSid);
}
clear(): void {
this.recordsByTrack.clear();
this.dirtyTracksCountValue = 0;
assert.equal(this.recordsByTrack.size, 0, 'clear must remove every tracked record');
assert.equal(this.dirtyTracksCountValue, 0, 'clear must reset the dirty count');
}
get trackedTracksCount(): number {
assert.ok(this.recordsByTrack.size <= VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP, 'frame stats records exceeded cap');
return this.recordsByTrack.size;
}
get dirtyTracksCount(): number {
assert.ok(this.dirtyTracksCountValue >= 0, 'dirty count must be non-negative');
return this.dirtyTracksCountValue;
}
get droppedUpdatesCount(): number {
assert.ok(this.droppedUpdatesCountValue >= 0, 'dropped updates count must be non-negative');
return this.droppedUpdatesCountValue;
}
}
@@ -15,12 +15,6 @@ export {
isVoiceEngineV2FrameReceivedEvent,
VOICE_ENGINE_V2_COALESCED_TRACKS_CAP,
} from './frameCoalescing';
export type {VoiceEngineV2FrameStatsEvent} from './frameStatsBatching';
export {
VOICE_ENGINE_V2_FRAME_STATS_FLUSH_INTERVAL_MS,
VOICE_ENGINE_V2_FRAME_STATS_TRACKS_CAP,
VoiceEngineV2FrameStatsAccumulator,
} from './frameStatsBatching';
export type {
VoiceEngineV2ClockPort,
VoiceEngineV2EntropySource,
@@ -5,7 +5,6 @@ import appVoiceSessionFixtureJson from '../../fixtures/event_logs/app_voice_sess
import type {VoiceEngineV2Event} from '../protocol/events';
import {createVoiceEngineV2MemoryEventLogSpillSink} from '../runtime/eventLogRing';
import {coalesceVoiceEngineV2EventSequence} from '../runtime/frameCoalescing';
import {VoiceEngineV2FrameStatsAccumulator} from '../runtime/frameStatsBatching';
import {VoiceEngineV2Runtime} from '../runtime/VoiceEngineV2Runtime';
import {
replayVoiceEngineV2EventLogFixture,
@@ -127,45 +126,6 @@ describe('voice engine v2 event-log fixture replay', () => {
expect(replay.finalSnapshot.inboundVideo.tracks.TR_screen?.frameCount).toBe(1);
});
it('replays a batched frame-stats log to the identical snapshot with a true cumulative frame count', () => {
const accumulator = new VoiceEngineV2FrameStatsAccumulator();
const runtime = makeCoalescingRuntime();
runtime.dispatch({
type: 'inboundVideo.trackSubscribed',
track: {participantSid: 'PA_alice', participantIdentity: 'alice', trackSid: 'TR_screen', source: 'screen'},
});
const burst = 30;
for (let index = 1; index <= burst; index += 1) {
accumulator.record({
participantSid: 'PA_alice',
participantIdentity: 'alice',
trackSid: 'TR_screen',
width: 1280,
height: 720,
timestampUs: index * 33_333,
byteLength: 1_382_400,
});
}
const flushedCount = accumulator.flushDirty((event) => {
runtime.dispatch(event);
});
expect(flushedCount).toBe(1);
expect(runtime.eventLog).toHaveLength(2);
expect(runtime.snapshot.inboundVideo.tracks.TR_screen?.frameCount).toBe(burst);
expect(runtime.snapshot.inboundVideo.tracks.TR_screen?.lastFrameTimestampUs).toBe(burst * 33_333);
const fixture: VoiceEngineV2EventLogFixture = {
version: VOICE_ENGINE_V2_EVENT_LOG_FIXTURE_VERSION,
name: 'batched_inbound_video_frame_stats',
steps: runtime.eventLog.map((entry) => ({event: entry.event, commands: entry.commands})),
expected: {finalSnapshot: runtime.snapshot},
};
const replay = replayVoiceEngineV2EventLogFixture(fixture);
expect(replay.finalSnapshot).toEqual(runtime.snapshot);
});
it('replays the same coalesced log deterministically across runtimes', () => {
const rawEvents = inboundVideoBurstEvents();
const first = makeCoalescingRuntime();