fix(voice): scope disconnects, correct VAD and stream lifecycle (#2597)

This commit is contained in:
Hampus authored and GitHub committed 2026-09-08 19:06:42 +02:00
1 parent 667ac7da8e
commit 20a15ac11d
15 files changed
+1069 -36

No files matched your search

@@ -121,7 +121,7 @@ describe('MediaEngineFacadeStateMachine', () => {
expect(snapshot.context.pendingServerDisconnectConnectionId).toBeNull();
});
it('disconnects immediately when the server removes the current local voice state', () => {
it('disconnects immediately when the server removes the current local voice state without a live transport', () => {
expect(
shouldImmediatelyDisconnectMediaEngineForServerVoiceStateRemoval({
voiceStateConnectionId: 'connection-1',
@@ -131,6 +131,28 @@ describe('MediaEngineFacadeStateMachine', () => {
connected: true,
connecting: false,
}),
).toBe(false);
expect(
shouldImmediatelyDisconnectMediaEngineForServerVoiceStateRemoval({
voiceStateConnectionId: 'connection-1',
voiceStateChannelId: null,
currentConnectionId: 'connection-1',
currentChannelId: 'channel-1',
connected: false,
connecting: true,
}),
).toBe(true);
expect(
shouldImmediatelyDisconnectMediaEngineForServerVoiceStateRemoval({
voiceStateConnectionId: 'connection-1',
voiceStateChannelId: null,
currentConnectionId: 'connection-1',
currentChannelId: 'channel-1',
connected: false,
connecting: false,
}),
).toBe(true);
expect(
@@ -156,6 +178,50 @@ describe('MediaEngineFacadeStateMachine', () => {
).toBe(false);
});
it('keeps the transport during a server move until the replacement connection arrives', () => {
let snapshot = createMediaEngineFacadeSnapshot();
expect(
shouldImmediatelyDisconnectMediaEngineForServerVoiceStateRemoval({
voiceStateConnectionId: 'connection-1',
voiceStateChannelId: null,
currentConnectionId: 'connection-1',
currentChannelId: 'channel-1',
connected: true,
connecting: false,
}),
).toBe(false);
expect(
shouldCancelMediaEngineReconnectForServerVoiceStateRemoval({
voiceStateConnectionId: 'connection-1',
voiceStateChannelId: null,
currentConnectionId: 'connection-1',
currentChannelId: 'channel-1',
connected: true,
connecting: false,
}),
).toBe(false);
snapshot = transitionMediaEngineFacadeSnapshot(snapshot, {
type: 'serverDisconnect.schedule',
connectionId: 'connection-1',
});
expect(
shouldRunMediaEngineDeferredDisconnect(snapshot, {
connectionId: 'connection-1',
currentConnectionId: 'connection-2',
connected: true,
currentVoiceStateChannelId: null,
}),
).toBe(false);
expect(
shouldRunMediaEngineDeferredDisconnect(snapshot, {
connectionId: 'connection-1',
currentConnectionId: 'connection-1',
connected: true,
currentVoiceStateChannelId: null,
}),
).toBe(true);
});
it('cancels reconnect when local voice-state removal arrives after transport disconnect', () => {
expect(
shouldCancelMediaEngineReconnectForServerVoiceStateRemoval({
@@ -527,7 +527,8 @@ export function shouldImmediatelyDisconnectMediaEngineForServerVoiceStateRemoval
input: MediaEngineFacadeServerVoiceStateRemovalInput,
): boolean {
if (!isCurrentServerVoiceStateRemoval(input)) return false;
return input.connected || input.connecting || input.currentChannelId != null;
if (input.connected) return false;
return input.connecting || input.currentChannelId != null;
}
export function shouldCancelMediaEngineReconnectForServerVoiceStateRemoval(
@@ -1,7 +1,21 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
getLocalSpeakingThresholdRms,
LOCAL_MAX_RMS,
LOCAL_MIN_RMS,
SPEAKING_LOCAL_RELEASE_MS,
} from '@app/features/voice/engine/VoiceSpeakingThreshold';
import {describe, expect, it} from 'vitest';
import {type LocalSpeakingOverrideInput, resolveLocalSpeakingOverrideState} from './VoiceLocalSpeakingGate';
import {
type LocalSpeakingOverrideInput,
resolveLocalSpeakingOverrideState,
resolveVoiceActivityGateState,
resolveVoiceActivityThresholdRms,
updateVoiceActivityNoiseFloorRms,
type VoiceActivityGateInput,
type VoiceActivityGateState,
} from './VoiceLocalSpeakingGate';
const UNMUTED_OPEN_MIC: LocalSpeakingOverrideInput = {
pushToTalkActive: false,
@@ -70,3 +84,131 @@ describe('resolveLocalSpeakingOverrideState', () => {
).toBe(false);
});
});
const GATE_TICK_MS = 50;
const KEYBOARD_NOISE_RMS = 0.012;
const SPEECH_RMS = 0.06;
function runGate(samples: ReadonlyArray<number>, thresholdRms: number): Array<boolean> {
let open = false;
let silenceStartedAtMs: number | null = null;
return samples.map((rms, index) => {
const input: VoiceActivityGateInput = {
rms,
thresholdRms,
nowMs: index * GATE_TICK_MS,
silenceStartedAtMs,
gateOpen: open,
releaseDelayMs: SPEAKING_LOCAL_RELEASE_MS,
};
const next = resolveVoiceActivityGateState(input);
open = next.open;
silenceStartedAtMs = next.silenceStartedAtMs;
return open;
});
}
describe('resolveVoiceActivityGateState', () => {
it('keeps keyboard noise off the wire when the activity threshold is raised', () => {
const opened = runGate(new Array(20).fill(KEYBOARD_NOISE_RMS), getLocalSpeakingThresholdRms(100));
expect(opened.some((open) => open)).toBe(false);
});
it('still transmits the same noise when the activity threshold is lowered', () => {
const opened = runGate(new Array(20).fill(KEYBOARD_NOISE_RMS), getLocalSpeakingThresholdRms(0));
expect(opened.every((open) => open)).toBe(true);
});
it('opens on the first sample at or above the threshold and clears the silence clock', () => {
const expected: VoiceActivityGateState = {open: true, silenceStartedAtMs: null};
expect(
resolveVoiceActivityGateState({
rms: SPEECH_RMS,
thresholdRms: getLocalSpeakingThresholdRms(100),
nowMs: 1_000,
silenceStartedAtMs: 800,
gateOpen: false,
releaseDelayMs: SPEAKING_LOCAL_RELEASE_MS,
}),
).toEqual(expected);
});
it('holds the gate open for the release window before closing', () => {
const threshold = getLocalSpeakingThresholdRms(100);
expect(
resolveVoiceActivityGateState({
rms: KEYBOARD_NOISE_RMS,
thresholdRms: threshold,
nowMs: 1_000,
silenceStartedAtMs: null,
gateOpen: true,
releaseDelayMs: SPEAKING_LOCAL_RELEASE_MS,
}),
).toEqual({open: true, silenceStartedAtMs: 1_000});
expect(
resolveVoiceActivityGateState({
rms: KEYBOARD_NOISE_RMS,
thresholdRms: threshold,
nowMs: 1_000 + SPEAKING_LOCAL_RELEASE_MS - 1,
silenceStartedAtMs: 1_000,
gateOpen: true,
releaseDelayMs: SPEAKING_LOCAL_RELEASE_MS,
}).open,
).toBe(true);
expect(
resolveVoiceActivityGateState({
rms: KEYBOARD_NOISE_RMS,
thresholdRms: threshold,
nowMs: 1_000 + SPEAKING_LOCAL_RELEASE_MS,
silenceStartedAtMs: 1_000,
gateOpen: true,
releaseDelayMs: SPEAKING_LOCAL_RELEASE_MS,
}).open,
).toBe(false);
});
it('closes after speech stops and reopens when speech returns', () => {
const threshold = getLocalSpeakingThresholdRms(100);
const opened = runGate(
[SPEECH_RMS, SPEECH_RMS, ...new Array(6).fill(KEYBOARD_NOISE_RMS), SPEECH_RMS, KEYBOARD_NOISE_RMS],
threshold,
);
expect(opened).toEqual([true, true, true, true, true, true, false, false, true, true]);
});
});
describe('resolveVoiceActivityThresholdRms', () => {
it('follows the activity threshold slider when auto sensitivity is off', () => {
expect(resolveVoiceActivityThresholdRms({autoSensitivity: false, vadThreshold: 100, noiseFloorRms: 0})).toBe(
getLocalSpeakingThresholdRms(100),
);
expect(resolveVoiceActivityThresholdRms({autoSensitivity: false, vadThreshold: 0, noiseFloorRms: 0})).toBe(
getLocalSpeakingThresholdRms(0),
);
});
it('ignores the slider and tracks the measured noise floor when auto sensitivity is on', () => {
expect(
resolveVoiceActivityThresholdRms({autoSensitivity: true, vadThreshold: 0, noiseFloorRms: 0.005}),
).toBeCloseTo(0.015, 6);
});
it('clamps the auto threshold to the local speaking range', () => {
expect(resolveVoiceActivityThresholdRms({autoSensitivity: true, vadThreshold: 50, noiseFloorRms: 0})).toBe(
LOCAL_MIN_RMS,
);
expect(resolveVoiceActivityThresholdRms({autoSensitivity: true, vadThreshold: 50, noiseFloorRms: 1})).toBe(
LOCAL_MAX_RMS,
);
});
});
describe('updateVoiceActivityNoiseFloorRms', () => {
it('converges on the level measured while the gate is closed', () => {
let noiseFloorRms = 0;
for (let i = 0; i < 200; i++) {
noiseFloorRms = updateVoiceActivityNoiseFloorRms(noiseFloorRms, KEYBOARD_NOISE_RMS);
}
expect(noiseFloorRms).toBeCloseTo(KEYBOARD_NOISE_RMS, 4);
});
});
@@ -1,5 +1,14 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
getLocalSpeakingThresholdRms,
LOCAL_MAX_RMS,
LOCAL_MIN_RMS,
} from '@app/features/voice/engine/VoiceSpeakingThreshold';
const VOICE_ACTIVITY_NOISE_FLOOR_HEADROOM = 3;
const VOICE_ACTIVITY_NOISE_FLOOR_SMOOTHING = 0.05;
export interface LocalSpeakingOverrideInput {
pushToTalkActive: boolean;
pushToMuteActive: boolean;
@@ -18,3 +27,45 @@ export function resolveLocalSpeakingOverrideState(input: LocalSpeakingOverrideIn
if (effectivelyMuted) return false;
return null;
}
export interface VoiceActivityGateInput {
rms: number;
thresholdRms: number;
nowMs: number;
silenceStartedAtMs: number | null;
gateOpen: boolean;
releaseDelayMs: number;
}
export interface VoiceActivityGateState {
open: boolean;
silenceStartedAtMs: number | null;
}
export function resolveVoiceActivityGateState(input: VoiceActivityGateInput): VoiceActivityGateState {
if (input.rms >= input.thresholdRms) {
return {open: true, silenceStartedAtMs: null};
}
const silenceStartedAtMs = input.silenceStartedAtMs ?? input.nowMs;
return {
open: input.gateOpen && input.nowMs - silenceStartedAtMs < input.releaseDelayMs,
silenceStartedAtMs,
};
}
export function resolveVoiceActivityThresholdRms(input: {
autoSensitivity: boolean;
vadThreshold: number;
noiseFloorRms: number;
}): number {
if (!input.autoSensitivity) {
return getLocalSpeakingThresholdRms(input.vadThreshold);
}
const tracked = input.noiseFloorRms * VOICE_ACTIVITY_NOISE_FLOOR_HEADROOM;
if (!Number.isFinite(tracked)) return LOCAL_MIN_RMS;
return Math.min(LOCAL_MAX_RMS, Math.max(LOCAL_MIN_RMS, tracked));
}
export function updateVoiceActivityNoiseFloorRms(noiseFloorRms: number, rms: number): number {
return noiseFloorRms * (1 - VOICE_ACTIVITY_NOISE_FLOOR_SMOOTHING) + rms * VOICE_ACTIVITY_NOISE_FLOOR_SMOOTHING;
}
@@ -1,8 +1,8 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
const LOCAL_MIN_RMS = 0.003;
export const LOCAL_MIN_RMS = 0.003;
const LOCAL_DEFAULT_RMS = 0.008;
const LOCAL_MAX_RMS = 0.04;
export const LOCAL_MAX_RMS = 0.04;
const REMOTE_MIN_RMS = 0.002;
const REMOTE_DEFAULT_RMS = 0.006;
const REMOTE_MAX_RMS = 0.035;
@@ -3,6 +3,7 @@
import assert from 'node:assert/strict';
import VoiceDevicePermissionState from '@app/features/voice/engine/VoiceDevicePermissionState';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {isVoiceActivityGateEnabled} from '@app/features/voice/utils/VoiceInputProcessor';
import {
getActiveInputDeviceLabel,
resolveVoiceProcessingFromStateForDeviceLabel,
@@ -13,6 +14,8 @@ export interface VoiceEngineV2AppAudioSettingsSnapshot {
readonly inputVolume: number;
readonly outputVolume: number;
readonly vadThreshold: number;
readonly vadAutoSensitivity: boolean;
readonly voiceActivityGate: boolean;
readonly requestedInputDeviceId: string;
readonly effectiveInputDeviceId: string;
readonly activeInputDeviceLabel: string | null;
@@ -39,6 +42,8 @@ function assertAudioSettingsSnapshot(snapshot: VoiceEngineV2AppAudioSettingsSnap
assert.equal(typeof snapshot.inputVolume, 'number', `${name}.inputVolume must be a number`);
assert.equal(typeof snapshot.outputVolume, 'number', `${name}.outputVolume must be a number`);
assert.equal(typeof snapshot.vadThreshold, 'number', `${name}.vadThreshold must be a number`);
assert.equal(typeof snapshot.vadAutoSensitivity, 'boolean', `${name}.vadAutoSensitivity must be a boolean`);
assert.equal(typeof snapshot.voiceActivityGate, 'boolean', `${name}.voiceActivityGate must be a boolean`);
assert.equal(typeof snapshot.requestedInputDeviceId, 'string', `${name}.requestedInputDeviceId must be a string`);
assert.equal(typeof snapshot.effectiveInputDeviceId, 'string', `${name}.effectiveInputDeviceId must be a string`);
assert.equal(typeof snapshot.echoCancellation, 'boolean', `${name}.echoCancellation must be a boolean`);
@@ -54,6 +59,8 @@ export function createVoiceEngineV2AppAudioSettingsSnapshot(): VoiceEngineV2AppA
inputVolume: VoiceSettings.getInputVolume(),
outputVolume: VoiceSettings.getOutputVolume(),
vadThreshold: VoiceSettings.getVadThreshold(),
vadAutoSensitivity: VoiceSettings.getVadAutoSensitivity(),
voiceActivityGate: isVoiceActivityGateEnabled(),
requestedInputDeviceId: VoiceSettings.getInputDeviceId(),
effectiveInputDeviceId: resolveEffectiveInputDeviceId(),
activeInputDeviceLabel,
@@ -90,5 +97,7 @@ export function hasVoiceEngineV2InputProcessorSettingsChanged(
assertAudioSettingsSnapshot(current, 'current');
if (previous.deepFilter !== current.deepFilter) return true;
if (previous.deepFilterNoiseReductionLevel !== current.deepFilterNoiseReductionLevel) return true;
if (previous.voiceActivityGate !== current.voiceActivityGate) return true;
if (previous.vadAutoSensitivity !== current.vadAutoSensitivity) return true;
return false;
}
@@ -806,7 +806,12 @@ export class VoiceEngineV2AppMediaExecutionAdapter extends Store {
assertObjectLike<VoiceEngineV2MicrophoneOptions>(options, 'enableMicrophone.options');
if (this.microphoneEnablePromise) {
await this.microphoneEnablePromise;
return;
if (this.hasLiveMicrophonePublication(room) || this.microphoneEnablePromise !== null) {
return;
}
logger.warn('Coalesced microphone enable left no live publication for this room; enabling again', {
channelId,
});
}
this.microphoneEnablePromise = this.enableMicrophoneNow(room, channelId, options);
try {
@@ -1524,6 +1529,9 @@ export class VoiceEngineV2AppMediaExecutionAdapter extends Store {
this.syncVoiceState({self_mute: targetMute});
this.updateMediaAudioControls();
this.syncLocalSpeakingOverride(room);
void this.refreshLocalVoiceInputProcessor(room).catch((error) => {
logger.warn('Failed to refresh voice input processor after transmit mode change', {error});
});
}
getMuteReason(voiceState: VoiceState | null, guildId?: string | null, channelId?: string | null): VoiceMuteReason {
@@ -0,0 +1,146 @@
// @vitest-environment happy-dom
// SPDX-License-Identifier: AGPL-3.0-or-later
import {installVoiceMenuTestBootstrap} from '@app/features/ui/action_menu/items/__fixtures__/VoiceMenuTestBootstrap';
import type {Room} from 'livekit-client';
import {expect, test, vi} from 'vitest';
vi.mock('@lingui/core/macro', () => {
const descriptor = (value: unknown): unknown => (typeof value === 'string' ? {message: value} : value);
return {msg: descriptor, t: descriptor, plural: () => '', select: () => '', selectOrdinal: () => ''};
});
vi.mock('@lingui/react/macro', () => ({
Trans: () => null,
useLingui: () => ({i18n: {_: (descriptor: {message?: string}) => descriptor.message ?? '', locale: 'en'}}),
}));
installVoiceMenuTestBootstrap();
await import('@app/features/voice/engine/MediaEngineFacade');
const {VoiceEngineV2AppMediaExecutionAdapter} = await import(
'@app/features/voice/engine/v2/VoiceEngineV2AppMediaExecutionAdapter'
);
interface FakeRoom {
audioTrackPublications: Map<string, unknown>;
localParticipant: {audioTrackPublications: Map<string, unknown>};
}
function createFakeRoom(): FakeRoom {
const audioTrackPublications = new Map<string, unknown>();
return {audioTrackPublications, localParticipant: {audioTrackPublications}};
}
function publishMicrophone(room: FakeRoom): void {
room.audioTrackPublications.set('mic', {
source: 'microphone',
track: {mediaStreamTrack: {readyState: 'live'}},
});
}
function unpublishMicrophone(room: FakeRoom): void {
room.audioTrackPublications.delete('mic');
}
function asRoom(room: FakeRoom): Room {
return room as unknown as Room;
}
function createDeferred(): {promise: Promise<void>; resolve: () => void} {
let resolve = (): void => undefined;
const promise = new Promise<void>((res) => {
resolve = () => res();
});
return {promise, resolve};
}
function createAdapter(onEnable: (room: FakeRoom, channelId: string | null) => Promise<void>): {
adapter: InstanceType<typeof VoiceEngineV2AppMediaExecutionAdapter>;
calls: Array<{room: FakeRoom; channelId: string | null}>;
} {
const adapter = new VoiceEngineV2AppMediaExecutionAdapter();
const calls: Array<{room: FakeRoom; channelId: string | null}> = [];
(
adapter as unknown as {enableMicrophoneNow: (room: Room, channelId: string | null) => Promise<void>}
).enableMicrophoneNow = async (room: Room, channelId: string | null): Promise<void> => {
const fakeRoom = room as unknown as FakeRoom;
calls.push({room: fakeRoom, channelId});
await onEnable(fakeRoom, channelId);
};
return {adapter, calls};
}
test('an enable for a second room is not satisfied by an enable in flight for the first room', async () => {
const roomA = createFakeRoom();
const roomB = createFakeRoom();
const gate = createDeferred();
const {adapter, calls} = createAdapter(async (room) => {
await gate.promise;
publishMicrophone(room);
});
const first = adapter.enableMicrophone(asRoom(roomA), 'channel-a');
await Promise.resolve();
const second = adapter.enableMicrophone(asRoom(roomB), 'channel-b');
gate.resolve();
await Promise.all([first, second]);
expect(calls.map((call) => call.channelId)).toEqual(['channel-a', 'channel-b']);
expect(roomA.audioTrackPublications.size).toBe(1);
expect(roomB.audioTrackPublications.size).toBe(1);
});
test('an enable is not satisfied by an in-flight enable whose publication was already unpublished', async () => {
const room = createFakeRoom();
const gate = createDeferred();
const {adapter, calls} = createAdapter(async (target) => {
publishMicrophone(target);
await gate.promise;
});
const first = adapter.enableMicrophone(asRoom(room), 'channel-a');
await Promise.resolve();
unpublishMicrophone(room);
const second = adapter.enableMicrophone(asRoom(room), 'channel-a');
gate.resolve();
await Promise.all([first, second]);
expect(calls).toHaveLength(2);
expect(room.audioTrackPublications.size).toBe(1);
});
test('a coalesced enable that already has a live publication does not enable again', async () => {
const room = createFakeRoom();
const gate = createDeferred();
const {adapter, calls} = createAdapter(async (target) => {
publishMicrophone(target);
await gate.promise;
});
const first = adapter.enableMicrophone(asRoom(room), 'channel-a');
await Promise.resolve();
const second = adapter.enableMicrophone(asRoom(room), 'channel-a');
gate.resolve();
await Promise.all([first, second]);
expect(calls).toHaveLength(1);
expect(room.audioTrackPublications.size).toBe(1);
});
test('a rejection from the in-flight enable still propagates to the coalesced caller', async () => {
const room = createFakeRoom();
const gate = createDeferred();
const {adapter, calls} = createAdapter(async () => {
await gate.promise;
throw new Error('enable failed');
});
const first = adapter.enableMicrophone(asRoom(room), 'channel-a');
await Promise.resolve();
const second = adapter.enableMicrophone(asRoom(room), 'channel-a');
gate.resolve();
await expect(first).rejects.toThrow('enable failed');
await expect(second).rejects.toThrow('enable failed');
expect(calls).toHaveLength(1);
});