fix(voice): stabilize voice input and noise suppression (#3087)

This commit is contained in:
Hampus authored and GitHub committed 2026-10-01 14:02:12 +02:00
1 parent 7c9564bcad
commit e3fcedbec5
98 files changed
+5610 -3433

No files matched your search

@@ -33,11 +33,13 @@ import {useMediaPermission} from '@app/features/user/components/modals/tabs/hook
import styles from '@app/features/user/components/modals/tabs/UserVoiceTab.module.css';
import * as VoiceSettingsCommands from '@app/features/voice/commands/VoiceSettingsCommands';
import MediaEngine from '@app/features/voice/engine/MediaEngineFacade';
import {supportsVoiceOutputDeviceSelection} from '@app/features/voice/engine/VoiceSharedAudioContext';
import type VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {
type ExternalAudioProcessorMatch,
findExternalProcessorForDevice,
} from '@app/features/voice/utils/ExternalAudioProcessor';
import {prefetchDeepFilterAssets} from '@app/features/voice/utils/noise_suppression/DeepFilter';
import type {VoiceNoiseSuppressionBackend} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionBackends';
import {
getNoiseSuppressionChoiceValues,
@@ -48,6 +50,7 @@ import {
} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionChoices';
import {
getNoiseSuppressionChoiceLabel,
getNoiseSuppressionFallbackMessage,
STEREO_MICROPHONE_DESCRIPTION_DESCRIPTOR,
STEREO_MICROPHONE_DESCRIPTOR,
} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionLabels';
@@ -245,7 +248,10 @@ export const VoiceTab: React.FC<VoiceTabProps> = observer(({voiceSettings, autoR
const defaultPttCombo = getDefaultKeybind('voice_push_to_talk', i18n);
const inputHasLabels = hasDeviceLabels(inputDevices);
const effectiveInputDeviceId = resolveEffectiveDeviceId(inputDeviceId, inputDevices) ?? 'default';
const effectiveOutputDeviceId = resolveEffectiveDeviceId(outputDeviceId, outputDevices) ?? 'default';
const canSelectOutputDevice = supportsVoiceOutputDeviceSelection();
const effectiveOutputDeviceId = canSelectOutputDevice
? (resolveEffectiveDeviceId(outputDeviceId, outputDevices) ?? 'default')
: 'default';
const activeInputDevice = inputDevices.find((d) => d.deviceId === effectiveInputDeviceId) ?? null;
const activeInputLabel = activeInputDevice?.label || null;
const voiceProcessingMode = voiceSettings.getVoiceProcessingModeForDeviceLabel(activeInputLabel);
@@ -256,6 +262,9 @@ export const VoiceTab: React.FC<VoiceTabProps> = observer(({voiceSettings, autoR
useEffect(() => {
if (pttReleaseDelay !== selectedPttReleaseDelay) Keybind.setPushToTalkReleaseDelay(selectedPttReleaseDelay);
}, [pttReleaseDelay, selectedPttReleaseDelay]);
useEffect(() => {
prefetchDeepFilterAssets();
}, [voiceProcessingMode]);
const handleInputDeviceChange = (value: string) => {
VoiceSettingsCommands.update({inputDeviceId: value});
};
@@ -267,8 +276,13 @@ export const VoiceTab: React.FC<VoiceTabProps> = observer(({voiceSettings, autoR
[deviceState, i18n.locale],
);
const outputDeviceOptions = useMemo(
() => buildSettingsDeviceOptions(deviceState, 'audiooutput', i18n),
[deviceState, i18n.locale],
() =>
buildSettingsDeviceOptions(
canSelectOutputDevice ? deviceState : {...deviceState, outputDevices: []},
'audiooutput',
i18n,
),
[canSelectOutputDevice, deviceState, i18n.locale],
);
const resetSliderLabel = i18n._(RESET_SLIDER_TO_DEFAULT_VALUE_DESCRIPTOR);
const profileOptions: Array<RadioOption<VoiceProcessingMode>> = [
@@ -289,6 +303,7 @@ export const VoiceTab: React.FC<VoiceTabProps> = observer(({voiceSettings, autoR
},
];
const noiseSuppressionChoice = getSelectedNoiseSuppressionChoice();
const noiseSuppressionFallbackMessage = getNoiseSuppressionFallbackMessage(i18n);
const noiseSuppressionOptions: Array<ComboboxOption<VoiceNoiseSuppressionBackend>> =
getNoiseSuppressionChoiceValues().map((backend) => ({
value: backend,
@@ -496,6 +511,7 @@ export const VoiceTab: React.FC<VoiceTabProps> = observer(({voiceSettings, autoR
)}
<CompactComboboxRow<VoiceNoiseSuppressionBackend>
label={i18n._(VOICE_NOISE_SUPPRESSION_DESCRIPTOR)}
description={noiseSuppressionFallbackMessage}
value={noiseSuppressionChoice}
options={noiseSuppressionOptions}
onChange={setNoiseSuppressionChoice}
@@ -569,8 +585,14 @@ export const VoiceTab: React.FC<VoiceTabProps> = observer(({voiceSettings, autoR
/>
<CompactComboboxRow
label={i18n._(VOICE_OUTPUT_DEVICE_DESCRIPTOR)}
description={
!canSelectOutputDevice ? (
<Trans>Voice uses your system output device in this browser. Change it in your system settings.</Trans>
) : null
}
value={effectiveOutputDeviceId}
options={outputDeviceOptions}
disabled={!canSelectOutputDevice}
onChange={(value) => VoiceSettingsCommands.update({outputDeviceId: value})}
controlWidth="wide"
menuMinWidth={280}
@@ -660,6 +682,9 @@ export const VoiceTab: React.FC<VoiceTabProps> = observer(({voiceSettings, autoR
aria-label={i18n._(SELECT_VOICE_PROCESSING_DESCRIPTOR)}
data-flx="user.voice-tab.radio-group.voice-processing-mode-change"
/>
{voiceProcessingMode === 'voice' && noiseSuppressionFallbackMessage && (
<p className={styles.pttSettingDescription}>{noiseSuppressionFallbackMessage}</p>
)}
{voiceProcessingMode === 'voice' && (
<div className={styles.profileSubSection} data-flx="user.voice-tab.profile-sub-section">
{renderPttControls()}
@@ -1,90 +1,40 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Logger} from '@app/features/platform/utils/AppLogger';
import {createVoiceSoftClipNode} from '@app/features/voice/engine/VoiceSharedAudioContext';
import {buildDeepFilterAudioChain, type DeepFilterAudioChain} from '@app/features/voice/utils/DeepFilterNoiseProcessor';
import {
buildNoiseSuppressionWorkletChain,
type NoiseSuppressionWorkletChain,
} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionChain';
import type {NoiseSuppressionWorkletBackend} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionWorkletTypes';
const logger = new Logger('MicTestAudioGraph');
import {type VoiceInputConfig, VoiceInputGraph} from '@app/features/voice/utils/VoiceInputProcessor';
export interface MicTestAudioGraph {
source: MediaStreamAudioSourceNode;
analyser: AnalyserNode;
inputGain: GainNode;
delay: DelayNode;
outputGain: GainNode;
softClipInput: GainNode;
softClipOutput: AudioNode;
playbackTarget: AudioNode;
dispose: () => Promise<void>;
configure: () => Promise<void>;
dispose: () => void;
}
interface CreateMicTestAudioGraphOptions {
audioContext: AudioContext;
signal: AbortSignal;
source: MediaStreamAudioSourceNode;
sourceTrack: MediaStreamTrack;
inputGain: number;
resolveConfig: () => VoiceInputConfig;
outputGain: number;
playbackTarget: AudioNode;
playbackDelaySeconds: number;
deepFilter: boolean;
workletBackend: NoiseSuppressionWorkletBackend | null;
}
export async function createMicTestAudioGraph({
audioContext,
export function createMicTestAudioGraph({
signal,
source,
sourceTrack,
inputGain,
resolveConfig,
outputGain,
playbackTarget,
playbackDelaySeconds,
deepFilter,
workletBackend,
}: CreateMicTestAudioGraphOptions): Promise<MicTestAudioGraph> {
const source = audioContext.createMediaStreamSource(new MediaStream([sourceTrack]));
const inputGainNode = audioContext.createGain();
inputGainNode.gain.value = inputGain;
source.connect(inputGainNode);
let deepFilterChain: DeepFilterAudioChain | null = null;
let workletChain: NoiseSuppressionWorkletChain | null = null;
let suppressedSource: MediaStreamAudioSourceNode | null = null;
let monitorHead: AudioNode = inputGainNode;
try {
if (deepFilter) {
deepFilterChain = await buildDeepFilterAudioChain(audioContext);
inputGainNode.connect(deepFilterChain.inputDestination);
suppressedSource = audioContext.createMediaStreamSource(new MediaStream([deepFilterChain.processedTrack]));
monitorHead = suppressedSource;
} else if (workletBackend) {
workletChain = await buildNoiseSuppressionWorkletChain({
audioContext,
backend: workletBackend,
});
inputGainNode.connect(workletChain.inputDestination);
suppressedSource = audioContext.createMediaStreamSource(new MediaStream([workletChain.processedTrack]));
monitorHead = suppressedSource;
}
} catch (error) {
inputGainNode.disconnect();
source.disconnect();
suppressedSource?.disconnect();
if (deepFilterChain) {
await deepFilterChain.dispose().catch((disposeError) => {
logger.debug('Failed to dispose DeepFilter chain after mic test graph init failure', disposeError);
});
}
if (workletChain) {
await workletChain.dispose().catch((disposeError) => {
logger.debug('Failed to dispose noise suppression chain after mic test graph init failure', disposeError);
});
}
throw error;
}
}: CreateMicTestAudioGraphOptions): MicTestAudioGraph {
signal.throwIfAborted();
const audioContext = source.context as AudioContext;
const inputGraph = new VoiceInputGraph(audioContext, 1, resolveConfig);
inputGraph.setSourceNode(sourceTrack, source);
const analyser = audioContext.createAnalyser();
const delay = audioContext.createDelay(Math.max(1, playbackDelaySeconds + 0.25));
const outputGainNode = audioContext.createGain();
@@ -95,42 +45,33 @@ export async function createMicTestAudioGraph({
const softClip = createVoiceSoftClipNode(audioContext);
const softClipInput = softClip?.input ?? outputGainNode;
const softClipOutput: AudioNode = softClip?.output ?? outputGainNode;
monitorHead.connect(analyser);
inputGraph.output.connect(analyser);
analyser.connect(delay);
delay.connect(outputGainNode);
if (softClip) outputGainNode.connect(softClip.input);
softClipOutput.connect(playbackTarget);
void inputGraph.configure();
const dispose = async () => {
source.disconnect();
let disposed = false;
const dispose = () => {
if (disposed) return;
disposed = true;
signal.removeEventListener('abort', dispose);
inputGraph.dispose();
analyser.disconnect();
inputGainNode.disconnect();
delay.disconnect();
outputGainNode.disconnect();
softClipInput.disconnect();
softClipOutput.disconnect();
suppressedSource?.disconnect();
if (deepFilterChain) {
await deepFilterChain.dispose().catch((error) => {
logger.warn('Failed to dispose DeepFilter chain for mic test graph', error);
});
}
if (workletChain) {
await workletChain.dispose().catch((error) => {
logger.warn('Failed to dispose noise suppression chain for mic test graph', error);
});
}
};
signal.addEventListener('abort', dispose, {once: true});
return {
source,
analyser,
inputGain: inputGainNode,
delay,
outputGain: outputGainNode,
softClipInput,
softClipOutput,
playbackTarget,
configure: () => inputGraph.configure(),
dispose,
};
}
@@ -9,29 +9,23 @@ import {
type MicTestAudioGraph,
} from '@app/features/user/components/modals/tabs/hooks/MicTestAudioGraph';
import {
getNoiseSuppressionBackendDescriptor,
type VoiceNoiseSuppressionBackend,
} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionBackends';
import {readNoiseSuppressionBackend} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionRuntime';
import type {NoiseSuppressionWorkletBackend} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionWorkletTypes';
acquireIdleVoiceInputSource,
type VoiceInputContextLease,
} from '@app/features/voice/engine/VoiceInputAudioContext';
import {beginMicrophoneSession} from '@app/features/voice/utils/noise_suppression/DeepFilter';
import NoiseSuppressionAvailability from '@app/features/voice/utils/noise_suppression/NoiseSuppressionAvailability';
import type {VoiceNoiseSuppressionBackend} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionBackends';
import {readRequestedNoiseSuppressionBackend} from '@app/features/voice/utils/noise_suppression/NoiseSuppressionRuntime';
import {
applyContentHintToTrack,
resolveVoiceProcessing,
type VoiceProcessingMode,
} from '@app/features/voice/utils/VoiceProcessingProfile';
import {
boostedVoiceVolumePercentToTrackVolume,
inputVoiceVolumePercentToGain,
} from '@app/features/voice/utils/VoiceVolumeUtils';
import {useCallback, useEffect, useMemo, useRef, useState} from 'react';
import {boostedVoiceVolumePercentToTrackVolume} from '@app/features/voice/utils/VoiceVolumeUtils';
import {useCallback, useEffect, useMemo, useRef, useState, useSyncExternalStore} from 'react';
const logger = new Logger('useMicTest');
interface SinkableAudioContext extends AudioContext {
setSinkId?: (sinkId: string | {type: 'none'}) => Promise<void>;
}
const TEST_AUDIO_CONTEXT_SAMPLE_RATE = 48000;
const MIC_TEST_MONITOR_DELAY_SECONDS = 0.9;
export interface MicTestSettings {
@@ -44,10 +38,8 @@ export interface MicTestSettings {
voiceProcessingMode: VoiceProcessingMode;
}
function resolveMicTestWorkletBackend(backend: VoiceNoiseSuppressionBackend): NoiseSuppressionWorkletBackend | null {
return getNoiseSuppressionBackendDescriptor(backend).engine === 'worklet'
? (backend as NoiseSuppressionWorkletBackend)
: null;
function readEffectiveNoiseSuppressionBackend(): VoiceNoiseSuppressionBackend {
return NoiseSuppressionAvailability.resolveEffectiveBackend(readRequestedNoiseSuppressionBackend());
}
function normalizeOutputDeviceId(deviceId: string): string {
@@ -59,7 +51,8 @@ export const useMicTest = (settings: MicTestSettings) => {
const [isStarting, setIsStarting] = useState(false);
const [level, setLevel] = useState(0);
const [peakLevel, setPeakLevel] = useState(0);
const audioContextRef = useRef<AudioContext | null>(null);
const inputLeaseRef = useRef<VoiceInputContextLease | null>(null);
const inputVolumeRef = useRef(settings.inputVolume);
const graphRef = useRef<MicTestAudioGraph | null>(null);
const playbackDestinationRef = useRef<MediaStreamAudioDestinationNode | null>(null);
const audioElementRef = useRef<HTMLAudioElement | null>(null);
@@ -68,10 +61,14 @@ export const useMicTest = (settings: MicTestSettings) => {
const timeDomainDataRef = useRef<Float32Array<ArrayBuffer> | null>(null);
const peakLevelRef = useRef(0);
const isStartingRef = useRef(false);
const restartPendingRef = useRef(false);
const attemptRef = useRef(0);
const abortRef = useRef<AbortController | null>(null);
const activeCaptureSignatureRef = useRef<string | null>(null);
const micExplicitlyDenied = MediaPermission.microphoneExplicitlyDenied;
const noiseSuppressionBackend = readNoiseSuppressionBackend();
const noiseSuppressionBackend = useSyncExternalStore(
NoiseSuppressionAvailability.subscribe,
readEffectiveNoiseSuppressionBackend,
);
const captureSignature = useMemo(
() =>
JSON.stringify({
@@ -114,7 +111,12 @@ export const useMicTest = (settings: MicTestSettings) => {
animationFrameRef.current = requestAnimationFrame(updateLevel);
}, []);
const stop = useCallback(() => {
if (animationFrameRef.current) {
attemptRef.current++;
abortRef.current?.abort();
abortRef.current = null;
isStartingRef.current = false;
setIsStarting(false);
if (animationFrameRef.current !== null) {
cancelAnimationFrame(animationFrameRef.current);
animationFrameRef.current = null;
}
@@ -125,11 +127,7 @@ export const useMicTest = (settings: MicTestSettings) => {
}
const graph = graphRef.current;
graphRef.current = null;
if (graph) {
void graph.dispose().catch((error) => {
logger.warn('Failed to dispose mic test graph', error);
});
}
graph?.dispose();
const playbackDestination = playbackDestinationRef.current;
playbackDestinationRef.current = null;
if (playbackDestination) {
@@ -140,12 +138,8 @@ export const useMicTest = (settings: MicTestSettings) => {
micStreamRef.current.getTracks().forEach((track) => track.stop());
micStreamRef.current = null;
}
const audioContext = audioContextRef.current;
audioContextRef.current = null;
if (audioContext && audioContext.state !== 'closed') {
void audioContext.close();
audioContextRef.current = null;
}
inputLeaseRef.current?.release();
inputLeaseRef.current = null;
timeDomainDataRef.current = null;
peakLevelRef.current = 0;
setIsTesting(false);
@@ -158,11 +152,17 @@ export const useMicTest = (settings: MicTestSettings) => {
handleMediaPermissionBlocked('microphone');
return;
}
stop();
const attempt = ++attemptRef.current;
const controller = new AbortController();
abortRef.current = controller;
const isCurrent = () => attempt === attemptRef.current && !controller.signal.aborted;
isStartingRef.current = true;
setIsStarting(true);
try {
stop();
beginMicrophoneSession(controller.signal);
const nativeResult = await ensureMacPermission('microphone', {behavior: 'interactive'});
if (!isCurrent()) return;
switch (nativeResult) {
case 'granted':
case 'unsupported-platform':
@@ -177,8 +177,7 @@ export const useMicTest = (settings: MicTestSettings) => {
return exhaustive;
}
}
const profile = resolveVoiceProcessing(settings, readNoiseSuppressionBackend(), false);
const workletBackend = resolveMicTestWorkletBackend(profile.noiseSuppressionBackend);
const profile = resolveVoiceProcessing(settings, readEffectiveNoiseSuppressionBackend(), false);
const baseAudioConstraints: MediaTrackConstraints & {voiceIsolation?: boolean} = {
echoCancellation: profile.echoCancellation,
noiseSuppression: profile.browserNoiseSuppression,
@@ -197,35 +196,48 @@ export const useMicTest = (settings: MicTestSettings) => {
try {
stream = await navigator.mediaDevices.getUserMedia({audio: buildAudioConstraints(true)});
} catch (error) {
if (!isCurrent()) return;
if (!useExactDeviceId || !(error instanceof Error) || error.name !== 'OverconstrainedError') {
throw error;
}
stream = await navigator.mediaDevices.getUserMedia({audio: buildAudioConstraints(false)});
}
if (!isCurrent()) {
stream.getTracks().forEach((track) => track.stop());
return;
}
micStreamRef.current = stream;
const sourceTrack = stream.getAudioTracks()[0];
if (!sourceTrack) {
throw new Error('getUserMedia returned no audio tracks for mic test');
}
applyContentHintToTrack(sourceTrack, profile.contentHint);
const audioContext = new AudioContext({sampleRate: TEST_AUDIO_CONTEXT_SAMPLE_RATE});
audioContextRef.current = audioContext;
if (audioContext.state === 'suspended') {
await audioContext.resume();
const acquired = acquireIdleVoiceInputSource(sourceTrack);
if (!acquired) {
throw new Error('No AudioContext is available for the mic test');
}
inputLeaseRef.current = acquired.lease;
const audioContext = acquired.lease.context;
inputVolumeRef.current = settings.inputVolume;
const outputSinkId = normalizeOutputDeviceId(settings.outputDeviceId);
const playbackDestination = audioContext.createMediaStreamDestination();
playbackDestinationRef.current = playbackDestination;
let playbackTarget: AudioNode = playbackDestination;
graphRef.current = await createMicTestAudioGraph({
audioContext,
graphRef.current = createMicTestAudioGraph({
signal: controller.signal,
source: acquired.source,
sourceTrack,
inputGain: inputVoiceVolumePercentToGain(settings.inputVolume),
resolveConfig: () => ({
backend: resolveVoiceProcessing(settings, readEffectiveNoiseSuppressionBackend(), false)
.noiseSuppressionBackend,
inputVolumePercent: inputVolumeRef.current,
gateEnabled: false,
gateAuto: true,
gateThresholdRms: 0,
}),
outputGain: boostedVoiceVolumePercentToTrackVolume(settings.outputVolume),
playbackTarget,
playbackDelaySeconds: MIC_TEST_MONITOR_DELAY_SECONDS,
deepFilter: profile.deepFilter,
workletBackend,
});
timeDomainDataRef.current = new Float32Array(graphRef.current.analyser.fftSize);
const audioElement = new Audio();
@@ -241,9 +253,11 @@ export const useMicTest = (settings: MicTestSettings) => {
logger.warn('Failed to set mic test media element output device', error);
}
}
if (!isCurrent()) return;
try {
await audioElement.play();
} catch (error) {
if (!isCurrent()) return;
logger.warn('Failed to start mic test media element playback; falling back to AudioContext destination', error);
audioElement.pause();
audioElement.srcObject = null;
@@ -253,17 +267,10 @@ export const useMicTest = (settings: MicTestSettings) => {
playbackDestinationRef.current = null;
graphRef.current.softClipOutput.disconnect();
playbackTarget = audioContext.destination;
const sinkableAudioContext = audioContext as SinkableAudioContext;
if (settings.outputDeviceId !== 'default' && sinkableAudioContext.setSinkId) {
try {
await sinkableAudioContext.setSinkId(outputSinkId);
} catch (sinkError) {
logger.warn('Failed to set mic test AudioContext output device', sinkError);
}
}
graphRef.current.softClipOutput.connect(playbackTarget);
graphRef.current.playbackTarget = playbackTarget;
}
if (!isCurrent()) return;
setIsTesting(true);
activeCaptureSignatureRef.current = captureSignature;
updateLevel();
@@ -271,6 +278,7 @@ export const useMicTest = (settings: MicTestSettings) => {
logger.info('Applied DeepFilterNet3 noise suppression for mic test');
}
} catch (error) {
if (!isCurrent()) return;
logger.error('Error starting mic test', error);
if (error instanceof Error && (error.name === 'NotAllowedError' || error.name === 'PermissionDeniedError')) {
MediaPermission.markMicrophoneExplicitlyDenied();
@@ -278,18 +286,17 @@ export const useMicTest = (settings: MicTestSettings) => {
}
stop();
} finally {
isStartingRef.current = false;
setIsStarting(false);
if (restartPendingRef.current) {
restartPendingRef.current = false;
void start();
if (isCurrent()) {
isStartingRef.current = false;
setIsStarting(false);
}
}
}, [captureSignature, settings, updateLevel, stop, micExplicitlyDenied]);
useEffect(() => {
if (!isTesting) return;
if (graphRef.current) {
graphRef.current.inputGain.gain.value = inputVoiceVolumePercentToGain(settings.inputVolume);
inputVolumeRef.current = settings.inputVolume;
void graphRef.current.configure();
graphRef.current.outputGain.gain.value = boostedVoiceVolumePercentToTrackVolume(settings.outputVolume);
}
}, [isTesting, settings.inputVolume, settings.outputVolume]);
@@ -302,10 +309,6 @@ export const useMicTest = (settings: MicTestSettings) => {
return;
}
activeCaptureSignatureRef.current = captureSignature;
if (isStartingRef.current) {
restartPendingRef.current = true;
return;
}
void start();
}, [captureSignature, isTesting, start]);
useEffect(() => {