diff --git a/fluxer_app/src/features/voice/engine/VoiceLocalSpeakingGate.test.ts b/fluxer_app/src/features/voice/engine/VoiceLocalSpeakingGate.test.ts index 1b15a122b..31a20d609 100644 --- a/fluxer_app/src/features/voice/engine/VoiceLocalSpeakingGate.test.ts +++ b/fluxer_app/src/features/voice/engine/VoiceLocalSpeakingGate.test.ts @@ -1,16 +1,13 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import { - type LocalSpeakingOverrideInput, - resolveLocalSpeakingOverrideState, + isLocalSpeakingSuppressed, + type LocalSpeakingSuppressionInput, } from '@app/features/voice/engine/VoiceLocalSpeakingGate'; import {describe, expect, it} from 'vitest'; -function input(overrides: Partial): LocalSpeakingOverrideInput { +function input(overrides: Partial): LocalSpeakingSuppressionInput { return { - pushToTalkActive: true, - pushToMuteActive: false, - pushToMuteHeld: false, selfDeaf: false, effectiveSelfMute: false, hasMicrophonePublication: true, @@ -19,15 +16,15 @@ function input(overrides: Partial): LocalSpeakingOve }; } -describe('resolveLocalSpeakingOverrideState in push to talk', () => { - it('never lights the speaking indicator for a self-muted user holding the key', () => { - expect(resolveLocalSpeakingOverrideState(input({effectiveSelfMute: true}))).toBe(false); - expect(resolveLocalSpeakingOverrideState(input({effectiveSelfMute: true, microphonePublicationMuted: true}))).toBe( - false, - ); +describe('isLocalSpeakingSuppressed', () => { + it('lets the measured outgoing audio decide for a live, unmuted microphone', () => { + expect(isLocalSpeakingSuppressed(input({}))).toBe(false); }); - it('lights the speaking indicator for an unmuted user holding the key', () => { - expect(resolveLocalSpeakingOverrideState(input({}))).toBe(true); + it('suppresses the indicator whenever nothing is sent', () => { + expect(isLocalSpeakingSuppressed(input({effectiveSelfMute: true}))).toBe(true); + expect(isLocalSpeakingSuppressed(input({selfDeaf: true}))).toBe(true); + expect(isLocalSpeakingSuppressed(input({hasMicrophonePublication: false}))).toBe(true); + expect(isLocalSpeakingSuppressed(input({microphonePublicationMuted: true}))).toBe(true); }); }); diff --git a/fluxer_app/src/features/voice/engine/VoiceLocalSpeakingGate.ts b/fluxer_app/src/features/voice/engine/VoiceLocalSpeakingGate.ts index ff5ecea2b..53a195dce 100644 --- a/fluxer_app/src/features/voice/engine/VoiceLocalSpeakingGate.ts +++ b/fluxer_app/src/features/voice/engine/VoiceLocalSpeakingGate.ts @@ -1,20 +1,14 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -export interface LocalSpeakingOverrideInput { - pushToTalkActive: boolean; - pushToMuteActive: boolean; - pushToMuteHeld: boolean; +export interface LocalSpeakingSuppressionInput { selfDeaf: boolean; effectiveSelfMute: boolean; hasMicrophonePublication: boolean; microphonePublicationMuted: boolean; } -export function resolveLocalSpeakingOverrideState(input: LocalSpeakingOverrideInput): boolean | null { - const effectivelyMuted = - input.selfDeaf || input.effectiveSelfMute || !input.hasMicrophonePublication || input.microphonePublicationMuted; - if (input.pushToTalkActive) return !effectivelyMuted; - if (input.pushToMuteActive && input.pushToMuteHeld) return false; - if (effectivelyMuted) return false; - return null; +export function isLocalSpeakingSuppressed(input: LocalSpeakingSuppressionInput): boolean { + return ( + input.selfDeaf || input.effectiveSelfMute || !input.hasMicrophonePublication || input.microphonePublicationMuted + ); } diff --git a/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.ts b/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.ts index ec07db6f4..69fd80434 100644 --- a/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.ts +++ b/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.ts @@ -1,6 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import {SPEAKING_REMOTE_ATTACK_MS, SPEAKING_REMOTE_RELEASE_MS} from '@app/features/voice/engine/VoiceSpeakingThreshold'; +import {advanceSpeakingHysteresis, type SpeakingHysteresis} from '@app/features/voice/engine/VoiceSpeakingThreshold'; import type {VoiceConnectionQuality} from '@app/features/voice/engine/VoiceTrackSource'; import {assign, initialTransition, type SnapshotFrom, setup, transition} from 'xstate'; @@ -25,12 +25,9 @@ export type LivekitParticipantSnapshot = Readonly<{ lastSpokeAt: number | null; }>; -interface RemoteSpeakingAnalyserState { +interface RemoteSpeakingAnalyserState extends SpeakingHysteresis { identity: string; track: unknown; - speaking: boolean; - belowSinceMs: number | null; - aboveSinceMs: number | null; playbackBoost: number; appliedBoost: number; } @@ -172,29 +169,12 @@ function tickRemoteAnalyser( if (event.trackEnded) return detachRemoteAnalyser(context, event.identity); const commands: Array = []; - let entry = existing; - if (event.rms >= event.threshold) { - const aboveSinceMs = entry.aboveSinceMs ?? event.nowMs; - entry = { - ...entry, - belowSinceMs: null, - aboveSinceMs, - }; - if (!entry.speaking && event.nowMs - aboveSinceMs >= SPEAKING_REMOTE_ATTACK_MS) { - entry = {...entry, speaking: true}; - commands.push({type: 'setAudioLevelSpeaking', identity: event.identity, speaking: true}); - } - } else { - const belowSinceMs = entry.belowSinceMs ?? event.nowMs; - entry = { - ...entry, - aboveSinceMs: null, - belowSinceMs, - }; - if (entry.speaking && event.nowMs - belowSinceMs >= SPEAKING_REMOTE_RELEASE_MS) { - entry = {...entry, speaking: false}; - commands.push({type: 'setAudioLevelSpeaking', identity: event.identity, speaking: false}); - } + let entry: RemoteSpeakingAnalyserState = { + ...existing, + ...advanceSpeakingHysteresis(existing, event.rms, event.threshold, event.nowMs), + }; + if (entry.speaking !== existing.speaking) { + commands.push({type: 'setAudioLevelSpeaking', identity: event.identity, speaking: entry.speaking}); } const boostUpdate = updateRemotePlaybackBoost(entry, event.rms, event.threshold, event.nowMs); entry = boostUpdate.entry; diff --git a/fluxer_app/src/features/voice/engine/VoiceSpeakingDetection.test.ts b/fluxer_app/src/features/voice/engine/VoiceSpeakingDetection.test.ts new file mode 100644 index 000000000..5ff601ca5 --- /dev/null +++ b/fluxer_app/src/features/voice/engine/VoiceSpeakingDetection.test.ts @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import vm from 'node:vm'; +import gateWorkletSource from '@app/features/voice/utils/voiceGateProcessor.worklet.js?raw'; +import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; + +const SAMPLE_RATE = 48000; +const QUANTUM = 128; +const TICK_SAMPLES = SAMPLE_RATE / 20; +const ANALYSER_WINDOW = 512; + +const audio = vi.hoisted(() => ({window: new Float32Array(512), state: 'running'})); + +vi.mock('@app/features/voice/state/VoiceSettings', () => ({default: {getVadThreshold: () => 50}})); + +vi.mock('@app/features/voice/engine/VoiceSharedAudioContext', () => ({ + getSharedVoiceAudioContext: () => ({ + get state() { + return audio.state; + }, + createMediaStreamSource: () => ({connect: () => undefined, disconnect: () => undefined}), + createAnalyser: () => ({ + fftSize: 0, + smoothingTimeConstant: 0, + disconnect: () => undefined, + getFloatTimeDomainData: (target: Float32Array) => target.set(audio.window), + }), + }), +})); + +const {LOCAL_AUTO_MIN_RMS, LOCAL_MAX_RMS, SPEAKING_LOCAL_RELEASE_MS, getLocalSpeakingThresholdRms} = await import( + '@app/features/voice/engine/VoiceSpeakingThreshold' +); +const {computeTimeDomainRms, readSpeakingThresholdRms, VoiceSpeakingMeter} = await import( + '@app/features/voice/engine/VoiceSpeakingDetection' +); +const {createVoiceRemoteSpeakingSnapshot, transitionVoiceRemoteSpeakingSnapshot, clearVoiceRemoteSpeakingCommands} = + await import('@app/features/voice/engine/VoiceParticipantStateMachine'); + +interface GateProcessor { + speaking: boolean; + process(inputs: Array>, outputs: Array>): boolean; +} + +type GateConstructor = new (options: {processorOptions: Record}) => GateProcessor; + +interface GateMode { + apply: boolean; + auto: boolean; + slider?: number; +} + +function loadGate(mode: GateMode): GateProcessor { + let Processor: GateConstructor | null = null; + const sandbox = vm.createContext({ + sampleRate: SAMPLE_RATE, + currentTime: 0, + AudioWorkletProcessor: class { + port = {postMessage: () => undefined, onmessage: null}; + }, + registerProcessor: (_name: string, processor: GateConstructor) => { + Processor = processor; + }, + }); + vm.runInContext(gateWorkletSource, sandbox); + if (!Processor) throw new Error('voice gate worklet did not register'); + return new (Processor as GateConstructor)({ + processorOptions: { + apply: mode.apply, + auto: mode.auto, + manualThresholdRms: getLocalSpeakingThresholdRms(mode.slider ?? 50), + channelCount: 1, + minRms: LOCAL_AUTO_MIN_RMS, + maxRms: LOCAL_MAX_RMS, + releaseMs: SPEAKING_LOCAL_RELEASE_MS, + }, + }); +} + +function createRandom(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let value = state; + value = Math.imul(value ^ (value >>> 15), value | 1); + value ^= value + Math.imul(value ^ (value >>> 7), value | 61); + return ((value ^ (value >>> 14)) >>> 0) / 4294967296; + }; +} + +const UNIFORM_RMS_TO_PEAK = Math.sqrt(3); + +function noise(random: () => number, rms: number): number { + return (random() * 2 - 1) * rms * UNIFORM_RMS_TO_PEAK; +} + +function syllable(index: number, periodSamples: number, onSamples: number): boolean { + return index % periodSamples < onSamples; +} + +function tone(index: number, rms: number): number { + return Math.sin((2 * Math.PI * 180 * index) / SAMPLE_RATE) * rms * Math.SQRT2; +} + +interface Timeline { + gateSpeaking: boolean; + localSpeaking: Array; + remoteSpeaking: Array; +} + +function runPipeline(mode: GateMode, seconds: number, signal: (index: number) => number, transmit = true): Timeline { + const gate = loadGate(mode); + const track = {readyState: 'live'} as MediaStreamTrack; + const meter = new VoiceSpeakingMeter( + () => track, + () => undefined, + ); + let remote = transitionVoiceRemoteSpeakingSnapshot(createVoiceRemoteSpeakingSnapshot(), { + type: 'remote.attach', + identity: 'peer', + track: 'track', + }); + const history = new Float32Array(ANALYSER_WINDOW); + const timeline: Timeline = {gateSpeaking: false, localSpeaking: [], remoteSpeaking: []}; + const seedEnd = 0.3 * SAMPLE_RATE; + const total = Math.floor(seconds * SAMPLE_RATE); + let nextTick = TICK_SAMPLES; + for (let start = 0; start < total; start += QUANTUM) { + const input = new Float32Array(QUANTUM); + for (let offset = 0; offset < QUANTUM; offset++) input[offset] = signal(start + offset); + const output = new Float32Array(QUANTUM); + gate.process([[input]], [[output]]); + if (start >= seedEnd && gate.speaking) timeline.gateSpeaking = true; + history.copyWithin(0, QUANTUM); + history.set(transmit ? output : new Float32Array(QUANTUM), ANALYSER_WINDOW - QUANTUM); + const end = start + QUANTUM; + if (end < nextTick) continue; + nextTick += TICK_SAMPLES; + const nowMs = (end / SAMPLE_RATE) * 1000; + audio.window.set(history); + meter.tick(nowMs); + remote = transitionVoiceRemoteSpeakingSnapshot(remote, { + type: 'remote.tick', + identity: 'peer', + rms: computeTimeDomainRms(history), + threshold: readSpeakingThresholdRms(), + nowMs, + }); + remote = clearVoiceRemoteSpeakingCommands(remote); + timeline.localSpeaking.push(meter.speaking); + timeline.remoteSpeaking.push(remote.context.analysers.get('peer')?.speaking ?? false); + } + meter.dispose(); + return timeline; +} + +const VAD_AUTO: GateMode = {apply: true, auto: true}; +const VAD_MANUAL: GateMode = {apply: true, auto: false, slider: 50}; +const VAD_MANUAL_SENSITIVE: GateMode = {apply: true, auto: false, slider: 10}; +const PUSH_TO_TALK: GateMode = {apply: false, auto: true}; +const MODES: Array<[string, GateMode]> = [ + ['voice activity with auto sensitivity', VAD_AUTO], + ['voice activity with a manual threshold', VAD_MANUAL], + ['voice activity with a sensitive manual threshold', VAD_MANUAL_SENSITIVE], + ['push to talk', PUSH_TO_TALK], +]; + +function suppressionResidue(seed: number): (index: number) => number { + const random = createRandom(seed); + return (index) => { + const click = syllable(index, 0.3 * SAMPLE_RATE, 0.02 * SAMPLE_RATE); + return noise(random, click ? 0.001 : 0.00003); + }; +} + +function hardwareMutedWhilePeersTalk(seed: number): (index: number) => number { + const random = createRandom(seed); + return (index) => { + if (index < SAMPLE_RATE) return 0; + const peerTalking = syllable(index, 0.25 * SAMPLE_RATE, 0.15 * SAMPLE_RATE); + return peerTalking ? tone(index, 0.00004) + noise(random, 0.00001) : 0; + }; +} + +function speechInRoomNoise(seed: number): (index: number) => number { + const random = createRandom(seed); + return (index) => { + const room = noise(random, 0.0003); + if (index < SAMPLE_RATE) return room; + return syllable(index, 0.2 * SAMPLE_RATE, 0.15 * SAMPLE_RATE) ? room + tone(index, 0.05) : room; + }; +} + +describe('local speaking indicator', () => { + beforeEach(() => { + vi.stubGlobal('MediaStream', class {}); + audio.window.fill(0); + audio.state = 'running'; + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it.each(MODES)('stays dark on noise left behind by noise suppression in %s', (_name, mode) => { + const timeline = runPipeline(mode, 3, suppressionResidue(1)); + expect(timeline.localSpeaking).not.toContain(true); + expect(timeline.remoteSpeaking).not.toContain(true); + }); + + it.each(MODES)('stays dark when a hardware-muted mic only carries peer bleed in %s', (_name, mode) => { + const timeline = runPipeline(mode, 4, hardwareMutedWhilePeersTalk(2)); + expect(timeline.localSpeaking).not.toContain(true); + expect(timeline.remoteSpeaking).not.toContain(true); + }); + + it.each(MODES)('lights for real speech in %s', (_name, mode) => { + const timeline = runPipeline(mode, 3, speechInRoomNoise(3)); + expect(timeline.localSpeaking).toContain(true); + expect(timeline.localSpeaking.slice(0, 18)).not.toContain(true); + }); + + it.each(MODES)('agrees tick for tick with how peers judge the same audio in %s', (_name, mode) => { + for (const signal of [suppressionResidue(4), hardwareMutedWhilePeersTalk(5), speechInRoomNoise(6)]) { + const timeline = runPipeline(mode, 3, signal); + expect(timeline.localSpeaking).toEqual(timeline.remoteSpeaking); + } + }); + + it('never lights while the outgoing track is muted, even for loud speech', () => { + for (const [, mode] of MODES) { + const timeline = runPipeline(mode, 3, speechInRoomNoise(7), false); + expect(timeline.localSpeaking).not.toContain(true); + } + }); + + it('keeps the transmit gate opening on those inaudible inputs, so the gate cannot drive the indicator', () => { + expect(runPipeline(VAD_AUTO, 3, suppressionResidue(1)).gateSpeaking).toBe(true); + expect(runPipeline(VAD_AUTO, 4, hardwareMutedWhilePeersTalk(2)).gateSpeaking).toBe(true); + }); + + it('does not hold a stale reading while the analysing context is not running', () => { + const track = {readyState: 'live'} as MediaStreamTrack; + const meter = new VoiceSpeakingMeter( + () => track, + () => undefined, + ); + audio.window.fill(0.1); + meter.tick(0); + meter.tick(50); + expect(meter.speaking).toBe(true); + audio.state = 'suspended'; + for (let nowMs = 100; nowMs <= 400; nowMs += 50) meter.tick(nowMs); + expect(meter.speaking).toBe(false); + meter.dispose(); + }); +}); diff --git a/fluxer_app/src/features/voice/engine/VoiceSpeakingDetection.ts b/fluxer_app/src/features/voice/engine/VoiceSpeakingDetection.ts new file mode 100644 index 000000000..3cc77ea7d --- /dev/null +++ b/fluxer_app/src/features/voice/engine/VoiceSpeakingDetection.ts @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {Logger} from '@app/features/platform/utils/AppLogger'; +import {getSharedVoiceAudioContext} from '@app/features/voice/engine/VoiceSharedAudioContext'; +import { + advanceSpeakingHysteresis, + getRemoteSpeakingThresholdRms, + SILENT_SPEAKING_HYSTERESIS, + type SpeakingHysteresis, +} from '@app/features/voice/engine/VoiceSpeakingThreshold'; +import VoiceSettings from '@app/features/voice/state/VoiceSettings'; + +const logger = new Logger('VoiceSpeakingDetection'); +export const SPEAKING_ANALYSER_INTERVAL_MS = 50; +const SPEAKING_ANALYSER_FFT_SIZE = 512; +const SPEAKING_ANALYSER_SMOOTHING = 0.2; + +export function computeTimeDomainRms(samples: Float32Array): number { + if (samples.length === 0) return 0; + let sumSquares = 0; + for (let i = 0; i < samples.length; i++) { + const sample = samples[i]!; + sumSquares += sample * sample; + } + return Math.sqrt(sumSquares / samples.length); +} + +export function readSpeakingThresholdRms(): number { + return getRemoteSpeakingThresholdRms(VoiceSettings.getVadThreshold()); +} + +export function createSpeakingAnalyser(context: BaseAudioContext): AnalyserNode { + const analyser = context.createAnalyser(); + analyser.fftSize = SPEAKING_ANALYSER_FFT_SIZE; + analyser.smoothingTimeConstant = SPEAKING_ANALYSER_SMOOTHING; + return analyser; +} + +export function createSpeakingSamples(): Float32Array { + return new Float32Array(new ArrayBuffer(SPEAKING_ANALYSER_FFT_SIZE * Float32Array.BYTES_PER_ELEMENT)); +} + +interface SpeakingMeterTap { + context: BaseAudioContext; + source: MediaStreamAudioSourceNode; + analyser: AnalyserNode; +} + +export class VoiceSpeakingMeter { + private track: MediaStreamTrack | null = null; + private tap: SpeakingMeterTap | null = null; + private hysteresis = SILENT_SPEAKING_HYSTERESIS; + private readonly samples = createSpeakingSamples(); + private readonly timer: ReturnType; + + constructor( + private readonly readTrack: () => MediaStreamTrack | null, + private readonly onChange: () => void, + ) { + this.timer = setInterval(() => this.tick(performance.now()), SPEAKING_ANALYSER_INTERVAL_MS); + } + + get speaking(): boolean { + return this.hysteresis.speaking; + } + + tick(nowMs: number): void { + const tap = this.syncTap(); + if (!tap) { + this.update(SILENT_SPEAKING_HYSTERESIS); + return; + } + const running = tap.context.state === 'running'; + if (running) tap.analyser.getFloatTimeDomainData(this.samples); + const rms = running ? computeTimeDomainRms(this.samples) : 0; + this.update(advanceSpeakingHysteresis(this.hysteresis, rms, readSpeakingThresholdRms(), nowMs)); + } + + dispose(): void { + clearInterval(this.timer); + this.closeTap(); + this.hysteresis = SILENT_SPEAKING_HYSTERESIS; + } + + private syncTap(): SpeakingMeterTap | null { + const candidate = this.readTrack(); + const track = candidate?.readyState === 'live' ? candidate : null; + if (track === this.track) return this.tap; + this.closeTap(); + this.track = track; + this.hysteresis = {...SILENT_SPEAKING_HYSTERESIS, speaking: this.hysteresis.speaking}; + const context = track ? getSharedVoiceAudioContext() : null; + if (!track || !context) return null; + try { + const source = context.createMediaStreamSource(new MediaStream([track])); + const analyser = createSpeakingAnalyser(context); + source.connect(analyser); + this.tap = {context, source, analyser}; + } catch (error) { + logger.warn('Failed to attach the local speaking analyser', {error}); + } + return this.tap; + } + + private closeTap(): void { + const tap = this.tap; + this.track = null; + this.tap = null; + if (!tap) return; + tap.source.disconnect(); + tap.analyser.disconnect(); + } + + private update(next: SpeakingHysteresis): void { + const changed = next.speaking !== this.hysteresis.speaking; + this.hysteresis = next; + if (changed) this.onChange(); + } +} diff --git a/fluxer_app/src/features/voice/engine/VoiceSpeakingThreshold.ts b/fluxer_app/src/features/voice/engine/VoiceSpeakingThreshold.ts index 70f9d036e..b7cc89b91 100644 --- a/fluxer_app/src/features/voice/engine/VoiceSpeakingThreshold.ts +++ b/fluxer_app/src/features/voice/engine/VoiceSpeakingThreshold.ts @@ -8,10 +8,18 @@ const REMOTE_MIN_RMS = 0.002; const REMOTE_DEFAULT_RMS = 0.006; const REMOTE_MAX_RMS = 0.035; export const SPEAKING_LOCAL_RELEASE_MS = 180; -export const SPEAKING_REMOTE_RELEASE_MS = 220; -export const SPEAKING_REMOTE_ATTACK_MS = 30; +const SPEAKING_REMOTE_RELEASE_MS = 220; +const SPEAKING_REMOTE_ATTACK_MS = 30; const SLIDER_DEFAULT = 50; +export interface SpeakingHysteresis { + speaking: boolean; + aboveSinceMs: number | null; + belowSinceMs: number | null; +} + +export const SILENT_SPEAKING_HYSTERESIS: SpeakingHysteresis = {speaking: false, aboveSinceMs: null, belowSinceMs: null}; + function clampSlider(slider: number): number { if (!Number.isFinite(slider)) return SLIDER_DEFAULT; if (slider < 0) return 0; @@ -40,3 +48,25 @@ export function getLocalSpeakingThresholdRms(slider: number): number { export function getRemoteSpeakingThresholdRms(slider: number): number { return interpolateThreshold(slider, REMOTE_MIN_RMS, REMOTE_DEFAULT_RMS, REMOTE_MAX_RMS); } + +export function advanceSpeakingHysteresis( + state: SpeakingHysteresis, + rms: number, + thresholdRms: number, + nowMs: number, +): SpeakingHysteresis { + if (rms >= thresholdRms) { + const aboveSinceMs = state.aboveSinceMs ?? nowMs; + return { + speaking: state.speaking || nowMs - aboveSinceMs >= SPEAKING_REMOTE_ATTACK_MS, + aboveSinceMs, + belowSinceMs: null, + }; + } + const belowSinceMs = state.belowSinceMs ?? nowMs; + return { + speaking: state.speaking && nowMs - belowSinceMs < SPEAKING_REMOTE_RELEASE_MS, + aboveSinceMs: null, + belowSinceMs, + }; +} diff --git a/fluxer_app/src/features/voice/engine/mesh/VoiceMeshEngine.ts b/fluxer_app/src/features/voice/engine/mesh/VoiceMeshEngine.ts index 6b15b47c5..9d58be2ec 100644 --- a/fluxer_app/src/features/voice/engine/mesh/VoiceMeshEngine.ts +++ b/fluxer_app/src/features/voice/engine/mesh/VoiceMeshEngine.ts @@ -29,14 +29,7 @@ import { } from '@app/features/voice/engine/mesh/VoiceMeshSignal'; import {VoiceMeshSignalClient} from '@app/features/voice/engine/mesh/VoiceMeshSignalClient'; import type {NormalizedVoiceState} from '@app/features/voice/engine/VoiceGatewayStateMachine'; -import {getSharedVoiceAudioContext} from '@app/features/voice/engine/VoiceSharedAudioContext'; -import { - getRemoteSpeakingThresholdRms, - SPEAKING_REMOTE_RELEASE_MS, -} from '@app/features/voice/engine/VoiceSpeakingThreshold'; -import {computeTimeDomainRms} from '@app/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter'; import voiceEngineV2AppVoiceStateAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppVoiceStateAdapter'; -import VoiceSettings from '@app/features/voice/state/VoiceSettings'; import {buildVoiceParticipantIdentity} from '@app/features/voice/utils/VoiceParticipantIdentity'; import {ME} from '@fluxer/constants/src/AppConstants'; import { @@ -56,7 +49,6 @@ import { type Room, RTCEngine, type SignalOptions, - SpeakerInfo, Track, TrackInfo, TrackInvalidError, @@ -68,7 +60,6 @@ const logger = new Logger('VoiceMeshEngine'); const INBOX_LIMIT = 64; const CONVERSION_GRACE_MS = 10_000; -const SPEAKING_TICK_MS = 50; const SCREEN_SHARE_BUDGET_BPS = 12_000_000; const CAMERA_BUDGET_BPS = 4_000_000; const MESH_VIDEO_CODEC_MIME_TYPES = ['video/H264', 'video/VP8']; @@ -81,16 +72,6 @@ export interface VoiceMeshTarget { userId: string; } -interface SpeakingAnalyser { - track: MediaStreamTrack; - source: MediaStreamAudioSourceNode; - analyser: AnalyserNode; - samples: Float32Array; - timer: ReturnType; - lastLoudAt: number | null; - active: boolean; -} - let activeEngine: VoiceMeshEngine | null = null; function mergeStatsReports(reports: ReadonlyArray): RTCStatsReport { @@ -156,7 +137,6 @@ export class VoiceMeshEngine extends RTCEngine { private tracks: {v: number; tracks: Array} = {v: this.version, tracks: []}; private conversionTimer: ReturnType | null = null; private conversionHeld = false; - private speaking: SpeakingAnalyser | null = null; constructor(options: InternalRoomOptions, target: VoiceMeshTarget) { super(options); @@ -237,7 +217,6 @@ export class VoiceMeshEngine extends RTCEngine { ), ]; this.syncRoster(); - this.syncSpeaking(); } override async createSender( @@ -296,7 +275,6 @@ export class VoiceMeshEngine extends RTCEngine { syncSender(src: VoiceMeshSource): Promise { this.refreshTracks(); - if (src === 'microphone') this.syncSpeaking(); return Promise.all(Array.from(this.peers.values(), (peer) => peer.requestSenderSync(src))).then(() => undefined); } @@ -584,55 +562,6 @@ export class VoiceMeshEngine extends RTCEngine { this.conversionTimer = null; } - private syncSpeaking(): void { - const microphone = this.senders.microphone; - const track = this.room && microphone.sid !== null ? microphone.track : null; - if ((this.speaking?.track ?? null) === track) return; - this.stopSpeaking(); - if (!track) return; - const context = getSharedVoiceAudioContext(); - if (!context) return; - const source = context.createMediaStreamSource(new MediaStream([track])); - const analyser = context.createAnalyser(); - analyser.fftSize = 512; - source.connect(analyser); - const speaking: SpeakingAnalyser = { - track, - source, - analyser, - samples: new Float32Array(analyser.fftSize), - timer: setInterval(() => this.tickSpeaking(speaking), SPEAKING_TICK_MS), - lastLoudAt: null, - active: false, - }; - this.speaking = speaking; - } - - private tickSpeaking(speaking: SpeakingAnalyser): void { - speaking.analyser.getFloatTimeDomainData(speaking.samples); - const level = computeTimeDomainRms(speaking.samples); - const now = performance.now(); - if (level >= getRemoteSpeakingThresholdRms(VoiceSettings.getVadThreshold())) speaking.lastLoudAt = now; - const active = speaking.lastLoudAt !== null && now - speaking.lastLoudAt < SPEAKING_REMOTE_RELEASE_MS; - if (active === speaking.active) return; - speaking.active = active; - this.emitLocalSpeaking(level, active); - } - - private stopSpeaking(): void { - const speaking = this.speaking; - if (!speaking) return; - this.speaking = null; - clearInterval(speaking.timer); - speaking.source.disconnect(); - speaking.analyser.disconnect(); - if (speaking.active) this.emitLocalSpeaking(0, false); - } - - private emitLocalSpeaking(level: number, active: boolean): void { - this.emit(EngineEvent.SpeakersChanged, [new SpeakerInfo({sid: this.localParticipantSid, level, active})]); - } - private stopMesh(): void { if (!this.room) return; this.room = null; @@ -646,7 +575,6 @@ export class VoiceMeshEngine extends RTCEngine { } this.peers.clear(); this.inbox.clear(); - this.stopSpeaking(); flushVoiceMeshConnectionReports(); } diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppMediaExecutionAdapter.tsx b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppMediaExecutionAdapter.tsx index 4ea382cdc..506485c18 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppMediaExecutionAdapter.tsx +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppMediaExecutionAdapter.tsx @@ -19,6 +19,7 @@ import {Store} from '@app/features/voice/engine/Store'; import VoiceDevicePermissionState from '@app/features/voice/engine/VoiceDevicePermissionState'; import type {EffectiveAudioState} from '@app/features/voice/engine/VoiceEffectiveAudioState'; import {getEffectiveAudioState} from '@app/features/voice/engine/VoiceEffectiveAudioState'; +import {noteLocalVoiceActivity} from '@app/features/voice/engine/VoiceIdleActivityBridge'; import {acquireVoiceInputContext, type VoiceInputContextLease} from '@app/features/voice/engine/VoiceInputAudioContext'; import { createVoiceMicrophoneFailureLatchSnapshot, @@ -28,7 +29,7 @@ import { transitionVoiceMicrophoneFailureLatchSnapshot, type VoiceMicrophoneFailureLatchEvent, } from '@app/features/voice/engine/VoiceLocalAudioReconcilePolicy'; -import {resolveLocalSpeakingOverrideState} from '@app/features/voice/engine/VoiceLocalSpeakingGate'; +import {isLocalSpeakingSuppressed} from '@app/features/voice/engine/VoiceLocalSpeakingGate'; import { getRoomFromMediaEngine, getVoiceConnectionContextFromMediaEngine, @@ -43,6 +44,7 @@ import { type VoiceMediaEvent, type VoiceMediaSnapshot, } from '@app/features/voice/engine/VoiceMediaStateMachine'; +import {VoiceSpeakingMeter} from '@app/features/voice/engine/VoiceSpeakingDetection'; import type {VoiceStateSyncPartial} from '@app/features/voice/engine/VoiceStateSyncTypes'; import { enforceLocalMediaPublicationCap, @@ -231,6 +233,7 @@ function getEffectiveSelfMuteForVoiceStatePayloadFromV2AudioControls(): boolean class VoiceEngineV2AppMediaExecutionAdapter extends Store { private speakingDetectorCleanup: (() => void) | null = null; + private localSpeakingMeter: VoiceSpeakingMeter | null = null; private mediaStateSnapshot: VoiceMediaSnapshot = createVoiceMediaSnapshot(); private microphoneEnablePromise: Promise | null = null; private microphoneRefreshQueue: Promise = Promise.resolve(); @@ -373,7 +376,7 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store { private async enforceSpeakPermissionMute(room: Room): Promise { await this.disableMicrophone(room); this.syncVoiceState({self_mute: true}); - this.syncLocalSpeakingOverride(room); + this.syncLocalSpeaking(room); } async prepareMicrophonePermissionForConnect(): Promise { @@ -485,7 +488,7 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store { } else { this.syncVoiceState({self_mute: audioState.selfMute}); } - this.syncLocalSpeakingOverride(room); + this.syncLocalSpeaking(room); } catch (e: unknown) { if (isPermissionDeniedError(e)) { this.handleMicrophonePermissionDenied(); @@ -542,7 +545,7 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store { } if (audioState.effectiveDeaf || audioState.serverMute || audioState.selfMute) { this.setAudioPublicationsMuted(room, true, 'voice state update'); - this.syncLocalSpeakingOverride(room); + this.syncLocalSpeaking(room); return; } if (!this.hasLiveMicrophonePublication(room)) { @@ -562,7 +565,7 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store { } } this.setAudioPublicationsMuted(room, this.shouldMuteMicrophonePublication(), 'voice state update'); - this.syncLocalSpeakingOverride(room); + this.syncLocalSpeaking(room); } private hasMicrophonePublication(room: Room): boolean { @@ -1506,7 +1509,7 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store { } this.syncVoiceState({self_mute: targetMute}); this.updateMediaAudioControls(); - this.syncLocalSpeakingOverride(room); + this.syncLocalSpeaking(room); } applyPushToMuteHold(held: boolean, room: Room | null, getCurrentUserVoiceState: () => VoiceState | null): void { @@ -1534,7 +1537,7 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store { } this.syncVoiceState({self_mute: targetMute}); this.updateMediaAudioControls(); - this.syncLocalSpeakingOverride(room); + this.syncLocalSpeaking(room); } handlePushToTalkModeChange(room: Room | null, getCurrentUserVoiceState: () => VoiceState | null): void { @@ -1560,7 +1563,7 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store { } this.syncVoiceState({self_mute: targetMute}); this.updateMediaAudioControls(); - this.syncLocalSpeakingOverride(room); + this.syncLocalSpeaking(room); } getMuteReason(voiceState: VoiceState | null, guildId?: string | null, channelId?: string | null): VoiceMuteReason { @@ -1671,13 +1674,9 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store { return true; } - private getLocalSpeakingOverrideState(room: Room): boolean | null { - if (!room.localParticipant) return false; + private shouldSuppressLocalSpeaking(room: Room): boolean { const publication = this.getMicrophonePublication(room); - return resolveLocalSpeakingOverrideState({ - pushToTalkActive: isPushToTalkActiveFromAppState(), - pushToMuteActive: isPushToMuteActiveFromAppState(), - pushToMuteHeld: Keybind.pushToMuteHeld, + return isLocalSpeakingSuppressed({ selfDeaf: LocalVoiceState.getSelfDeaf(), effectiveSelfMute: getEffectiveSelfMuteForVoiceStatePayloadFromV2AudioControls(), hasMicrophonePublication: Boolean(publication), @@ -1703,10 +1702,9 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store { return selectVoiceEngineV2AppParticipant(snapshot, identity)?.isAudioLevelSpeaking ?? null; } - private syncLocalSpeakingOverride(room: Room | null): void { + private syncLocalSpeaking(room: Room | null): void { if (!room?.localParticipant) return; - const speaking = this.getLocalSpeakingOverrideState(room); - if (speaking === null) return; + const speaking = !this.shouldSuppressLocalSpeaking(room) && (this.localSpeakingMeter?.speaking ?? false); this.setLocalParticipantAudioLevelSpeaking(room, speaking); } @@ -1714,28 +1712,27 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store { this.stopLocalSpeakingDetector(room); const track = this.getLocalAudioTrack(room); if (!track || !room.localParticipant) return; - let stopLevel: (() => void) | null = null; - const bind = (): void => { - stopLevel?.(); - stopLevel = null; - this.setLocalParticipantAudioLevelSpeaking(room, this.getLocalSpeakingOverrideState(room) ?? false); + const meter = new VoiceSpeakingMeter( + () => track.mediaStreamTrack, + () => { + this.syncLocalSpeaking(room); + if (meter.speaking) noteLocalVoiceActivity(room.localParticipant, {force: true, speaking: true}); + }, + ); + const reportRuntime = (): void => { const processor = readVoiceInputProcessor(track); - if (!processor) { - this.reportRawMicrophoneRuntime(track); - return; - } - processor.reportRuntime(); - stopLevel = processor.onLevel((level) => { - this.setLocalParticipantAudioLevelSpeaking(room, this.getLocalSpeakingOverrideState(room) ?? level.speaking); - }); + if (processor) processor.reportRuntime(); + else this.reportRawMicrophoneRuntime(track); }; - track.on(TrackEvent.TrackProcessorUpdate, bind); + track.on(TrackEvent.TrackProcessorUpdate, reportRuntime); + this.localSpeakingMeter = meter; this.speakingDetectorCleanup = () => { - track.off(TrackEvent.TrackProcessorUpdate, bind); - stopLevel?.(); + track.off(TrackEvent.TrackProcessorUpdate, reportRuntime); + meter.dispose(); NoiseSuppressionAvailability.clearVoiceInputRuntime(track); }; - bind(); + reportRuntime(); + meter.tick(performance.now()); this.transitionMediaState({type: 'speakingDetector.attach'}); } @@ -1743,6 +1740,7 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store { this.transitionMediaState({type: 'speakingDetector.detach'}); this.speakingDetectorCleanup?.(); this.speakingDetectorCleanup = null; + this.localSpeakingMeter = null; if (room) this.setLocalParticipantAudioLevelSpeaking(room, false); } } diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppPushToTalkSelfMute.test.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppPushToTalkSelfMute.test.ts index 05c82a108..a0790814e 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppPushToTalkSelfMute.test.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppPushToTalkSelfMute.test.ts @@ -3,10 +3,11 @@ import type {Room} from 'livekit-client'; import {Track} from 'livekit-client'; -import {beforeEach, describe, expect, it, vi} from 'vitest'; +import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; const syncLocalVoiceStateWithServer = vi.fn(); const setVoiceEngineV2ParticipantAudioLevelSpeaking = vi.fn(() => true); +const outgoingAudio = vi.hoisted(() => ({rms: 0})); const keybind = { pushToTalkEnabled: true, @@ -63,6 +64,18 @@ vi.mock('@app/features/voice/state/LocalVoiceState', () => ({default: localVoice vi.mock('@app/features/voice/engine/MediaEngineFacade', () => ({default: {}, useMediaEngineVersion: () => 0})); +vi.mock('@app/features/voice/engine/VoiceSharedAudioContext', async (importOriginal) => ({ + ...(await importOriginal()), + getSharedVoiceAudioContext: () => ({ + state: 'running', + createMediaStreamSource: () => ({connect: () => undefined, disconnect: () => undefined}), + createAnalyser: () => ({ + disconnect: () => undefined, + getFloatTimeDomainData: (target: Float32Array) => target.fill(outgoingAudio.rms), + }), + }), +})); + vi.mock('@app/features/voice/engine/VoiceMediaEngineBridge', () => ({ getRoomFromMediaEngine: () => null, getVoiceConnectionContextFromMediaEngine: () => null, @@ -82,7 +95,12 @@ const {default: adapter} = await import('@app/features/voice/engine/v2/VoiceEngi interface FakeMicrophonePublication { source: Track.Source; isMuted: boolean; - track: {mediaStreamTrack: {readyState: string}}; + track: { + mediaStreamTrack: {readyState: string; getSettings(): MediaTrackSettings; getConstraints(): MediaTrackConstraints}; + getProcessor(): undefined; + on(): void; + off(): void; + }; mute: ReturnType; unmute: ReturnType; } @@ -91,7 +109,12 @@ function createMicrophonePublication(): FakeMicrophonePublication { const publication: FakeMicrophonePublication = { source: Track.Source.Microphone, isMuted: true, - track: {mediaStreamTrack: {readyState: 'live'}}, + track: { + mediaStreamTrack: {readyState: 'live', getSettings: () => ({}), getConstraints: () => ({})}, + getProcessor: () => undefined, + on: () => undefined, + off: () => undefined, + }, mute: vi.fn(async () => { publication.isMuted = true; }), @@ -120,7 +143,7 @@ function lastSyncedSelfMute(): unknown { return (call?.[0] as {self_mute?: boolean} | undefined)?.self_mute; } -function lastSpeakingOverride(): unknown { +function lastSpeaking(): unknown { return setVoiceEngineV2ParticipantAudioLevelSpeaking.mock.calls.at(-1)?.[1 as never]; } @@ -147,7 +170,7 @@ describe('push to talk with self-mute', () => { expect(publication.mute).toHaveBeenCalled(); expect(publication.isMuted).toBe(true); expect(lastSyncedSelfMute()).toBe(true); - expect(lastSpeakingOverride()).toBe(false); + expect(lastSpeaking()).toBe(false); adapter.applyPushToTalkHold(false, room, noServerVoiceState); expect(publication.unmute).not.toHaveBeenCalled(); expect(lastSyncedSelfMute()).toBe(true); @@ -160,11 +183,11 @@ describe('push to talk with self-mute', () => { adapter.applyPushToTalkHold(true, room, noServerVoiceState); expect(publication.unmute).toHaveBeenCalledTimes(1); expect(lastSyncedSelfMute()).toBe(false); - expect(lastSpeakingOverride()).toBe(true); + expect(lastSpeaking()).toBe(false); adapter.applyPushToTalkHold(false, room, noServerVoiceState); expect(publication.isMuted).toBe(true); expect(lastSyncedSelfMute()).toBe(true); - expect(lastSpeakingOverride()).toBe(false); + expect(lastSpeaking()).toBe(false); }); it('transmits when the user unmutes while the key is still held', () => { @@ -228,3 +251,71 @@ describe('push to talk with self-mute', () => { enableMicrophone.mockRestore(); }); }); + +interface LocalSpeakingMeterProbe { + startLocalSpeakingDetector(room: Room): void; + stopLocalSpeakingDetector(room?: Room | null): void; + localSpeakingMeter: {tick(nowMs: number): void} | null; +} + +describe('local speaking indicator in push to talk', () => { + const probe = adapter as unknown as LocalSpeakingMeterProbe; + let nowMs = 0; + + function tick(rms: number, ticks = 1): void { + outgoingAudio.rms = rms; + for (let index = 0; index < ticks; index++) { + nowMs += 50; + probe.localSpeakingMeter?.tick(nowMs); + } + } + + beforeEach(() => { + vi.stubGlobal('MediaStream', class {}); + keybind.pushToTalkEnabled = true; + keybind.pushToTalkHeld = false; + localVoiceState.selfMute = false; + localVoiceState.selfDeaf = false; + outgoingAudio.rms = 0; + setVoiceEngineV2ParticipantAudioLevelSpeaking.mockClear(); + }); + + afterEach(() => { + probe.stopLocalSpeakingDetector(); + vi.unstubAllGlobals(); + }); + + function startWithMicrophone(): {room: Room; publication: FakeMicrophonePublication} { + const {room, publications} = createRoom(); + const publication = createMicrophonePublication(); + publications.set('mic', publication); + probe.startLocalSpeakingDetector(room); + return {room, publication}; + } + + it('stays dark while the key is held and nothing audible is sent', () => { + const {room} = startWithMicrophone(); + adapter.applyPushToTalkHold(true, room, noServerVoiceState); + tick(0.001, 10); + expect(setVoiceEngineV2ParticipantAudioLevelSpeaking).not.toHaveBeenCalledWith(expect.anything(), true); + }); + + it('lights for speech sent while the key is held and goes dark the moment it is released', () => { + const {room} = startWithMicrophone(); + adapter.applyPushToTalkHold(true, room, noServerVoiceState); + tick(0.05, 2); + expect(lastSpeaking()).toBe(true); + adapter.applyPushToTalkHold(false, room, noServerVoiceState); + expect(lastSpeaking()).toBe(false); + tick(0.05, 4); + expect(setVoiceEngineV2ParticipantAudioLevelSpeaking).toHaveBeenLastCalledWith(expect.anything(), false); + }); + + it('never lights for a self-muted user, however loud the microphone is', () => { + const {room} = startWithMicrophone(); + localVoiceState.selfMute = true; + adapter.applyPushToTalkHold(true, room, noServerVoiceState); + tick(0.2, 10); + expect(setVoiceEngineV2ParticipantAudioLevelSpeaking).not.toHaveBeenCalledWith(expect.anything(), true); + }); +}); diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.ts index 5a1ebda92..028d1f5e1 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.ts @@ -18,7 +18,13 @@ import { getSharedVoiceAudioContext, resolveAudioContextConstructor, } from '@app/features/voice/engine/VoiceSharedAudioContext'; -import {getRemoteSpeakingThresholdRms} from '@app/features/voice/engine/VoiceSpeakingThreshold'; +import { + computeTimeDomainRms, + createSpeakingAnalyser, + createSpeakingSamples, + readSpeakingThresholdRms, + SPEAKING_ANALYSER_INTERVAL_MS, +} from '@app/features/voice/engine/VoiceSpeakingDetection'; import {VoiceTrackKind, VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource'; import { assertNonEmptyString, @@ -30,23 +36,11 @@ import { clearRemoteVoicePlaybackBoost, setRemoteVoicePlaybackBoost, } from '@app/features/voice/state/RemoteVoicePlaybackBoost'; -import VoiceSettings from '@app/features/voice/state/VoiceSettings'; import type {Participant, RemoteAudioTrack, RemoteTrack, RemoteTrackPublication, Room} from 'livekit-client'; const logger = new Logger('VoiceEngineV2AppRemoteSpeakingAdapter'); -const REMOTE_SPEAKING_ANALYSER_INTERVAL_MS = 50; const REMOTE_SPEAKING_ANALYSER_HANDLES_CAP = 256; -export function computeTimeDomainRms(samples: Float32Array): number { - if (samples.length === 0) return 0; - let sumSquares = 0; - for (let i = 0; i < samples.length; i++) { - const sample = samples[i]!; - sumSquares += sample * sample; - } - return Math.sqrt(sumSquares / samples.length); -} - interface AnalyserHandle { identity: string; track: MediaStreamTrack; @@ -171,17 +165,15 @@ class VoiceEngineV2AppRemoteSpeakingAdapter { } try { const source = ctx.createMediaStreamSource(new MediaStream([mediaStreamTrack])); - const analyser = ctx.createAnalyser(); - analyser.fftSize = 512; - analyser.smoothingTimeConstant = 0.2; + const analyser = createSpeakingAnalyser(ctx); source.connect(analyser); const handle: AnalyserHandle = { identity, track: mediaStreamTrack, source, analyser, - samples: new Float32Array(new ArrayBuffer(analyser.fftSize * Float32Array.BYTES_PER_ELEMENT)), - nextTickAtMs: this.nowMs() + REMOTE_SPEAKING_ANALYSER_INTERVAL_MS, + samples: createSpeakingSamples(), + nextTickAtMs: this.nowMs() + SPEAKING_ANALYSER_INTERVAL_MS, }; this.analysers.set(identity, handle); this.applyCommands(this.transition({type: 'remote.attach', identity, track: mediaStreamTrack})); @@ -301,9 +293,9 @@ class VoiceEngineV2AppRemoteSpeakingAdapter { } handle.analyser.getFloatTimeDomainData(handle.samples); const rms = computeTimeDomainRms(handle.samples); - const threshold = getRemoteSpeakingThresholdRms(VoiceSettings.getVadThreshold()); + const threshold = readSpeakingThresholdRms(); this.applyCommands(this.transition({type: 'remote.tick', identity: handle.identity, rms, threshold, nowMs})); - handle.nextTickAtMs = nowMs + REMOTE_SPEAKING_ANALYSER_INTERVAL_MS; + handle.nextTickAtMs = nowMs + SPEAKING_ANALYSER_INTERVAL_MS; } private disposeHandle(handle: AnalyserHandle): void { @@ -332,7 +324,7 @@ class VoiceEngineV2AppRemoteSpeakingAdapter { if (typeof window === 'undefined') return; this.autoScheduleTimerId = window.setInterval(() => { this.tickHost(this.nowMs()); - }, REMOTE_SPEAKING_ANALYSER_INTERVAL_MS); + }, SPEAKING_ANALYSER_INTERVAL_MS); } private stopAutoSchedule(): void { diff --git a/fluxer_app/src/features/voice/utils/VoiceActivityGate.ts b/fluxer_app/src/features/voice/utils/VoiceActivityGate.ts index befe80f25..acb5eb4fa 100644 --- a/fluxer_app/src/features/voice/utils/VoiceActivityGate.ts +++ b/fluxer_app/src/features/voice/utils/VoiceActivityGate.ts @@ -17,7 +17,6 @@ export interface VoiceInputLevel { rms: number; floorRms: number; thresholdRms: number; - speaking: boolean; nonFinite: boolean; frame: number; contextTime: number; diff --git a/fluxer_app/src/features/voice/utils/VoiceInputProcessor.ts b/fluxer_app/src/features/voice/utils/VoiceInputProcessor.ts index ac614c337..cfe2f1d4c 100644 --- a/fluxer_app/src/features/voice/utils/VoiceInputProcessor.ts +++ b/fluxer_app/src/features/voice/utils/VoiceInputProcessor.ts @@ -461,8 +461,6 @@ export class VoiceInputTrackProcessor implements TrackProcessor void>(); - private latestLevel: VoiceInputLevel | null = null; private hasRunningLevel = false; private lease: VoiceInputContextLease | null = null; private graph: VoiceInputGraph | null = null; @@ -485,7 +483,7 @@ export class VoiceInputTrackProcessor implements TrackProcessor this.emitLevel(level, true), + () => this.noteLevel(), () => this.events.onGraphFailed(this), () => this.reportRuntime(), ); @@ -528,22 +526,10 @@ export class VoiceInputTrackProcessor implements TrackProcessor void): () => void { - this.levelListeners.add(listener); - if (this.latestLevel) listener(this.latestLevel); - return () => { - this.levelListeners.delete(listener); - }; - } - - private emitLevel(level: VoiceInputLevel, fromWorklet = false): void { - if (this.state !== 'live') return; - if (fromWorklet && !this.hasRunningLevel && this.lease?.context.state === 'running') { - this.hasRunningLevel = true; - NoiseSuppressionAvailability.setVoiceInputGraphUnavailable(false); - } - this.latestLevel = level; - for (const listener of this.levelListeners) listener(level); + private noteLevel(): void { + if (this.state !== 'live' || this.hasRunningLevel || this.lease?.context.state !== 'running') return; + this.hasRunningLevel = true; + NoiseSuppressionAvailability.setVoiceInputGraphUnavailable(false); } private watchContextState(): void { @@ -553,7 +539,6 @@ export class VoiceInputTrackProcessor implements TrackProcessor { this.stallTimer = null; if (this.state === 'live' && context.state !== 'running') this.events.onGraphStalled(this); @@ -568,8 +553,6 @@ export class VoiceInputTrackProcessor implements TrackProcessor { if (this.state === 'destroyed') return; - if (this.latestLevel) this.emitLevel({...this.latestLevel, rms: 0, speaking: false}); - this.levelListeners.clear(); this.state = 'destroyed'; NoiseSuppressionAvailability.clearVoiceInputRuntime(this); this.clearStallTimer(); diff --git a/fluxer_app/src/features/voice/utils/voiceGateProcessor.worklet.js b/fluxer_app/src/features/voice/utils/voiceGateProcessor.worklet.js index 8affe2133..ec284d609 100644 --- a/fluxer_app/src/features/voice/utils/voiceGateProcessor.worklet.js +++ b/fluxer_app/src/features/voice/utils/voiceGateProcessor.worklet.js @@ -104,7 +104,6 @@ class VoiceGateProcessor extends AudioWorkletProcessor { const thresholdRms = this.auto ? Math.min(this.maxRms, Math.max(this.minRms, floorRms * 3)) : this.manualThresholdRms; - const wasSpeaking = this.speaking; if (this.auto && this.frame < this.seedUntilFrame) { this.speaking = false; this.silenceFrames = 0; @@ -129,13 +128,12 @@ class VoiceGateProcessor extends AudioWorkletProcessor { if (this.delayFrames > 0) this.delayIndex = (this.delayIndex + 1) % this.delayFrames; } this.frame += length; - if (wasSpeaking !== this.speaking || this.frame >= this.reportFrame) { + if (this.frame >= this.reportFrame) { this.port.postMessage({ type: 'level', rms, floorRms, thresholdRms, - speaking: this.speaking, nonFinite: this.nonFinite, frame: this.frame, contextTime: currentTime,