fix(voice): judge local speaking from the sent audio (#3503)

fluxer_app/src/features/voice/engine/VoiceLocalSpeakingGate.test.ts

@@ -1,16 +1,13 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import {
-	type LocalSpeakingOverrideInput,
-	resolveLocalSpeakingOverrideState,
+	isLocalSpeakingSuppressed,
+	type LocalSpeakingSuppressionInput,
 } from '@app/features/voice/engine/VoiceLocalSpeakingGate';
 import {describe, expect, it} from 'vitest';
 
-function input(overrides: Partial<LocalSpeakingOverrideInput>): LocalSpeakingOverrideInput {
+function input(overrides: Partial<LocalSpeakingSuppressionInput>): LocalSpeakingSuppressionInput {
 	return {
-		pushToTalkActive: true,
-		pushToMuteActive: false,
-		pushToMuteHeld: false,
 		selfDeaf: false,
 		effectiveSelfMute: false,
 		hasMicrophonePublication: true,
@@ -19,15 +16,15 @@ function input(overrides: Partial<LocalSpeakingOverrideInput>): LocalSpeakingOve
 	};
 }
 
-describe('resolveLocalSpeakingOverrideState in push to talk', () => {
-	it('never lights the speaking indicator for a self-muted user holding the key', () => {
-		expect(resolveLocalSpeakingOverrideState(input({effectiveSelfMute: true}))).toBe(false);
-		expect(resolveLocalSpeakingOverrideState(input({effectiveSelfMute: true, microphonePublicationMuted: true}))).toBe(
-			false,
-		);
+describe('isLocalSpeakingSuppressed', () => {
+	it('lets the measured outgoing audio decide for a live, unmuted microphone', () => {
+		expect(isLocalSpeakingSuppressed(input({}))).toBe(false);
 	});
 
-	it('lights the speaking indicator for an unmuted user holding the key', () => {
-		expect(resolveLocalSpeakingOverrideState(input({}))).toBe(true);
+	it('suppresses the indicator whenever nothing is sent', () => {
+		expect(isLocalSpeakingSuppressed(input({effectiveSelfMute: true}))).toBe(true);
+		expect(isLocalSpeakingSuppressed(input({selfDeaf: true}))).toBe(true);
+		expect(isLocalSpeakingSuppressed(input({hasMicrophonePublication: false}))).toBe(true);
+		expect(isLocalSpeakingSuppressed(input({microphonePublicationMuted: true}))).toBe(true);
 	});
 });

fluxer_app/src/features/voice/engine/VoiceLocalSpeakingGate.ts

@@ -1,20 +1,14 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-export interface LocalSpeakingOverrideInput {
-	pushToTalkActive: boolean;
-	pushToMuteActive: boolean;
-	pushToMuteHeld: boolean;
+export interface LocalSpeakingSuppressionInput {
 	selfDeaf: boolean;
 	effectiveSelfMute: boolean;
 	hasMicrophonePublication: boolean;
 	microphonePublicationMuted: boolean;
 }
 
-export function resolveLocalSpeakingOverrideState(input: LocalSpeakingOverrideInput): boolean | null {
-	const effectivelyMuted =
-		input.selfDeaf || input.effectiveSelfMute || !input.hasMicrophonePublication || input.microphonePublicationMuted;
-	if (input.pushToTalkActive) return !effectivelyMuted;
-	if (input.pushToMuteActive && input.pushToMuteHeld) return false;
-	if (effectivelyMuted) return false;
-	return null;
+export function isLocalSpeakingSuppressed(input: LocalSpeakingSuppressionInput): boolean {
+	return (
+		input.selfDeaf || input.effectiveSelfMute || !input.hasMicrophonePublication || input.microphonePublicationMuted
+	);
 }

fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.ts

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-import {SPEAKING_REMOTE_ATTACK_MS, SPEAKING_REMOTE_RELEASE_MS} from '@app/features/voice/engine/VoiceSpeakingThreshold';
+import {advanceSpeakingHysteresis, type SpeakingHysteresis} from '@app/features/voice/engine/VoiceSpeakingThreshold';
 import type {VoiceConnectionQuality} from '@app/features/voice/engine/VoiceTrackSource';
 import {assign, initialTransition, type SnapshotFrom, setup, transition} from 'xstate';
 
@@ -25,12 +25,9 @@ export type LivekitParticipantSnapshot = Readonly<{
 	lastSpokeAt: number | null;
 }>;
 
-interface RemoteSpeakingAnalyserState {
+interface RemoteSpeakingAnalyserState extends SpeakingHysteresis {
 	identity: string;
 	track: unknown;
-	speaking: boolean;
-	belowSinceMs: number | null;
-	aboveSinceMs: number | null;
 	playbackBoost: number;
 	appliedBoost: number;
 }
@@ -172,29 +169,12 @@ function tickRemoteAnalyser(
 	if (event.trackEnded) return detachRemoteAnalyser(context, event.identity);
 
 	const commands: Array<VoiceRemoteSpeakingCommand> = [];
-	let entry = existing;
-	if (event.rms >= event.threshold) {
-		const aboveSinceMs = entry.aboveSinceMs ?? event.nowMs;
-		entry = {
-			...entry,
-			belowSinceMs: null,
-			aboveSinceMs,
-		};
-		if (!entry.speaking && event.nowMs - aboveSinceMs >= SPEAKING_REMOTE_ATTACK_MS) {
-			entry = {...entry, speaking: true};
-			commands.push({type: 'setAudioLevelSpeaking', identity: event.identity, speaking: true});
-		}
-	} else {
-		const belowSinceMs = entry.belowSinceMs ?? event.nowMs;
-		entry = {
-			...entry,
-			aboveSinceMs: null,
-			belowSinceMs,
-		};
-		if (entry.speaking && event.nowMs - belowSinceMs >= SPEAKING_REMOTE_RELEASE_MS) {
-			entry = {...entry, speaking: false};
-			commands.push({type: 'setAudioLevelSpeaking', identity: event.identity, speaking: false});
-		}
+	let entry: RemoteSpeakingAnalyserState = {
+		...existing,
+		...advanceSpeakingHysteresis(existing, event.rms, event.threshold, event.nowMs),
+	};
+	if (entry.speaking !== existing.speaking) {
+		commands.push({type: 'setAudioLevelSpeaking', identity: event.identity, speaking: entry.speaking});
 	}
 	const boostUpdate = updateRemotePlaybackBoost(entry, event.rms, event.threshold, event.nowMs);
 	entry = boostUpdate.entry;

fluxer_app/src/features/voice/engine/VoiceSpeakingDetection.test.ts

@@ -0,0 +1,257 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import vm from 'node:vm';
+import gateWorkletSource from '@app/features/voice/utils/voiceGateProcessor.worklet.js?raw';
+import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
+
+const SAMPLE_RATE = 48000;
+const QUANTUM = 128;
+const TICK_SAMPLES = SAMPLE_RATE / 20;
+const ANALYSER_WINDOW = 512;
+
+const audio = vi.hoisted(() => ({window: new Float32Array(512), state: 'running'}));
+
+vi.mock('@app/features/voice/state/VoiceSettings', () => ({default: {getVadThreshold: () => 50}}));
+
+vi.mock('@app/features/voice/engine/VoiceSharedAudioContext', () => ({
+	getSharedVoiceAudioContext: () => ({
+		get state() {
+			return audio.state;
+		},
+		createMediaStreamSource: () => ({connect: () => undefined, disconnect: () => undefined}),
+		createAnalyser: () => ({
+			fftSize: 0,
+			smoothingTimeConstant: 0,
+			disconnect: () => undefined,
+			getFloatTimeDomainData: (target: Float32Array) => target.set(audio.window),
+		}),
+	}),
+}));
+
+const {LOCAL_AUTO_MIN_RMS, LOCAL_MAX_RMS, SPEAKING_LOCAL_RELEASE_MS, getLocalSpeakingThresholdRms} = await import(
+	'@app/features/voice/engine/VoiceSpeakingThreshold'
+);
+const {computeTimeDomainRms, readSpeakingThresholdRms, VoiceSpeakingMeter} = await import(
+	'@app/features/voice/engine/VoiceSpeakingDetection'
+);
+const {createVoiceRemoteSpeakingSnapshot, transitionVoiceRemoteSpeakingSnapshot, clearVoiceRemoteSpeakingCommands} =
+	await import('@app/features/voice/engine/VoiceParticipantStateMachine');
+
+interface GateProcessor {
+	speaking: boolean;
+	process(inputs: Array<Array<Float32Array>>, outputs: Array<Array<Float32Array>>): boolean;
+}
+
+type GateConstructor = new (options: {processorOptions: Record<string, unknown>}) => GateProcessor;
+
+interface GateMode {
+	apply: boolean;
+	auto: boolean;
+	slider?: number;
+}
+
+function loadGate(mode: GateMode): GateProcessor {
+	let Processor: GateConstructor | null = null;
+	const sandbox = vm.createContext({
+		sampleRate: SAMPLE_RATE,
+		currentTime: 0,
+		AudioWorkletProcessor: class {
+			port = {postMessage: () => undefined, onmessage: null};
+		},
+		registerProcessor: (_name: string, processor: GateConstructor) => {
+			Processor = processor;
+		},
+	});
+	vm.runInContext(gateWorkletSource, sandbox);
+	if (!Processor) throw new Error('voice gate worklet did not register');
+	return new (Processor as GateConstructor)({
+		processorOptions: {
+			apply: mode.apply,
+			auto: mode.auto,
+			manualThresholdRms: getLocalSpeakingThresholdRms(mode.slider ?? 50),
+			channelCount: 1,
+			minRms: LOCAL_AUTO_MIN_RMS,
+			maxRms: LOCAL_MAX_RMS,
+			releaseMs: SPEAKING_LOCAL_RELEASE_MS,
+		},
+	});
+}
+
+function createRandom(seed: number): () => number {
+	let state = seed >>> 0;
+	return () => {
+		state = (state + 0x6d2b79f5) >>> 0;
+		let value = state;
+		value = Math.imul(value ^ (value >>> 15), value | 1);
+		value ^= value + Math.imul(value ^ (value >>> 7), value | 61);
+		return ((value ^ (value >>> 14)) >>> 0) / 4294967296;
+	};
+}
+
+const UNIFORM_RMS_TO_PEAK = Math.sqrt(3);
+
+function noise(random: () => number, rms: number): number {
+	return (random() * 2 - 1) * rms * UNIFORM_RMS_TO_PEAK;
+}
+
+function syllable(index: number, periodSamples: number, onSamples: number): boolean {
+	return index % periodSamples < onSamples;
+}
+
+function tone(index: number, rms: number): number {
+	return Math.sin((2 * Math.PI * 180 * index) / SAMPLE_RATE) * rms * Math.SQRT2;
+}
+
+interface Timeline {
+	gateSpeaking: boolean;
+	localSpeaking: Array<boolean>;
+	remoteSpeaking: Array<boolean>;
+}
+
+function runPipeline(mode: GateMode, seconds: number, signal: (index: number) => number, transmit = true): Timeline {
+	const gate = loadGate(mode);
+	const track = {readyState: 'live'} as MediaStreamTrack;
+	const meter = new VoiceSpeakingMeter(
+		() => track,
+		() => undefined,
+	);
+	let remote = transitionVoiceRemoteSpeakingSnapshot(createVoiceRemoteSpeakingSnapshot(), {
+		type: 'remote.attach',
+		identity: 'peer',
+		track: 'track',
+	});
+	const history = new Float32Array(ANALYSER_WINDOW);
+	const timeline: Timeline = {gateSpeaking: false, localSpeaking: [], remoteSpeaking: []};
+	const seedEnd = 0.3 * SAMPLE_RATE;
+	const total = Math.floor(seconds * SAMPLE_RATE);
+	let nextTick = TICK_SAMPLES;
+	for (let start = 0; start < total; start += QUANTUM) {
+		const input = new Float32Array(QUANTUM);
+		for (let offset = 0; offset < QUANTUM; offset++) input[offset] = signal(start + offset);
+		const output = new Float32Array(QUANTUM);
+		gate.process([[input]], [[output]]);
+		if (start >= seedEnd && gate.speaking) timeline.gateSpeaking = true;
+		history.copyWithin(0, QUANTUM);
+		history.set(transmit ? output : new Float32Array(QUANTUM), ANALYSER_WINDOW - QUANTUM);
+		const end = start + QUANTUM;
+		if (end < nextTick) continue;
+		nextTick += TICK_SAMPLES;
+		const nowMs = (end / SAMPLE_RATE) * 1000;
+		audio.window.set(history);
+		meter.tick(nowMs);
+		remote = transitionVoiceRemoteSpeakingSnapshot(remote, {
+			type: 'remote.tick',
+			identity: 'peer',
+			rms: computeTimeDomainRms(history),
+			threshold: readSpeakingThresholdRms(),
+			nowMs,
+		});
+		remote = clearVoiceRemoteSpeakingCommands(remote);
+		timeline.localSpeaking.push(meter.speaking);
+		timeline.remoteSpeaking.push(remote.context.analysers.get('peer')?.speaking ?? false);
+	}
+	meter.dispose();
+	return timeline;
+}
+
+const VAD_AUTO: GateMode = {apply: true, auto: true};
+const VAD_MANUAL: GateMode = {apply: true, auto: false, slider: 50};
+const VAD_MANUAL_SENSITIVE: GateMode = {apply: true, auto: false, slider: 10};
+const PUSH_TO_TALK: GateMode = {apply: false, auto: true};
+const MODES: Array<[string, GateMode]> = [
+	['voice activity with auto sensitivity', VAD_AUTO],
+	['voice activity with a manual threshold', VAD_MANUAL],
+	['voice activity with a sensitive manual threshold', VAD_MANUAL_SENSITIVE],
+	['push to talk', PUSH_TO_TALK],
+];
+
+function suppressionResidue(seed: number): (index: number) => number {
+	const random = createRandom(seed);
+	return (index) => {
+		const click = syllable(index, 0.3 * SAMPLE_RATE, 0.02 * SAMPLE_RATE);
+		return noise(random, click ? 0.001 : 0.00003);
+	};
+}
+
+function hardwareMutedWhilePeersTalk(seed: number): (index: number) => number {
+	const random = createRandom(seed);
+	return (index) => {
+		if (index < SAMPLE_RATE) return 0;
+		const peerTalking = syllable(index, 0.25 * SAMPLE_RATE, 0.15 * SAMPLE_RATE);
+		return peerTalking ? tone(index, 0.00004) + noise(random, 0.00001) : 0;
+	};
+}
+
+function speechInRoomNoise(seed: number): (index: number) => number {
+	const random = createRandom(seed);
+	return (index) => {
+		const room = noise(random, 0.0003);
+		if (index < SAMPLE_RATE) return room;
+		return syllable(index, 0.2 * SAMPLE_RATE, 0.15 * SAMPLE_RATE) ? room + tone(index, 0.05) : room;
+	};
+}
+
+describe('local speaking indicator', () => {
+	beforeEach(() => {
+		vi.stubGlobal('MediaStream', class {});
+		audio.window.fill(0);
+		audio.state = 'running';
+	});
+
+	afterEach(() => {
+		vi.unstubAllGlobals();
+	});
+
+	it.each(MODES)('stays dark on noise left behind by noise suppression in %s', (_name, mode) => {
+		const timeline = runPipeline(mode, 3, suppressionResidue(1));
+		expect(timeline.localSpeaking).not.toContain(true);
+		expect(timeline.remoteSpeaking).not.toContain(true);
+	});
+
+	it.each(MODES)('stays dark when a hardware-muted mic only carries peer bleed in %s', (_name, mode) => {
+		const timeline = runPipeline(mode, 4, hardwareMutedWhilePeersTalk(2));
+		expect(timeline.localSpeaking).not.toContain(true);
+		expect(timeline.remoteSpeaking).not.toContain(true);
+	});
+
+	it.each(MODES)('lights for real speech in %s', (_name, mode) => {
+		const timeline = runPipeline(mode, 3, speechInRoomNoise(3));
+		expect(timeline.localSpeaking).toContain(true);
+		expect(timeline.localSpeaking.slice(0, 18)).not.toContain(true);
+	});
+
+	it.each(MODES)('agrees tick for tick with how peers judge the same audio in %s', (_name, mode) => {
+		for (const signal of [suppressionResidue(4), hardwareMutedWhilePeersTalk(5), speechInRoomNoise(6)]) {
+			const timeline = runPipeline(mode, 3, signal);
+			expect(timeline.localSpeaking).toEqual(timeline.remoteSpeaking);
+		}
+	});
+
+	it('never lights while the outgoing track is muted, even for loud speech', () => {
+		for (const [, mode] of MODES) {
+			const timeline = runPipeline(mode, 3, speechInRoomNoise(7), false);
+			expect(timeline.localSpeaking).not.toContain(true);
+		}
+	});
+
+	it('keeps the transmit gate opening on those inaudible inputs, so the gate cannot drive the indicator', () => {
+		expect(runPipeline(VAD_AUTO, 3, suppressionResidue(1)).gateSpeaking).toBe(true);
+		expect(runPipeline(VAD_AUTO, 4, hardwareMutedWhilePeersTalk(2)).gateSpeaking).toBe(true);
+	});
+
+	it('does not hold a stale reading while the analysing context is not running', () => {
+		const track = {readyState: 'live'} as MediaStreamTrack;
+		const meter = new VoiceSpeakingMeter(
+			() => track,
+			() => undefined,
+		);
+		audio.window.fill(0.1);
+		meter.tick(0);
+		meter.tick(50);
+		expect(meter.speaking).toBe(true);
+		audio.state = 'suspended';
+		for (let nowMs = 100; nowMs <= 400; nowMs += 50) meter.tick(nowMs);
+		expect(meter.speaking).toBe(false);
+		meter.dispose();
+	});
+});

fluxer_app/src/features/voice/engine/VoiceSpeakingDetection.ts

@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {Logger} from '@app/features/platform/utils/AppLogger';
+import {getSharedVoiceAudioContext} from '@app/features/voice/engine/VoiceSharedAudioContext';
+import {
+	advanceSpeakingHysteresis,
+	getRemoteSpeakingThresholdRms,
+	SILENT_SPEAKING_HYSTERESIS,
+	type SpeakingHysteresis,
+} from '@app/features/voice/engine/VoiceSpeakingThreshold';
+import VoiceSettings from '@app/features/voice/state/VoiceSettings';
+
+const logger = new Logger('VoiceSpeakingDetection');
+export const SPEAKING_ANALYSER_INTERVAL_MS = 50;
+const SPEAKING_ANALYSER_FFT_SIZE = 512;
+const SPEAKING_ANALYSER_SMOOTHING = 0.2;
+
+export function computeTimeDomainRms(samples: Float32Array): number {
+	if (samples.length === 0) return 0;
+	let sumSquares = 0;
+	for (let i = 0; i < samples.length; i++) {
+		const sample = samples[i]!;
+		sumSquares += sample * sample;
+	}
+	return Math.sqrt(sumSquares / samples.length);
+}
+
+export function readSpeakingThresholdRms(): number {
+	return getRemoteSpeakingThresholdRms(VoiceSettings.getVadThreshold());
+}
+
+export function createSpeakingAnalyser(context: BaseAudioContext): AnalyserNode {
+	const analyser = context.createAnalyser();
+	analyser.fftSize = SPEAKING_ANALYSER_FFT_SIZE;
+	analyser.smoothingTimeConstant = SPEAKING_ANALYSER_SMOOTHING;
+	return analyser;
+}
+
+export function createSpeakingSamples(): Float32Array<ArrayBuffer> {
+	return new Float32Array(new ArrayBuffer(SPEAKING_ANALYSER_FFT_SIZE * Float32Array.BYTES_PER_ELEMENT));
+}
+
+interface SpeakingMeterTap {
+	context: BaseAudioContext;
+	source: MediaStreamAudioSourceNode;
+	analyser: AnalyserNode;
+}
+
+export class VoiceSpeakingMeter {
+	private track: MediaStreamTrack | null = null;
+	private tap: SpeakingMeterTap | null = null;
+	private hysteresis = SILENT_SPEAKING_HYSTERESIS;
+	private readonly samples = createSpeakingSamples();
+	private readonly timer: ReturnType<typeof setInterval>;
+
+	constructor(
+		private readonly readTrack: () => MediaStreamTrack | null,
+		private readonly onChange: () => void,
+	) {
+		this.timer = setInterval(() => this.tick(performance.now()), SPEAKING_ANALYSER_INTERVAL_MS);
+	}
+
+	get speaking(): boolean {
+		return this.hysteresis.speaking;
+	}
+
+	tick(nowMs: number): void {
+		const tap = this.syncTap();
+		if (!tap) {
+			this.update(SILENT_SPEAKING_HYSTERESIS);
+			return;
+		}
+		const running = tap.context.state === 'running';
+		if (running) tap.analyser.getFloatTimeDomainData(this.samples);
+		const rms = running ? computeTimeDomainRms(this.samples) : 0;
+		this.update(advanceSpeakingHysteresis(this.hysteresis, rms, readSpeakingThresholdRms(), nowMs));
+	}
+
+	dispose(): void {
+		clearInterval(this.timer);
+		this.closeTap();
+		this.hysteresis = SILENT_SPEAKING_HYSTERESIS;
+	}
+
+	private syncTap(): SpeakingMeterTap | null {
+		const candidate = this.readTrack();
+		const track = candidate?.readyState === 'live' ? candidate : null;
+		if (track === this.track) return this.tap;
+		this.closeTap();
+		this.track = track;
+		this.hysteresis = {...SILENT_SPEAKING_HYSTERESIS, speaking: this.hysteresis.speaking};
+		const context = track ? getSharedVoiceAudioContext() : null;
+		if (!track || !context) return null;
+		try {
+			const source = context.createMediaStreamSource(new MediaStream([track]));
+			const analyser = createSpeakingAnalyser(context);
+			source.connect(analyser);
+			this.tap = {context, source, analyser};
+		} catch (error) {
+			logger.warn('Failed to attach the local speaking analyser', {error});
+		}
+		return this.tap;
+	}
+
+	private closeTap(): void {
+		const tap = this.tap;
+		this.track = null;
+		this.tap = null;
+		if (!tap) return;
+		tap.source.disconnect();
+		tap.analyser.disconnect();
+	}
+
+	private update(next: SpeakingHysteresis): void {
+		const changed = next.speaking !== this.hysteresis.speaking;
+		this.hysteresis = next;
+		if (changed) this.onChange();
+	}
+}

fluxer_app/src/features/voice/engine/VoiceSpeakingThreshold.ts

@@ -8,10 +8,18 @@ const REMOTE_MIN_RMS = 0.002;
 const REMOTE_DEFAULT_RMS = 0.006;
 const REMOTE_MAX_RMS = 0.035;
 export const SPEAKING_LOCAL_RELEASE_MS = 180;
-export const SPEAKING_REMOTE_RELEASE_MS = 220;
-export const SPEAKING_REMOTE_ATTACK_MS = 30;
+const SPEAKING_REMOTE_RELEASE_MS = 220;
+const SPEAKING_REMOTE_ATTACK_MS = 30;
 const SLIDER_DEFAULT = 50;
 
+export interface SpeakingHysteresis {
+	speaking: boolean;
+	aboveSinceMs: number | null;
+	belowSinceMs: number | null;
+}
+
+export const SILENT_SPEAKING_HYSTERESIS: SpeakingHysteresis = {speaking: false, aboveSinceMs: null, belowSinceMs: null};
+
 function clampSlider(slider: number): number {
 	if (!Number.isFinite(slider)) return SLIDER_DEFAULT;
 	if (slider < 0) return 0;
@@ -40,3 +48,25 @@ export function getLocalSpeakingThresholdRms(slider: number): number {
 export function getRemoteSpeakingThresholdRms(slider: number): number {
 	return interpolateThreshold(slider, REMOTE_MIN_RMS, REMOTE_DEFAULT_RMS, REMOTE_MAX_RMS);
 }
+
+export function advanceSpeakingHysteresis(
+	state: SpeakingHysteresis,
+	rms: number,
+	thresholdRms: number,
+	nowMs: number,
+): SpeakingHysteresis {
+	if (rms >= thresholdRms) {
+		const aboveSinceMs = state.aboveSinceMs ?? nowMs;
+		return {
+			speaking: state.speaking || nowMs - aboveSinceMs >= SPEAKING_REMOTE_ATTACK_MS,
+			aboveSinceMs,
+			belowSinceMs: null,
+		};
+	}
+	const belowSinceMs = state.belowSinceMs ?? nowMs;
+	return {
+		speaking: state.speaking && nowMs - belowSinceMs < SPEAKING_REMOTE_RELEASE_MS,
+		aboveSinceMs: null,
+		belowSinceMs,
+	};
+}

fluxer_app/src/features/voice/engine/mesh/VoiceMeshEngine.ts

@@ -29,14 +29,7 @@ import {
 } from '@app/features/voice/engine/mesh/VoiceMeshSignal';
 import {VoiceMeshSignalClient} from '@app/features/voice/engine/mesh/VoiceMeshSignalClient';
 import type {NormalizedVoiceState} from '@app/features/voice/engine/VoiceGatewayStateMachine';
-import {getSharedVoiceAudioContext} from '@app/features/voice/engine/VoiceSharedAudioContext';
-import {
-	getRemoteSpeakingThresholdRms,
-	SPEAKING_REMOTE_RELEASE_MS,
-} from '@app/features/voice/engine/VoiceSpeakingThreshold';
-import {computeTimeDomainRms} from '@app/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter';
 import voiceEngineV2AppVoiceStateAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppVoiceStateAdapter';
-import VoiceSettings from '@app/features/voice/state/VoiceSettings';
 import {buildVoiceParticipantIdentity} from '@app/features/voice/utils/VoiceParticipantIdentity';
 import {ME} from '@fluxer/constants/src/AppConstants';
 import {
@@ -56,7 +49,6 @@ import {
 	type Room,
 	RTCEngine,
 	type SignalOptions,
-	SpeakerInfo,
 	Track,
 	TrackInfo,
 	TrackInvalidError,
@@ -68,7 +60,6 @@ const logger = new Logger('VoiceMeshEngine');
 
 const INBOX_LIMIT = 64;
 const CONVERSION_GRACE_MS = 10_000;
-const SPEAKING_TICK_MS = 50;
 const SCREEN_SHARE_BUDGET_BPS = 12_000_000;
 const CAMERA_BUDGET_BPS = 4_000_000;
 const MESH_VIDEO_CODEC_MIME_TYPES = ['video/H264', 'video/VP8'];
@@ -81,16 +72,6 @@ export interface VoiceMeshTarget {
 	userId: string;
 }
 
-interface SpeakingAnalyser {
-	track: MediaStreamTrack;
-	source: MediaStreamAudioSourceNode;
-	analyser: AnalyserNode;
-	samples: Float32Array<ArrayBuffer>;
-	timer: ReturnType<typeof setInterval>;
-	lastLoudAt: number | null;
-	active: boolean;
-}
-
 let activeEngine: VoiceMeshEngine | null = null;
 
 function mergeStatsReports(reports: ReadonlyArray<readonly [string, RTCStatsReport]>): RTCStatsReport {
@@ -156,7 +137,6 @@ export class VoiceMeshEngine extends RTCEngine {
 	private tracks: {v: number; tracks: Array<VoiceMeshTrackInfo>} = {v: this.version, tracks: []};
 	private conversionTimer: ReturnType<typeof setTimeout> | null = null;
 	private conversionHeld = false;
-	private speaking: SpeakingAnalyser | null = null;
 
 	constructor(options: InternalRoomOptions, target: VoiceMeshTarget) {
 		super(options);
@@ -237,7 +217,6 @@ export class VoiceMeshEngine extends RTCEngine {
 			),
 		];
 		this.syncRoster();
-		this.syncSpeaking();
 	}
 
 	override async createSender(
@@ -296,7 +275,6 @@ export class VoiceMeshEngine extends RTCEngine {
 
 	syncSender(src: VoiceMeshSource): Promise<void> {
 		this.refreshTracks();
-		if (src === 'microphone') this.syncSpeaking();
 		return Promise.all(Array.from(this.peers.values(), (peer) => peer.requestSenderSync(src))).then(() => undefined);
 	}
 
@@ -584,55 +562,6 @@ export class VoiceMeshEngine extends RTCEngine {
 		this.conversionTimer = null;
 	}
 
-	private syncSpeaking(): void {
-		const microphone = this.senders.microphone;
-		const track = this.room && microphone.sid !== null ? microphone.track : null;
-		if ((this.speaking?.track ?? null) === track) return;
-		this.stopSpeaking();
-		if (!track) return;
-		const context = getSharedVoiceAudioContext();
-		if (!context) return;
-		const source = context.createMediaStreamSource(new MediaStream([track]));
-		const analyser = context.createAnalyser();
-		analyser.fftSize = 512;
-		source.connect(analyser);
-		const speaking: SpeakingAnalyser = {
-			track,
-			source,
-			analyser,
-			samples: new Float32Array(analyser.fftSize),
-			timer: setInterval(() => this.tickSpeaking(speaking), SPEAKING_TICK_MS),
-			lastLoudAt: null,
-			active: false,
-		};
-		this.speaking = speaking;
-	}
-
-	private tickSpeaking(speaking: SpeakingAnalyser): void {
-		speaking.analyser.getFloatTimeDomainData(speaking.samples);
-		const level = computeTimeDomainRms(speaking.samples);
-		const now = performance.now();
-		if (level >= getRemoteSpeakingThresholdRms(VoiceSettings.getVadThreshold())) speaking.lastLoudAt = now;
-		const active = speaking.lastLoudAt !== null && now - speaking.lastLoudAt < SPEAKING_REMOTE_RELEASE_MS;
-		if (active === speaking.active) return;
-		speaking.active = active;
-		this.emitLocalSpeaking(level, active);
-	}
-
-	private stopSpeaking(): void {
-		const speaking = this.speaking;
-		if (!speaking) return;
-		this.speaking = null;
-		clearInterval(speaking.timer);
-		speaking.source.disconnect();
-		speaking.analyser.disconnect();
-		if (speaking.active) this.emitLocalSpeaking(0, false);
-	}
-
-	private emitLocalSpeaking(level: number, active: boolean): void {
-		this.emit(EngineEvent.SpeakersChanged, [new SpeakerInfo({sid: this.localParticipantSid, level, active})]);
-	}
-
 	private stopMesh(): void {
 		if (!this.room) return;
 		this.room = null;
@@ -646,7 +575,6 @@ export class VoiceMeshEngine extends RTCEngine {
 		}
 		this.peers.clear();
 		this.inbox.clear();
-		this.stopSpeaking();
 		flushVoiceMeshConnectionReports();
 	}
 

fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppMediaExecutionAdapter.tsx

@@ -19,6 +19,7 @@ import {Store} from '@app/features/voice/engine/Store';
 import VoiceDevicePermissionState from '@app/features/voice/engine/VoiceDevicePermissionState';
 import type {EffectiveAudioState} from '@app/features/voice/engine/VoiceEffectiveAudioState';
 import {getEffectiveAudioState} from '@app/features/voice/engine/VoiceEffectiveAudioState';
+import {noteLocalVoiceActivity} from '@app/features/voice/engine/VoiceIdleActivityBridge';
 import {acquireVoiceInputContext, type VoiceInputContextLease} from '@app/features/voice/engine/VoiceInputAudioContext';
 import {
 	createVoiceMicrophoneFailureLatchSnapshot,
@@ -28,7 +29,7 @@ import {
 	transitionVoiceMicrophoneFailureLatchSnapshot,
 	type VoiceMicrophoneFailureLatchEvent,
 } from '@app/features/voice/engine/VoiceLocalAudioReconcilePolicy';
-import {resolveLocalSpeakingOverrideState} from '@app/features/voice/engine/VoiceLocalSpeakingGate';
+import {isLocalSpeakingSuppressed} from '@app/features/voice/engine/VoiceLocalSpeakingGate';
 import {
 	getRoomFromMediaEngine,
 	getVoiceConnectionContextFromMediaEngine,
@@ -43,6 +44,7 @@ import {
 	type VoiceMediaEvent,
 	type VoiceMediaSnapshot,
 } from '@app/features/voice/engine/VoiceMediaStateMachine';
+import {VoiceSpeakingMeter} from '@app/features/voice/engine/VoiceSpeakingDetection';
 import type {VoiceStateSyncPartial} from '@app/features/voice/engine/VoiceStateSyncTypes';
 import {
 	enforceLocalMediaPublicationCap,
@@ -231,6 +233,7 @@ function getEffectiveSelfMuteForVoiceStatePayloadFromV2AudioControls(): boolean
 
 class VoiceEngineV2AppMediaExecutionAdapter extends Store {
 	private speakingDetectorCleanup: (() => void) | null = null;
+	private localSpeakingMeter: VoiceSpeakingMeter | null = null;
 	private mediaStateSnapshot: VoiceMediaSnapshot = createVoiceMediaSnapshot();
 	private microphoneEnablePromise: Promise<void> | null = null;
 	private microphoneRefreshQueue: Promise<void> = Promise.resolve();
@@ -373,7 +376,7 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store {
 	private async enforceSpeakPermissionMute(room: Room): Promise<void> {
 		await this.disableMicrophone(room);
 		this.syncVoiceState({self_mute: true});
-		this.syncLocalSpeakingOverride(room);
+		this.syncLocalSpeaking(room);
 	}
 
 	async prepareMicrophonePermissionForConnect(): Promise<boolean> {
@@ -485,7 +488,7 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store {
 			} else {
 				this.syncVoiceState({self_mute: audioState.selfMute});
 			}
-			this.syncLocalSpeakingOverride(room);
+			this.syncLocalSpeaking(room);
 		} catch (e: unknown) {
 			if (isPermissionDeniedError(e)) {
 				this.handleMicrophonePermissionDenied();
@@ -542,7 +545,7 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store {
 		}
 		if (audioState.effectiveDeaf || audioState.serverMute || audioState.selfMute) {
 			this.setAudioPublicationsMuted(room, true, 'voice state update');
-			this.syncLocalSpeakingOverride(room);
+			this.syncLocalSpeaking(room);
 			return;
 		}
 		if (!this.hasLiveMicrophonePublication(room)) {
@@ -562,7 +565,7 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store {
 			}
 		}
 		this.setAudioPublicationsMuted(room, this.shouldMuteMicrophonePublication(), 'voice state update');
-		this.syncLocalSpeakingOverride(room);
+		this.syncLocalSpeaking(room);
 	}
 
 	private hasMicrophonePublication(room: Room): boolean {
@@ -1506,7 +1509,7 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store {
 		}
 		this.syncVoiceState({self_mute: targetMute});
 		this.updateMediaAudioControls();
-		this.syncLocalSpeakingOverride(room);
+		this.syncLocalSpeaking(room);
 	}
 
 	applyPushToMuteHold(held: boolean, room: Room | null, getCurrentUserVoiceState: () => VoiceState | null): void {
@@ -1534,7 +1537,7 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store {
 		}
 		this.syncVoiceState({self_mute: targetMute});
 		this.updateMediaAudioControls();
-		this.syncLocalSpeakingOverride(room);
+		this.syncLocalSpeaking(room);
 	}
 
 	handlePushToTalkModeChange(room: Room | null, getCurrentUserVoiceState: () => VoiceState | null): void {
@@ -1560,7 +1563,7 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store {
 		}
 		this.syncVoiceState({self_mute: targetMute});
 		this.updateMediaAudioControls();
-		this.syncLocalSpeakingOverride(room);
+		this.syncLocalSpeaking(room);
 	}
 
 	getMuteReason(voiceState: VoiceState | null, guildId?: string | null, channelId?: string | null): VoiceMuteReason {
@@ -1671,13 +1674,9 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store {
 		return true;
 	}
 
-	private getLocalSpeakingOverrideState(room: Room): boolean | null {
-		if (!room.localParticipant) return false;
+	private shouldSuppressLocalSpeaking(room: Room): boolean {
 		const publication = this.getMicrophonePublication(room);
-		return resolveLocalSpeakingOverrideState({
-			pushToTalkActive: isPushToTalkActiveFromAppState(),
-			pushToMuteActive: isPushToMuteActiveFromAppState(),
-			pushToMuteHeld: Keybind.pushToMuteHeld,
+		return isLocalSpeakingSuppressed({
 			selfDeaf: LocalVoiceState.getSelfDeaf(),
 			effectiveSelfMute: getEffectiveSelfMuteForVoiceStatePayloadFromV2AudioControls(),
 			hasMicrophonePublication: Boolean(publication),
@@ -1703,10 +1702,9 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store {
 		return selectVoiceEngineV2AppParticipant(snapshot, identity)?.isAudioLevelSpeaking ?? null;
 	}
 
-	private syncLocalSpeakingOverride(room: Room | null): void {
+	private syncLocalSpeaking(room: Room | null): void {
 		if (!room?.localParticipant) return;
-		const speaking = this.getLocalSpeakingOverrideState(room);
-		if (speaking === null) return;
+		const speaking = !this.shouldSuppressLocalSpeaking(room) && (this.localSpeakingMeter?.speaking ?? false);
 		this.setLocalParticipantAudioLevelSpeaking(room, speaking);
 	}
 
@@ -1714,28 +1712,27 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store {
 		this.stopLocalSpeakingDetector(room);
 		const track = this.getLocalAudioTrack(room);
 		if (!track || !room.localParticipant) return;
-		let stopLevel: (() => void) | null = null;
-		const bind = (): void => {
-			stopLevel?.();
-			stopLevel = null;
-			this.setLocalParticipantAudioLevelSpeaking(room, this.getLocalSpeakingOverrideState(room) ?? false);
+		const meter = new VoiceSpeakingMeter(
+			() => track.mediaStreamTrack,
+			() => {
+				this.syncLocalSpeaking(room);
+				if (meter.speaking) noteLocalVoiceActivity(room.localParticipant, {force: true, speaking: true});
+			},
+		);
+		const reportRuntime = (): void => {
 			const processor = readVoiceInputProcessor(track);
-			if (!processor) {
-				this.reportRawMicrophoneRuntime(track);
-				return;
-			}
-			processor.reportRuntime();
-			stopLevel = processor.onLevel((level) => {
-				this.setLocalParticipantAudioLevelSpeaking(room, this.getLocalSpeakingOverrideState(room) ?? level.speaking);
-			});
+			if (processor) processor.reportRuntime();
+			else this.reportRawMicrophoneRuntime(track);
 		};
-		track.on(TrackEvent.TrackProcessorUpdate, bind);
+		track.on(TrackEvent.TrackProcessorUpdate, reportRuntime);
+		this.localSpeakingMeter = meter;
 		this.speakingDetectorCleanup = () => {
-			track.off(TrackEvent.TrackProcessorUpdate, bind);
-			stopLevel?.();
+			track.off(TrackEvent.TrackProcessorUpdate, reportRuntime);
+			meter.dispose();
 			NoiseSuppressionAvailability.clearVoiceInputRuntime(track);
 		};
-		bind();
+		reportRuntime();
+		meter.tick(performance.now());
 		this.transitionMediaState({type: 'speakingDetector.attach'});
 	}
 
@@ -1743,6 +1740,7 @@ class VoiceEngineV2AppMediaExecutionAdapter extends Store {
 		this.transitionMediaState({type: 'speakingDetector.detach'});
 		this.speakingDetectorCleanup?.();
 		this.speakingDetectorCleanup = null;
+		this.localSpeakingMeter = null;
 		if (room) this.setLocalParticipantAudioLevelSpeaking(room, false);
 	}
 }

fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppPushToTalkSelfMute.test.ts

@@ -3,10 +3,11 @@
 
 import type {Room} from 'livekit-client';
 import {Track} from 'livekit-client';
-import {beforeEach, describe, expect, it, vi} from 'vitest';
+import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
 
 const syncLocalVoiceStateWithServer = vi.fn();
 const setVoiceEngineV2ParticipantAudioLevelSpeaking = vi.fn(() => true);
+const outgoingAudio = vi.hoisted(() => ({rms: 0}));
 
 const keybind = {
 	pushToTalkEnabled: true,
@@ -63,6 +64,18 @@ vi.mock('@app/features/voice/state/LocalVoiceState', () => ({default: localVoice
 
 vi.mock('@app/features/voice/engine/MediaEngineFacade', () => ({default: {}, useMediaEngineVersion: () => 0}));
 
+vi.mock('@app/features/voice/engine/VoiceSharedAudioContext', async (importOriginal) => ({
+	...(await importOriginal<typeof import('@app/features/voice/engine/VoiceSharedAudioContext')>()),
+	getSharedVoiceAudioContext: () => ({
+		state: 'running',
+		createMediaStreamSource: () => ({connect: () => undefined, disconnect: () => undefined}),
+		createAnalyser: () => ({
+			disconnect: () => undefined,
+			getFloatTimeDomainData: (target: Float32Array) => target.fill(outgoingAudio.rms),
+		}),
+	}),
+}));
+
 vi.mock('@app/features/voice/engine/VoiceMediaEngineBridge', () => ({
 	getRoomFromMediaEngine: () => null,
 	getVoiceConnectionContextFromMediaEngine: () => null,
@@ -82,7 +95,12 @@ const {default: adapter} = await import('@app/features/voice/engine/v2/VoiceEngi
 interface FakeMicrophonePublication {
 	source: Track.Source;
 	isMuted: boolean;
-	track: {mediaStreamTrack: {readyState: string}};
+	track: {
+		mediaStreamTrack: {readyState: string; getSettings(): MediaTrackSettings; getConstraints(): MediaTrackConstraints};
+		getProcessor(): undefined;
+		on(): void;
+		off(): void;
+	};
 	mute: ReturnType<typeof vi.fn>;
 	unmute: ReturnType<typeof vi.fn>;
 }
@@ -91,7 +109,12 @@ function createMicrophonePublication(): FakeMicrophonePublication {
 	const publication: FakeMicrophonePublication = {
 		source: Track.Source.Microphone,
 		isMuted: true,
-		track: {mediaStreamTrack: {readyState: 'live'}},
+		track: {
+			mediaStreamTrack: {readyState: 'live', getSettings: () => ({}), getConstraints: () => ({})},
+			getProcessor: () => undefined,
+			on: () => undefined,
+			off: () => undefined,
+		},
 		mute: vi.fn(async () => {
 			publication.isMuted = true;
 		}),
@@ -120,7 +143,7 @@ function lastSyncedSelfMute(): unknown {
 	return (call?.[0] as {self_mute?: boolean} | undefined)?.self_mute;
 }
 
-function lastSpeakingOverride(): unknown {
+function lastSpeaking(): unknown {
 	return setVoiceEngineV2ParticipantAudioLevelSpeaking.mock.calls.at(-1)?.[1 as never];
 }
 
@@ -147,7 +170,7 @@ describe('push to talk with self-mute', () => {
 		expect(publication.mute).toHaveBeenCalled();
 		expect(publication.isMuted).toBe(true);
 		expect(lastSyncedSelfMute()).toBe(true);
-		expect(lastSpeakingOverride()).toBe(false);
+		expect(lastSpeaking()).toBe(false);
 		adapter.applyPushToTalkHold(false, room, noServerVoiceState);
 		expect(publication.unmute).not.toHaveBeenCalled();
 		expect(lastSyncedSelfMute()).toBe(true);
@@ -160,11 +183,11 @@ describe('push to talk with self-mute', () => {
 		adapter.applyPushToTalkHold(true, room, noServerVoiceState);
 		expect(publication.unmute).toHaveBeenCalledTimes(1);
 		expect(lastSyncedSelfMute()).toBe(false);
-		expect(lastSpeakingOverride()).toBe(true);
+		expect(lastSpeaking()).toBe(false);
 		adapter.applyPushToTalkHold(false, room, noServerVoiceState);
 		expect(publication.isMuted).toBe(true);
 		expect(lastSyncedSelfMute()).toBe(true);
-		expect(lastSpeakingOverride()).toBe(false);
+		expect(lastSpeaking()).toBe(false);
 	});
 
 	it('transmits when the user unmutes while the key is still held', () => {
@@ -228,3 +251,71 @@ describe('push to talk with self-mute', () => {
 		enableMicrophone.mockRestore();
 	});
 });
+
+interface LocalSpeakingMeterProbe {
+	startLocalSpeakingDetector(room: Room): void;
+	stopLocalSpeakingDetector(room?: Room | null): void;
+	localSpeakingMeter: {tick(nowMs: number): void} | null;
+}
+
+describe('local speaking indicator in push to talk', () => {
+	const probe = adapter as unknown as LocalSpeakingMeterProbe;
+	let nowMs = 0;
+
+	function tick(rms: number, ticks = 1): void {
+		outgoingAudio.rms = rms;
+		for (let index = 0; index < ticks; index++) {
+			nowMs += 50;
+			probe.localSpeakingMeter?.tick(nowMs);
+		}
+	}
+
+	beforeEach(() => {
+		vi.stubGlobal('MediaStream', class {});
+		keybind.pushToTalkEnabled = true;
+		keybind.pushToTalkHeld = false;
+		localVoiceState.selfMute = false;
+		localVoiceState.selfDeaf = false;
+		outgoingAudio.rms = 0;
+		setVoiceEngineV2ParticipantAudioLevelSpeaking.mockClear();
+	});
+
+	afterEach(() => {
+		probe.stopLocalSpeakingDetector();
+		vi.unstubAllGlobals();
+	});
+
+	function startWithMicrophone(): {room: Room; publication: FakeMicrophonePublication} {
+		const {room, publications} = createRoom();
+		const publication = createMicrophonePublication();
+		publications.set('mic', publication);
+		probe.startLocalSpeakingDetector(room);
+		return {room, publication};
+	}
+
+	it('stays dark while the key is held and nothing audible is sent', () => {
+		const {room} = startWithMicrophone();
+		adapter.applyPushToTalkHold(true, room, noServerVoiceState);
+		tick(0.001, 10);
+		expect(setVoiceEngineV2ParticipantAudioLevelSpeaking).not.toHaveBeenCalledWith(expect.anything(), true);
+	});
+
+	it('lights for speech sent while the key is held and goes dark the moment it is released', () => {
+		const {room} = startWithMicrophone();
+		adapter.applyPushToTalkHold(true, room, noServerVoiceState);
+		tick(0.05, 2);
+		expect(lastSpeaking()).toBe(true);
+		adapter.applyPushToTalkHold(false, room, noServerVoiceState);
+		expect(lastSpeaking()).toBe(false);
+		tick(0.05, 4);
+		expect(setVoiceEngineV2ParticipantAudioLevelSpeaking).toHaveBeenLastCalledWith(expect.anything(), false);
+	});
+
+	it('never lights for a self-muted user, however loud the microphone is', () => {
+		const {room} = startWithMicrophone();
+		localVoiceState.selfMute = true;
+		adapter.applyPushToTalkHold(true, room, noServerVoiceState);
+		tick(0.2, 10);
+		expect(setVoiceEngineV2ParticipantAudioLevelSpeaking).not.toHaveBeenCalledWith(expect.anything(), true);
+	});
+});

fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.ts

@@ -18,7 +18,13 @@ import {
 	getSharedVoiceAudioContext,
 	resolveAudioContextConstructor,
 } from '@app/features/voice/engine/VoiceSharedAudioContext';
-import {getRemoteSpeakingThresholdRms} from '@app/features/voice/engine/VoiceSpeakingThreshold';
+import {
+	computeTimeDomainRms,
+	createSpeakingAnalyser,
+	createSpeakingSamples,
+	readSpeakingThresholdRms,
+	SPEAKING_ANALYSER_INTERVAL_MS,
+} from '@app/features/voice/engine/VoiceSpeakingDetection';
 import {VoiceTrackKind, VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
 import {
 	assertNonEmptyString,
@@ -30,23 +36,11 @@ import {
 	clearRemoteVoicePlaybackBoost,
 	setRemoteVoicePlaybackBoost,
 } from '@app/features/voice/state/RemoteVoicePlaybackBoost';
-import VoiceSettings from '@app/features/voice/state/VoiceSettings';
 import type {Participant, RemoteAudioTrack, RemoteTrack, RemoteTrackPublication, Room} from 'livekit-client';
 
 const logger = new Logger('VoiceEngineV2AppRemoteSpeakingAdapter');
-const REMOTE_SPEAKING_ANALYSER_INTERVAL_MS = 50;
 const REMOTE_SPEAKING_ANALYSER_HANDLES_CAP = 256;
 
-export function computeTimeDomainRms(samples: Float32Array): number {
-	if (samples.length === 0) return 0;
-	let sumSquares = 0;
-	for (let i = 0; i < samples.length; i++) {
-		const sample = samples[i]!;
-		sumSquares += sample * sample;
-	}
-	return Math.sqrt(sumSquares / samples.length);
-}
-
 interface AnalyserHandle {
 	identity: string;
 	track: MediaStreamTrack;
@@ -171,17 +165,15 @@ class VoiceEngineV2AppRemoteSpeakingAdapter {
 		}
 		try {
 			const source = ctx.createMediaStreamSource(new MediaStream([mediaStreamTrack]));
-			const analyser = ctx.createAnalyser();
-			analyser.fftSize = 512;
-			analyser.smoothingTimeConstant = 0.2;
+			const analyser = createSpeakingAnalyser(ctx);
 			source.connect(analyser);
 			const handle: AnalyserHandle = {
 				identity,
 				track: mediaStreamTrack,
 				source,
 				analyser,
-				samples: new Float32Array(new ArrayBuffer(analyser.fftSize * Float32Array.BYTES_PER_ELEMENT)),
-				nextTickAtMs: this.nowMs() + REMOTE_SPEAKING_ANALYSER_INTERVAL_MS,
+				samples: createSpeakingSamples(),
+				nextTickAtMs: this.nowMs() + SPEAKING_ANALYSER_INTERVAL_MS,
 			};
 			this.analysers.set(identity, handle);
 			this.applyCommands(this.transition({type: 'remote.attach', identity, track: mediaStreamTrack}));
@@ -301,9 +293,9 @@ class VoiceEngineV2AppRemoteSpeakingAdapter {
 		}
 		handle.analyser.getFloatTimeDomainData(handle.samples);
 		const rms = computeTimeDomainRms(handle.samples);
-		const threshold = getRemoteSpeakingThresholdRms(VoiceSettings.getVadThreshold());
+		const threshold = readSpeakingThresholdRms();
 		this.applyCommands(this.transition({type: 'remote.tick', identity: handle.identity, rms, threshold, nowMs}));
-		handle.nextTickAtMs = nowMs + REMOTE_SPEAKING_ANALYSER_INTERVAL_MS;
+		handle.nextTickAtMs = nowMs + SPEAKING_ANALYSER_INTERVAL_MS;
 	}
 
 	private disposeHandle(handle: AnalyserHandle): void {
@@ -332,7 +324,7 @@ class VoiceEngineV2AppRemoteSpeakingAdapter {
 		if (typeof window === 'undefined') return;
 		this.autoScheduleTimerId = window.setInterval(() => {
 			this.tickHost(this.nowMs());
-		}, REMOTE_SPEAKING_ANALYSER_INTERVAL_MS);
+		}, SPEAKING_ANALYSER_INTERVAL_MS);
 	}
 
 	private stopAutoSchedule(): void {

fluxer_app/src/features/voice/utils/VoiceActivityGate.ts

@@ -17,7 +17,6 @@ export interface VoiceInputLevel {
 	rms: number;
 	floorRms: number;
 	thresholdRms: number;
-	speaking: boolean;
 	nonFinite: boolean;
 	frame: number;
 	contextTime: number;

fluxer_app/src/features/voice/utils/VoiceInputProcessor.ts

@@ -461,8 +461,6 @@ export class VoiceInputTrackProcessor implements TrackProcessor<Track.Kind.Audio
 	readonly name = 'fluxer-voice-input-processor';
 	processedTrack?: MediaStreamTrack;
 	private state: 'idle' | 'live' | 'destroyed' = 'idle';
-	private readonly levelListeners = new Set<(level: VoiceInputLevel) => void>();
-	private latestLevel: VoiceInputLevel | null = null;
 	private hasRunningLevel = false;
 	private lease: VoiceInputContextLease | null = null;
 	private graph: VoiceInputGraph | null = null;
@@ -485,7 +483,7 @@ export class VoiceInputTrackProcessor implements TrackProcessor<Track.Kind.Audio
 			context,
 			this.channelCount,
 			resolveVoiceInputConfig,
-			(level) => this.emitLevel(level, true),
+			() => this.noteLevel(),
 			() => this.events.onGraphFailed(this),
 			() => this.reportRuntime(),
 		);
@@ -528,22 +526,10 @@ export class VoiceInputTrackProcessor implements TrackProcessor<Track.Kind.Audio
 		if (this.state === 'live') this.graph?.reportRuntime(this);
 	}
 
-	onLevel(listener: (level: VoiceInputLevel) => void): () => void {
-		this.levelListeners.add(listener);
-		if (this.latestLevel) listener(this.latestLevel);
-		return () => {
-			this.levelListeners.delete(listener);
-		};
-	}
-
-	private emitLevel(level: VoiceInputLevel, fromWorklet = false): void {
-		if (this.state !== 'live') return;
-		if (fromWorklet && !this.hasRunningLevel && this.lease?.context.state === 'running') {
-			this.hasRunningLevel = true;
-			NoiseSuppressionAvailability.setVoiceInputGraphUnavailable(false);
-		}
-		this.latestLevel = level;
-		for (const listener of this.levelListeners) listener(level);
+	private noteLevel(): void {
+		if (this.state !== 'live' || this.hasRunningLevel || this.lease?.context.state !== 'running') return;
+		this.hasRunningLevel = true;
+		NoiseSuppressionAvailability.setVoiceInputGraphUnavailable(false);
 	}
 
 	private watchContextState(): void {
@@ -553,7 +539,6 @@ export class VoiceInputTrackProcessor implements TrackProcessor<Track.Kind.Audio
 			this.clearStallTimer();
 			return;
 		}
-		if (this.latestLevel) this.emitLevel({...this.latestLevel, rms: 0, speaking: false});
 		this.stallTimer ??= setTimeout(() => {
 			this.stallTimer = null;
 			if (this.state === 'live' && context.state !== 'running') this.events.onGraphStalled(this);
@@ -568,8 +553,6 @@ export class VoiceInputTrackProcessor implements TrackProcessor<Track.Kind.Audio
 
 	async destroy(): Promise<void> {
 		if (this.state === 'destroyed') return;
-		if (this.latestLevel) this.emitLevel({...this.latestLevel, rms: 0, speaking: false});
-		this.levelListeners.clear();
 		this.state = 'destroyed';
 		NoiseSuppressionAvailability.clearVoiceInputRuntime(this);
 		this.clearStallTimer();

fluxer_app/src/features/voice/utils/voiceGateProcessor.worklet.js

@@ -104,7 +104,6 @@ class VoiceGateProcessor extends AudioWorkletProcessor {
 		const thresholdRms = this.auto
 			? Math.min(this.maxRms, Math.max(this.minRms, floorRms * 3))
 			: this.manualThresholdRms;
-		const wasSpeaking = this.speaking;
 		if (this.auto && this.frame < this.seedUntilFrame) {
 			this.speaking = false;
 			this.silenceFrames = 0;
@@ -129,13 +128,12 @@ class VoiceGateProcessor extends AudioWorkletProcessor {
 			if (this.delayFrames > 0) this.delayIndex = (this.delayIndex + 1) % this.delayFrames;
 		}
 		this.frame += length;
-		if (wasSpeaking !== this.speaking || this.frame >= this.reportFrame) {
+		if (this.frame >= this.reportFrame) {
 			this.port.postMessage({
 				type: 'level',
 				rms,
 				floorRms,
 				thresholdRms,
-				speaking: this.speaking,
 				nonFinite: this.nonFinite,
 				frame: this.frame,
 				contextTime: currentTime,