fix(voice): retune the remote playback leveller and measure it in float (#1784)

fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.test.ts

@@ -8,6 +8,7 @@ import {
 import type {Participant, Room} from 'livekit-client';
 import {describe, expect, it} from 'vitest';
 import {
+	__TEST__,
 	createLivekitParticipantSnapshot,
 	createLivekitParticipantSnapshotsFromRoom,
 	createVoiceParticipantSnapshot,
@@ -59,6 +60,48 @@ function commandsOf(snapshot: VoiceRemoteSpeakingSnapshot): ReadonlyArray<VoiceR
 	return snapshot.context.commands;
 }
 
+const LEVELLER_IDENTITY = 'user_1_a';
+const LEVELLER_TICK_MS = 50;
+
+function attachedRemoteSnapshot(): VoiceRemoteSpeakingSnapshot {
+	const attached = transitionVoiceRemoteSpeakingSnapshot(createVoiceRemoteSpeakingSnapshot(), {
+		type: 'remote.attach',
+		identity: LEVELLER_IDENTITY,
+		track: {},
+	});
+	return transitionVoiceRemoteSpeakingSnapshot(attached, {type: 'remote.clearCommands'});
+}
+
+function runRemoteTicks(
+	snapshot: VoiceRemoteSpeakingSnapshot,
+	options: {rms: number; threshold: number; ticks: number; startMs: number},
+): {snapshot: VoiceRemoteSpeakingSnapshot; commands: Array<VoiceRemoteSpeakingCommand>; endMs: number} {
+	let next = snapshot;
+	let nowMs = options.startMs;
+	const commands: Array<VoiceRemoteSpeakingCommand> = [];
+	for (let i = 0; i < options.ticks; i++) {
+		next = transitionVoiceRemoteSpeakingSnapshot(next, {
+			type: 'remote.tick',
+			identity: LEVELLER_IDENTITY,
+			rms: options.rms,
+			threshold: options.threshold,
+			nowMs,
+		});
+		commands.push(...commandsOf(next));
+		next = transitionVoiceRemoteSpeakingSnapshot(next, {type: 'remote.clearCommands'});
+		nowMs += LEVELLER_TICK_MS;
+	}
+	return {snapshot: next, commands, endMs: nowMs};
+}
+
+function playbackBoostOf(snapshot: VoiceRemoteSpeakingSnapshot): number {
+	return snapshot.context.analysers.get(LEVELLER_IDENTITY)?.playbackBoost ?? Number.NaN;
+}
+
+function appliedBoostOf(snapshot: VoiceRemoteSpeakingSnapshot): number {
+	return snapshot.context.analysers.get(LEVELLER_IDENTITY)?.appliedBoost ?? Number.NaN;
+}
+
 describe('VoiceParticipantStateMachine participants', () => {
 	it('keeps participant snapshot references stable when an upsert is equal', () => {
 		const p = participant('user_42_conn-a', {
@@ -304,45 +347,88 @@ describe('VoiceParticipantStateMachine remote speaking', () => {
 		expect(commandsOf(snapshot)).toEqual([{type: 'rehydrateRemoteAnalysers'}]);
 	});
 
-	it('resets playback boost after quiet audio and on detach', () => {
+	it('holds the playback boost through silence and releases it back to unity', () => {
-		let snapshot = createVoiceRemoteSpeakingSnapshot();
+		let snapshot = transitionVoiceRemoteSpeakingSnapshot(createVoiceRemoteSpeakingSnapshot(), {
-		snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {
 			type: 'remote.attach',
-			identity: 'user_1_a',
+			identity: LEVELLER_IDENTITY,
 			track: {},
 		});
-		expect(commandsOf(snapshot)).toEqual([{type: 'setPlaybackBoost', identity: 'user_1_a', boost: 1}]);
+		expect(commandsOf(snapshot)).toEqual([{type: 'setPlaybackBoost', identity: LEVELLER_IDENTITY, boost: 1}]);
+		expect(playbackBoostOf(snapshot)).toBe(1);
+		expect(appliedBoostOf(snapshot)).toBe(1);
 		snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {type: 'remote.clearCommands'});
 
-		snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {
+		const settled = runRemoteTicks(snapshot, {rms: 0.03, threshold: 0.006, ticks: 60, startMs: 0});
-			type: 'remote.tick',
+		expect(playbackBoostOf(settled.snapshot)).toBeGreaterThan(2.5);
-			identity: 'user_1_a',
+		expect(settled.commands.some((command) => command.type === 'setPlaybackBoost')).toBe(true);
-			rms: 0.006,
+
-			threshold: 1,
+		const settledBoost = playbackBoostOf(settled.snapshot);
-			nowMs: 10,
+		const held = runRemoteTicks(settled.snapshot, {rms: 0, threshold: 0.006, ticks: 60, startMs: settled.endMs});
+		expect(held.endMs - settled.endMs).toBe(__TEST__.REMOTE_PLAYBACK_SILENCE_HOLD_MS);
+		expect(playbackBoostOf(held.snapshot)).toBe(settledBoost);
+		expect(held.commands.some((command) => command.type === 'clearPlaybackBoost')).toBe(false);
+
+		const released = runRemoteTicks(held.snapshot, {rms: 0, threshold: 0.006, ticks: 700, startMs: held.endMs});
+		expect(released.commands.filter((command) => command.type === 'clearPlaybackBoost')).toHaveLength(1);
+		expect(playbackBoostOf(released.snapshot)).toBe(1);
+		expect(appliedBoostOf(released.snapshot)).toBe(1);
+
+		snapshot = transitionVoiceRemoteSpeakingSnapshot(released.snapshot, {
+			type: 'remote.detach',
+			identity: LEVELLER_IDENTITY,
 		});
-		expect(commandsOf(snapshot)[0]).toMatchObject({type: 'setPlaybackBoost', identity: 'user_1_a'});
-		expect(snapshot.context.analysers.get('user_1_a')?.playbackBoost).toBeGreaterThan(1);
-		snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {type: 'remote.clearCommands'});
-
-		snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {
-			type: 'remote.tick',
-			identity: 'user_1_a',
-			rms: 0,
-			threshold: 1,
-			nowMs: 20,
-		});
-		expect(commandsOf(snapshot)).toEqual([{type: 'clearPlaybackBoost', identity: 'user_1_a'}]);
-		expect(snapshot.context.analysers.get('user_1_a')?.playbackBoost).toBe(1);
-
-		snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {type: 'remote.clearCommands'});
-		snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {type: 'remote.detach', identity: 'user_1_a'});
 		expect(commandsOf(snapshot)).toEqual([
-			{type: 'setAudioLevelSpeaking', identity: 'user_1_a', speaking: false},
+			{type: 'setAudioLevelSpeaking', identity: LEVELLER_IDENTITY, speaking: false},
-			{type: 'clearPlaybackBoost', identity: 'user_1_a'},
+			{type: 'clearPlaybackBoost', identity: LEVELLER_IDENTITY},
 		]);
 	});
 
+	it('holds the boost through an inter-syllabic pause instead of pumping', () => {
+		const settled = runRemoteTicks(attachedRemoteSnapshot(), {rms: 0.02, threshold: 0.006, ticks: 120, startMs: 0});
+		expect(playbackBoostOf(settled.snapshot)).toBeCloseTo(__TEST__.REMOTE_PLAYBACK_MAX_BOOST, 2);
+
+		const settledBoost = playbackBoostOf(settled.snapshot);
+		const paused = runRemoteTicks(settled.snapshot, {rms: 0.002, threshold: 0.006, ticks: 4, startMs: settled.endMs});
+		expect(Math.abs(playbackBoostOf(paused.snapshot) - settledBoost)).toBeLessThan(settledBoost * 0.01);
+		expect(paused.commands).toHaveLength(0);
+	});
+
+	it('integrates steps the emit guard suppresses', () => {
+		const settled = runRemoteTicks(attachedRemoteSnapshot(), {rms: 0.086, threshold: 0.006, ticks: 200, startMs: 0});
+		expect(playbackBoostOf(settled.snapshot)).toBeCloseTo(1.0465, 4);
+		expect(appliedBoostOf(settled.snapshot)).toBeLessThan(playbackBoostOf(settled.snapshot));
+	});
+
+	it('targets roughly -21 dBFS of playback level', () => {
+		const settled = runRemoteTicks(attachedRemoteSnapshot(), {rms: 0.045, threshold: 0.006, ticks: 200, startMs: 0});
+		expect(playbackBoostOf(settled.snapshot)).toBeCloseTo(2, 4);
+		expect(playbackBoostOf(settled.snapshot) * 0.045).toBeCloseTo(__TEST__.REMOTE_PLAYBACK_TARGET_RMS, 4);
+	});
+
+	it('never integrates on a zero threshold', () => {
+		const settled = runRemoteTicks(attachedRemoteSnapshot(), {rms: 0, threshold: 0, ticks: 10, startMs: 0});
+		expect(playbackBoostOf(settled.snapshot)).toBe(1);
+		expect(settled.commands.some((command) => command.type === 'setPlaybackBoost')).toBe(false);
+	});
+
+	it('clears a stale boost when reattaching a different track', () => {
+		const settled = runRemoteTicks(attachedRemoteSnapshot(), {rms: 0.02, threshold: 0.006, ticks: 60, startMs: 0});
+		expect(appliedBoostOf(settled.snapshot)).toBeGreaterThan(1);
+
+		const reattached = transitionVoiceRemoteSpeakingSnapshot(settled.snapshot, {
+			type: 'remote.attach',
+			identity: LEVELLER_IDENTITY,
+			track: {},
+		});
+		expect(commandsOf(reattached)).toEqual([
+			{type: 'setAudioLevelSpeaking', identity: LEVELLER_IDENTITY, speaking: false},
+			{type: 'clearPlaybackBoost', identity: LEVELLER_IDENTITY},
+			{type: 'setPlaybackBoost', identity: LEVELLER_IDENTITY, boost: 1},
+		]);
+		expect(playbackBoostOf(reattached)).toBe(1);
+		expect(appliedBoostOf(reattached)).toBe(1);
+	});
+
 	it('clears stale speaking flags after detach and clear', () => {
 		let snapshot = createVoiceRemoteSpeakingSnapshot();
 		snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {

fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.ts

@@ -51,6 +51,7 @@ export interface RemoteSpeakingAnalyserState {
 	belowSinceMs: number | null;
 	aboveSinceMs: number | null;
 	playbackBoost: number;
+	appliedBoost: number;
 }
 
 export type VoiceRemoteSpeakingCommand =
@@ -76,10 +77,15 @@ export type VoiceRemoteSpeakingEvent =
 
 const EMPTY_PARTICIPANTS: Readonly<Record<string, LivekitParticipantSnapshot>> = {};
 const EMPTY_REMOTE_COMMANDS: ReadonlyArray<VoiceRemoteSpeakingCommand> = [];
-const REMOTE_PLAYBACK_TARGET_RMS = 0.025;
+const REMOTE_PLAYBACK_TARGET_RMS = 0.09;
-const REMOTE_PLAYBACK_MIN_RMS = 0.004;
+const REMOTE_PLAYBACK_MIN_RMS = 0.0025;
 const REMOTE_PLAYBACK_MAX_BOOST = 3;
-const REMOTE_PLAYBACK_MIN_CHANGE = 0.05;
+const REMOTE_PLAYBACK_MIN_CHANGE_RATIO = 1.03;
+const REMOTE_PLAYBACK_BOOST_UP_COEFF = 0.08;
+const REMOTE_PLAYBACK_BOOST_DOWN_COEFF = 0.35;
+const REMOTE_PLAYBACK_SILENCE_HOLD_MS = 3000;
+const REMOTE_PLAYBACK_SILENCE_RELEASE = 0.99;
+const REMOTE_PLAYBACK_UNITY_EPSILON = 0.01;
 
 export const extractParticipantUserId = (identity: string): string | null => {
 	const match = identity.match(/^user_(\d+)(?:_(.+))?$/);
@@ -308,38 +314,62 @@ function attachRemoteAnalyser(
 		belowSinceMs: null,
 		aboveSinceMs: null,
 		playbackBoost: 1,
+		appliedBoost: 1,
 	});
 	const commands: Array<VoiceRemoteSpeakingCommand> = [];
 	if (existing?.speaking) commands.push({type: 'setAudioLevelSpeaking', identity, speaking: false});
-	if (existing && existing.playbackBoost !== 1) commands.push({type: 'clearPlaybackBoost', identity});
+	if (existing && existing.appliedBoost !== 1) commands.push({type: 'clearPlaybackBoost', identity});
 	commands.push({type: 'setPlaybackBoost', identity, boost: 1});
 	return appendRemoteCommands({...context, analysers}, commands);
 }
 
+function settleRemotePlaybackBoost(
+	entry: RemoteSpeakingAnalyserState,
+	nextBoost: number,
+): {entry: RemoteSpeakingAnalyserState; command: VoiceRemoteSpeakingCommand | null} {
+	const clampedBoost = Math.max(1, Math.min(REMOTE_PLAYBACK_MAX_BOOST, nextBoost));
+	const ratio =
+		clampedBoost >= entry.appliedBoost ? clampedBoost / entry.appliedBoost : entry.appliedBoost / clampedBoost;
+	if (ratio < REMOTE_PLAYBACK_MIN_CHANGE_RATIO) {
+		if (clampedBoost === entry.playbackBoost) return {entry, command: null};
+		return {entry: {...entry, playbackBoost: clampedBoost}, command: null};
+	}
+	return {
+		entry: {...entry, playbackBoost: clampedBoost, appliedBoost: clampedBoost},
+		command: {type: 'setPlaybackBoost', identity: entry.identity, boost: clampedBoost},
+	};
+}
+
+function snapRemotePlaybackBoost(entry: RemoteSpeakingAnalyserState): {
+	entry: RemoteSpeakingAnalyserState;
+	command: VoiceRemoteSpeakingCommand;
+} {
+	return {
+		entry: {...entry, playbackBoost: 1, appliedBoost: 1},
+		command: {type: 'clearPlaybackBoost', identity: entry.identity},
+	};
+}
+
 function updateRemotePlaybackBoost(
 	entry: RemoteSpeakingAnalyserState,
 	rms: number,
+	threshold: number,
+	nowMs: number,
 ): {entry: RemoteSpeakingAnalyserState; command: VoiceRemoteSpeakingCommand | null} {
-	if (!Number.isFinite(rms) || rms <= REMOTE_PLAYBACK_MIN_RMS) {
+	const isActiveSpeech = Number.isFinite(rms) && rms >= REMOTE_PLAYBACK_MIN_RMS && rms >= threshold && entry.speaking;
-		if (entry.playbackBoost === 1) return {entry, command: null};
+	if (isActiveSpeech) {
-		return {
+		const desiredBoost = Math.max(1, Math.min(REMOTE_PLAYBACK_MAX_BOOST, REMOTE_PLAYBACK_TARGET_RMS / rms));
-			entry: {...entry, playbackBoost: 1},
+		const coefficient =
-			command: {type: 'clearPlaybackBoost', identity: entry.identity},
+			desiredBoost > entry.playbackBoost ? REMOTE_PLAYBACK_BOOST_UP_COEFF : REMOTE_PLAYBACK_BOOST_DOWN_COEFF;
-		};
+		return settleRemotePlaybackBoost(entry, entry.playbackBoost + (desiredBoost - entry.playbackBoost) * coefficient);
 	}
-	const desiredBoost = Math.max(1, Math.min(REMOTE_PLAYBACK_MAX_BOOST, REMOTE_PLAYBACK_TARGET_RMS / rms));
+	if (entry.playbackBoost <= 1) return {entry, command: null};
-	const nextBoost =
+	if (entry.belowSinceMs === null || nowMs - entry.belowSinceMs < REMOTE_PLAYBACK_SILENCE_HOLD_MS) {
-		desiredBoost > entry.playbackBoost
-			? entry.playbackBoost + (desiredBoost - entry.playbackBoost) * 0.4
-			: entry.playbackBoost + (desiredBoost - entry.playbackBoost) * 0.15;
-	const clampedBoost = Math.max(1, Math.min(REMOTE_PLAYBACK_MAX_BOOST, nextBoost));
-	if (Math.abs(clampedBoost - entry.playbackBoost) < REMOTE_PLAYBACK_MIN_CHANGE) {
 		return {entry, command: null};
 	}
-	return {
+	const releasedBoost = 1 + (entry.playbackBoost - 1) * REMOTE_PLAYBACK_SILENCE_RELEASE;
-		entry: {...entry, playbackBoost: clampedBoost},
+	if (releasedBoost - 1 <= REMOTE_PLAYBACK_UNITY_EPSILON) return snapRemotePlaybackBoost(entry);
-		command: {type: 'setPlaybackBoost', identity: entry.identity, boost: clampedBoost},
+	return settleRemotePlaybackBoost(entry, releasedBoost);
-	};
 }
 
 function tickRemoteAnalyser(
@@ -351,8 +381,7 @@ function tickRemoteAnalyser(
 	if (event.trackEnded) return detachRemoteAnalyser(context, event.identity);
 
 	const commands: Array<VoiceRemoteSpeakingCommand> = [];
-	let {entry, command} = updateRemotePlaybackBoost(existing, event.rms);
+	let entry = existing;
-	if (command) commands.push(command);
 	if (event.rms >= event.threshold) {
 		const aboveSinceMs = entry.aboveSinceMs ?? event.nowMs;
 		entry = {
@@ -376,6 +405,9 @@ function tickRemoteAnalyser(
 			commands.push({type: 'setAudioLevelSpeaking', identity: event.identity, speaking: false});
 		}
 	}
+	const boostUpdate = updateRemotePlaybackBoost(entry, event.rms, event.threshold, event.nowMs);
+	entry = boostUpdate.entry;
+	if (boostUpdate.command) commands.push(boostUpdate.command);
 	if (entry === existing && commands.length === 0) return context;
 	const analysers = new Map(context.analysers);
 	analysers.set(event.identity, entry);
@@ -509,6 +541,13 @@ export function clearVoiceRemoteSpeakingCommands(snapshot: VoiceRemoteSpeakingSn
 	return transitionVoiceRemoteSpeakingSnapshot(snapshot, {type: 'remote.clearCommands'});
 }
 
+export const __TEST__ = {
+	REMOTE_PLAYBACK_TARGET_RMS,
+	REMOTE_PLAYBACK_MIN_RMS,
+	REMOTE_PLAYBACK_MAX_BOOST,
+	REMOTE_PLAYBACK_SILENCE_HOLD_MS,
+};
+
 export function findParticipantSnapshotByUserIdAndConnectionId(
 	participants: Readonly<Record<string, LivekitParticipantSnapshot>>,
 	userId: string,

fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.test.ts

@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {describe, expect, it, vi} from 'vitest';
+import {computeTimeDomainRms} from './VoiceEngineV2AppRemoteSpeakingAdapter';
+
+vi.mock('@app/features/voice/state/ParticipantVolume', () => ({default: {}}));
+vi.mock('@app/features/voice/state/VoiceSettings', () => ({default: {getVadThreshold: () => 50}}));
+
+describe('computeTimeDomainRms', () => {
+	it('returns the amplitude of an alternating full-scale-relative signal', () => {
+		const samples = new Float32Array(512);
+		for (let i = 0; i < samples.length; i++) {
+			samples[i] = i % 2 === 0 ? 0.09 : -0.09;
+		}
+		expect(computeTimeDomainRms(samples)).toBe(Math.fround(0.09));
+	});
+
+	it('returns zero for digital silence', () => {
+		expect(computeTimeDomainRms(new Float32Array(512))).toBe(0);
+	});
+
+	it('returns zero for an empty buffer', () => {
+		expect(computeTimeDomainRms(new Float32Array(0))).toBe(0);
+	});
+
+	it('spreads a single full-scale sample across the window', () => {
+		const samples = new Float32Array(512);
+		samples[17] = 1;
+		expect(computeTimeDomainRms(samples)).toBeCloseTo(1 / Math.sqrt(512), 12);
+	});
+
+	it('resolves levels far below one 8-bit quantisation step', () => {
+		expect(computeTimeDomainRms(Float32Array.of(0.0005, -0.0005))).toBe(Math.fround(0.0005));
+	});
+});

fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.ts

@@ -30,12 +30,22 @@ const logger = new Logger('VoiceEngineV2AppRemoteSpeakingAdapter');
 export const REMOTE_SPEAKING_ANALYSER_INTERVAL_MS = 50;
 export const REMOTE_SPEAKING_ANALYSER_HANDLES_CAP = 256;
 
+export function computeTimeDomainRms(samples: Float32Array): number {
+	if (samples.length === 0) return 0;
+	let sumSquares = 0;
+	for (let i = 0; i < samples.length; i++) {
+		const sample = samples[i]!;
+		sumSquares += sample * sample;
+	}
+	return Math.sqrt(sumSquares / samples.length);
+}
+
 interface AnalyserHandle {
 	identity: string;
 	track: MediaStreamTrack;
 	source: MediaStreamAudioSourceNode;
 	analyser: AnalyserNode;
-	samples: Uint8Array<ArrayBuffer>;
+	samples: Float32Array<ArrayBuffer>;
 	nextTickAtMs: number;
 }
 
@@ -73,9 +83,7 @@ export class VoiceEngineV2AppRemoteSpeakingAdapter {
 					break;
 				case 'setPlaybackBoost':
 					setRemoteVoicePlaybackBoost(command.identity, command.boost);
-					if (command.boost !== 1) {
+					this.reapplyParticipantVolume(command.identity);
-						this.reapplyParticipantVolume(command.identity);
-					}
 					break;
 				case 'clearPlaybackBoost':
 					clearRemoteVoicePlaybackBoost(command.identity);
@@ -193,7 +201,7 @@ export class VoiceEngineV2AppRemoteSpeakingAdapter {
 				track: mediaStreamTrack,
 				source,
 				analyser,
-				samples: new Uint8Array(new ArrayBuffer(analyser.fftSize)),
+				samples: new Float32Array(new ArrayBuffer(analyser.fftSize * Float32Array.BYTES_PER_ELEMENT)),
 				nextTickAtMs: this.nowMs() + REMOTE_SPEAKING_ANALYSER_INTERVAL_MS,
 			};
 			this.analysers.set(identity, handle);
@@ -315,13 +323,8 @@ export class VoiceEngineV2AppRemoteSpeakingAdapter {
 			}
 			return;
 		}
-		handle.analyser.getByteTimeDomainData(handle.samples);
+		handle.analyser.getFloatTimeDomainData(handle.samples);
-		let sumSquares = 0;
+		const rms = computeTimeDomainRms(handle.samples);
-		for (let i = 0; i < handle.samples.length; i++) {
-			const normalized = (handle.samples[i]! - 128) / 128;
-			sumSquares += normalized * normalized;
-		}
-		const rms = Math.sqrt(sumSquares / handle.samples.length);
 		const threshold = getRemoteSpeakingThresholdRms(VoiceSettings.getVadThreshold());
 		this.applyCommands(this.transition({type: 'remote.tick', identity: handle.identity, rms, threshold, nowMs}));
 		handle.nextTickAtMs = nowMs + REMOTE_SPEAKING_ANALYSER_INTERVAL_MS;