diff --git a/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.test.ts b/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.test.ts index dc1e696a3..1b87131b5 100644 --- a/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.test.ts +++ b/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.test.ts @@ -8,6 +8,7 @@ import { import type {Participant, Room} from 'livekit-client'; import {describe, expect, it} from 'vitest'; import { + __TEST__, createLivekitParticipantSnapshot, createLivekitParticipantSnapshotsFromRoom, createVoiceParticipantSnapshot, @@ -59,6 +60,48 @@ function commandsOf(snapshot: VoiceRemoteSpeakingSnapshot): ReadonlyArray; endMs: number} { + let next = snapshot; + let nowMs = options.startMs; + const commands: Array = []; + for (let i = 0; i < options.ticks; i++) { + next = transitionVoiceRemoteSpeakingSnapshot(next, { + type: 'remote.tick', + identity: LEVELLER_IDENTITY, + rms: options.rms, + threshold: options.threshold, + nowMs, + }); + commands.push(...commandsOf(next)); + next = transitionVoiceRemoteSpeakingSnapshot(next, {type: 'remote.clearCommands'}); + nowMs += LEVELLER_TICK_MS; + } + return {snapshot: next, commands, endMs: nowMs}; +} + +function playbackBoostOf(snapshot: VoiceRemoteSpeakingSnapshot): number { + return snapshot.context.analysers.get(LEVELLER_IDENTITY)?.playbackBoost ?? Number.NaN; +} + +function appliedBoostOf(snapshot: VoiceRemoteSpeakingSnapshot): number { + return snapshot.context.analysers.get(LEVELLER_IDENTITY)?.appliedBoost ?? Number.NaN; +} + describe('VoiceParticipantStateMachine participants', () => { it('keeps participant snapshot references stable when an upsert is equal', () => { const p = participant('user_42_conn-a', { @@ -304,45 +347,88 @@ describe('VoiceParticipantStateMachine remote speaking', () => { expect(commandsOf(snapshot)).toEqual([{type: 'rehydrateRemoteAnalysers'}]); }); - it('resets playback boost after quiet audio and on detach', () => { - let snapshot = createVoiceRemoteSpeakingSnapshot(); - snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, { + it('holds the playback boost through silence and releases it back to unity', () => { + let snapshot = transitionVoiceRemoteSpeakingSnapshot(createVoiceRemoteSpeakingSnapshot(), { type: 'remote.attach', - identity: 'user_1_a', + identity: LEVELLER_IDENTITY, track: {}, }); - expect(commandsOf(snapshot)).toEqual([{type: 'setPlaybackBoost', identity: 'user_1_a', boost: 1}]); + expect(commandsOf(snapshot)).toEqual([{type: 'setPlaybackBoost', identity: LEVELLER_IDENTITY, boost: 1}]); + expect(playbackBoostOf(snapshot)).toBe(1); + expect(appliedBoostOf(snapshot)).toBe(1); snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {type: 'remote.clearCommands'}); - snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, { - type: 'remote.tick', - identity: 'user_1_a', - rms: 0.006, - threshold: 1, - nowMs: 10, + const settled = runRemoteTicks(snapshot, {rms: 0.03, threshold: 0.006, ticks: 60, startMs: 0}); + expect(playbackBoostOf(settled.snapshot)).toBeGreaterThan(2.5); + expect(settled.commands.some((command) => command.type === 'setPlaybackBoost')).toBe(true); + + const settledBoost = playbackBoostOf(settled.snapshot); + const held = runRemoteTicks(settled.snapshot, {rms: 0, threshold: 0.006, ticks: 60, startMs: settled.endMs}); + expect(held.endMs - settled.endMs).toBe(__TEST__.REMOTE_PLAYBACK_SILENCE_HOLD_MS); + expect(playbackBoostOf(held.snapshot)).toBe(settledBoost); + expect(held.commands.some((command) => command.type === 'clearPlaybackBoost')).toBe(false); + + const released = runRemoteTicks(held.snapshot, {rms: 0, threshold: 0.006, ticks: 700, startMs: held.endMs}); + expect(released.commands.filter((command) => command.type === 'clearPlaybackBoost')).toHaveLength(1); + expect(playbackBoostOf(released.snapshot)).toBe(1); + expect(appliedBoostOf(released.snapshot)).toBe(1); + + snapshot = transitionVoiceRemoteSpeakingSnapshot(released.snapshot, { + type: 'remote.detach', + identity: LEVELLER_IDENTITY, }); - expect(commandsOf(snapshot)[0]).toMatchObject({type: 'setPlaybackBoost', identity: 'user_1_a'}); - expect(snapshot.context.analysers.get('user_1_a')?.playbackBoost).toBeGreaterThan(1); - snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {type: 'remote.clearCommands'}); - - snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, { - type: 'remote.tick', - identity: 'user_1_a', - rms: 0, - threshold: 1, - nowMs: 20, - }); - expect(commandsOf(snapshot)).toEqual([{type: 'clearPlaybackBoost', identity: 'user_1_a'}]); - expect(snapshot.context.analysers.get('user_1_a')?.playbackBoost).toBe(1); - - snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {type: 'remote.clearCommands'}); - snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, {type: 'remote.detach', identity: 'user_1_a'}); expect(commandsOf(snapshot)).toEqual([ - {type: 'setAudioLevelSpeaking', identity: 'user_1_a', speaking: false}, - {type: 'clearPlaybackBoost', identity: 'user_1_a'}, + {type: 'setAudioLevelSpeaking', identity: LEVELLER_IDENTITY, speaking: false}, + {type: 'clearPlaybackBoost', identity: LEVELLER_IDENTITY}, ]); }); + it('holds the boost through an inter-syllabic pause instead of pumping', () => { + const settled = runRemoteTicks(attachedRemoteSnapshot(), {rms: 0.02, threshold: 0.006, ticks: 120, startMs: 0}); + expect(playbackBoostOf(settled.snapshot)).toBeCloseTo(__TEST__.REMOTE_PLAYBACK_MAX_BOOST, 2); + + const settledBoost = playbackBoostOf(settled.snapshot); + const paused = runRemoteTicks(settled.snapshot, {rms: 0.002, threshold: 0.006, ticks: 4, startMs: settled.endMs}); + expect(Math.abs(playbackBoostOf(paused.snapshot) - settledBoost)).toBeLessThan(settledBoost * 0.01); + expect(paused.commands).toHaveLength(0); + }); + + it('integrates steps the emit guard suppresses', () => { + const settled = runRemoteTicks(attachedRemoteSnapshot(), {rms: 0.086, threshold: 0.006, ticks: 200, startMs: 0}); + expect(playbackBoostOf(settled.snapshot)).toBeCloseTo(1.0465, 4); + expect(appliedBoostOf(settled.snapshot)).toBeLessThan(playbackBoostOf(settled.snapshot)); + }); + + it('targets roughly -21 dBFS of playback level', () => { + const settled = runRemoteTicks(attachedRemoteSnapshot(), {rms: 0.045, threshold: 0.006, ticks: 200, startMs: 0}); + expect(playbackBoostOf(settled.snapshot)).toBeCloseTo(2, 4); + expect(playbackBoostOf(settled.snapshot) * 0.045).toBeCloseTo(__TEST__.REMOTE_PLAYBACK_TARGET_RMS, 4); + }); + + it('never integrates on a zero threshold', () => { + const settled = runRemoteTicks(attachedRemoteSnapshot(), {rms: 0, threshold: 0, ticks: 10, startMs: 0}); + expect(playbackBoostOf(settled.snapshot)).toBe(1); + expect(settled.commands.some((command) => command.type === 'setPlaybackBoost')).toBe(false); + }); + + it('clears a stale boost when reattaching a different track', () => { + const settled = runRemoteTicks(attachedRemoteSnapshot(), {rms: 0.02, threshold: 0.006, ticks: 60, startMs: 0}); + expect(appliedBoostOf(settled.snapshot)).toBeGreaterThan(1); + + const reattached = transitionVoiceRemoteSpeakingSnapshot(settled.snapshot, { + type: 'remote.attach', + identity: LEVELLER_IDENTITY, + track: {}, + }); + expect(commandsOf(reattached)).toEqual([ + {type: 'setAudioLevelSpeaking', identity: LEVELLER_IDENTITY, speaking: false}, + {type: 'clearPlaybackBoost', identity: LEVELLER_IDENTITY}, + {type: 'setPlaybackBoost', identity: LEVELLER_IDENTITY, boost: 1}, + ]); + expect(playbackBoostOf(reattached)).toBe(1); + expect(appliedBoostOf(reattached)).toBe(1); + }); + it('clears stale speaking flags after detach and clear', () => { let snapshot = createVoiceRemoteSpeakingSnapshot(); snapshot = transitionVoiceRemoteSpeakingSnapshot(snapshot, { diff --git a/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.ts b/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.ts index 09af54d7d..4164d921b 100644 --- a/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.ts +++ b/fluxer_app/src/features/voice/engine/VoiceParticipantStateMachine.ts @@ -51,6 +51,7 @@ export interface RemoteSpeakingAnalyserState { belowSinceMs: number | null; aboveSinceMs: number | null; playbackBoost: number; + appliedBoost: number; } export type VoiceRemoteSpeakingCommand = @@ -76,10 +77,15 @@ export type VoiceRemoteSpeakingEvent = const EMPTY_PARTICIPANTS: Readonly> = {}; const EMPTY_REMOTE_COMMANDS: ReadonlyArray = []; -const REMOTE_PLAYBACK_TARGET_RMS = 0.025; -const REMOTE_PLAYBACK_MIN_RMS = 0.004; +const REMOTE_PLAYBACK_TARGET_RMS = 0.09; +const REMOTE_PLAYBACK_MIN_RMS = 0.0025; const REMOTE_PLAYBACK_MAX_BOOST = 3; -const REMOTE_PLAYBACK_MIN_CHANGE = 0.05; +const REMOTE_PLAYBACK_MIN_CHANGE_RATIO = 1.03; +const REMOTE_PLAYBACK_BOOST_UP_COEFF = 0.08; +const REMOTE_PLAYBACK_BOOST_DOWN_COEFF = 0.35; +const REMOTE_PLAYBACK_SILENCE_HOLD_MS = 3000; +const REMOTE_PLAYBACK_SILENCE_RELEASE = 0.99; +const REMOTE_PLAYBACK_UNITY_EPSILON = 0.01; export const extractParticipantUserId = (identity: string): string | null => { const match = identity.match(/^user_(\d+)(?:_(.+))?$/); @@ -308,38 +314,62 @@ function attachRemoteAnalyser( belowSinceMs: null, aboveSinceMs: null, playbackBoost: 1, + appliedBoost: 1, }); const commands: Array = []; if (existing?.speaking) commands.push({type: 'setAudioLevelSpeaking', identity, speaking: false}); - if (existing && existing.playbackBoost !== 1) commands.push({type: 'clearPlaybackBoost', identity}); + if (existing && existing.appliedBoost !== 1) commands.push({type: 'clearPlaybackBoost', identity}); commands.push({type: 'setPlaybackBoost', identity, boost: 1}); return appendRemoteCommands({...context, analysers}, commands); } +function settleRemotePlaybackBoost( + entry: RemoteSpeakingAnalyserState, + nextBoost: number, +): {entry: RemoteSpeakingAnalyserState; command: VoiceRemoteSpeakingCommand | null} { + const clampedBoost = Math.max(1, Math.min(REMOTE_PLAYBACK_MAX_BOOST, nextBoost)); + const ratio = + clampedBoost >= entry.appliedBoost ? clampedBoost / entry.appliedBoost : entry.appliedBoost / clampedBoost; + if (ratio < REMOTE_PLAYBACK_MIN_CHANGE_RATIO) { + if (clampedBoost === entry.playbackBoost) return {entry, command: null}; + return {entry: {...entry, playbackBoost: clampedBoost}, command: null}; + } + return { + entry: {...entry, playbackBoost: clampedBoost, appliedBoost: clampedBoost}, + command: {type: 'setPlaybackBoost', identity: entry.identity, boost: clampedBoost}, + }; +} + +function snapRemotePlaybackBoost(entry: RemoteSpeakingAnalyserState): { + entry: RemoteSpeakingAnalyserState; + command: VoiceRemoteSpeakingCommand; +} { + return { + entry: {...entry, playbackBoost: 1, appliedBoost: 1}, + command: {type: 'clearPlaybackBoost', identity: entry.identity}, + }; +} + function updateRemotePlaybackBoost( entry: RemoteSpeakingAnalyserState, rms: number, + threshold: number, + nowMs: number, ): {entry: RemoteSpeakingAnalyserState; command: VoiceRemoteSpeakingCommand | null} { - if (!Number.isFinite(rms) || rms <= REMOTE_PLAYBACK_MIN_RMS) { - if (entry.playbackBoost === 1) return {entry, command: null}; - return { - entry: {...entry, playbackBoost: 1}, - command: {type: 'clearPlaybackBoost', identity: entry.identity}, - }; + const isActiveSpeech = Number.isFinite(rms) && rms >= REMOTE_PLAYBACK_MIN_RMS && rms >= threshold && entry.speaking; + if (isActiveSpeech) { + const desiredBoost = Math.max(1, Math.min(REMOTE_PLAYBACK_MAX_BOOST, REMOTE_PLAYBACK_TARGET_RMS / rms)); + const coefficient = + desiredBoost > entry.playbackBoost ? REMOTE_PLAYBACK_BOOST_UP_COEFF : REMOTE_PLAYBACK_BOOST_DOWN_COEFF; + return settleRemotePlaybackBoost(entry, entry.playbackBoost + (desiredBoost - entry.playbackBoost) * coefficient); } - const desiredBoost = Math.max(1, Math.min(REMOTE_PLAYBACK_MAX_BOOST, REMOTE_PLAYBACK_TARGET_RMS / rms)); - const nextBoost = - desiredBoost > entry.playbackBoost - ? entry.playbackBoost + (desiredBoost - entry.playbackBoost) * 0.4 - : entry.playbackBoost + (desiredBoost - entry.playbackBoost) * 0.15; - const clampedBoost = Math.max(1, Math.min(REMOTE_PLAYBACK_MAX_BOOST, nextBoost)); - if (Math.abs(clampedBoost - entry.playbackBoost) < REMOTE_PLAYBACK_MIN_CHANGE) { + if (entry.playbackBoost <= 1) return {entry, command: null}; + if (entry.belowSinceMs === null || nowMs - entry.belowSinceMs < REMOTE_PLAYBACK_SILENCE_HOLD_MS) { return {entry, command: null}; } - return { - entry: {...entry, playbackBoost: clampedBoost}, - command: {type: 'setPlaybackBoost', identity: entry.identity, boost: clampedBoost}, - }; + const releasedBoost = 1 + (entry.playbackBoost - 1) * REMOTE_PLAYBACK_SILENCE_RELEASE; + if (releasedBoost - 1 <= REMOTE_PLAYBACK_UNITY_EPSILON) return snapRemotePlaybackBoost(entry); + return settleRemotePlaybackBoost(entry, releasedBoost); } function tickRemoteAnalyser( @@ -351,8 +381,7 @@ function tickRemoteAnalyser( if (event.trackEnded) return detachRemoteAnalyser(context, event.identity); const commands: Array = []; - let {entry, command} = updateRemotePlaybackBoost(existing, event.rms); - if (command) commands.push(command); + let entry = existing; if (event.rms >= event.threshold) { const aboveSinceMs = entry.aboveSinceMs ?? event.nowMs; entry = { @@ -376,6 +405,9 @@ function tickRemoteAnalyser( commands.push({type: 'setAudioLevelSpeaking', identity: event.identity, speaking: false}); } } + const boostUpdate = updateRemotePlaybackBoost(entry, event.rms, event.threshold, event.nowMs); + entry = boostUpdate.entry; + if (boostUpdate.command) commands.push(boostUpdate.command); if (entry === existing && commands.length === 0) return context; const analysers = new Map(context.analysers); analysers.set(event.identity, entry); @@ -509,6 +541,13 @@ export function clearVoiceRemoteSpeakingCommands(snapshot: VoiceRemoteSpeakingSn return transitionVoiceRemoteSpeakingSnapshot(snapshot, {type: 'remote.clearCommands'}); } +export const __TEST__ = { + REMOTE_PLAYBACK_TARGET_RMS, + REMOTE_PLAYBACK_MIN_RMS, + REMOTE_PLAYBACK_MAX_BOOST, + REMOTE_PLAYBACK_SILENCE_HOLD_MS, +}; + export function findParticipantSnapshotByUserIdAndConnectionId( participants: Readonly>, userId: string, diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.test.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.test.ts new file mode 100644 index 000000000..383f2c420 --- /dev/null +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.test.ts @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {describe, expect, it, vi} from 'vitest'; +import {computeTimeDomainRms} from './VoiceEngineV2AppRemoteSpeakingAdapter'; + +vi.mock('@app/features/voice/state/ParticipantVolume', () => ({default: {}})); +vi.mock('@app/features/voice/state/VoiceSettings', () => ({default: {getVadThreshold: () => 50}})); + +describe('computeTimeDomainRms', () => { + it('returns the amplitude of an alternating full-scale-relative signal', () => { + const samples = new Float32Array(512); + for (let i = 0; i < samples.length; i++) { + samples[i] = i % 2 === 0 ? 0.09 : -0.09; + } + expect(computeTimeDomainRms(samples)).toBe(Math.fround(0.09)); + }); + + it('returns zero for digital silence', () => { + expect(computeTimeDomainRms(new Float32Array(512))).toBe(0); + }); + + it('returns zero for an empty buffer', () => { + expect(computeTimeDomainRms(new Float32Array(0))).toBe(0); + }); + + it('spreads a single full-scale sample across the window', () => { + const samples = new Float32Array(512); + samples[17] = 1; + expect(computeTimeDomainRms(samples)).toBeCloseTo(1 / Math.sqrt(512), 12); + }); + + it('resolves levels far below one 8-bit quantisation step', () => { + expect(computeTimeDomainRms(Float32Array.of(0.0005, -0.0005))).toBe(Math.fround(0.0005)); + }); +}); diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.ts index 05f7ac70a..08e671d8e 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppRemoteSpeakingAdapter.ts @@ -30,12 +30,22 @@ const logger = new Logger('VoiceEngineV2AppRemoteSpeakingAdapter'); export const REMOTE_SPEAKING_ANALYSER_INTERVAL_MS = 50; export const REMOTE_SPEAKING_ANALYSER_HANDLES_CAP = 256; +export function computeTimeDomainRms(samples: Float32Array): number { + if (samples.length === 0) return 0; + let sumSquares = 0; + for (let i = 0; i < samples.length; i++) { + const sample = samples[i]!; + sumSquares += sample * sample; + } + return Math.sqrt(sumSquares / samples.length); +} + interface AnalyserHandle { identity: string; track: MediaStreamTrack; source: MediaStreamAudioSourceNode; analyser: AnalyserNode; - samples: Uint8Array; + samples: Float32Array; nextTickAtMs: number; } @@ -73,9 +83,7 @@ export class VoiceEngineV2AppRemoteSpeakingAdapter { break; case 'setPlaybackBoost': setRemoteVoicePlaybackBoost(command.identity, command.boost); - if (command.boost !== 1) { - this.reapplyParticipantVolume(command.identity); - } + this.reapplyParticipantVolume(command.identity); break; case 'clearPlaybackBoost': clearRemoteVoicePlaybackBoost(command.identity); @@ -193,7 +201,7 @@ export class VoiceEngineV2AppRemoteSpeakingAdapter { track: mediaStreamTrack, source, analyser, - samples: new Uint8Array(new ArrayBuffer(analyser.fftSize)), + samples: new Float32Array(new ArrayBuffer(analyser.fftSize * Float32Array.BYTES_PER_ELEMENT)), nextTickAtMs: this.nowMs() + REMOTE_SPEAKING_ANALYSER_INTERVAL_MS, }; this.analysers.set(identity, handle); @@ -315,13 +323,8 @@ export class VoiceEngineV2AppRemoteSpeakingAdapter { } return; } - handle.analyser.getByteTimeDomainData(handle.samples); - let sumSquares = 0; - for (let i = 0; i < handle.samples.length; i++) { - const normalized = (handle.samples[i]! - 128) / 128; - sumSquares += normalized * normalized; - } - const rms = Math.sqrt(sumSquares / handle.samples.length); + handle.analyser.getFloatTimeDomainData(handle.samples); + const rms = computeTimeDomainRms(handle.samples); const threshold = getRemoteSpeakingThresholdRms(VoiceSettings.getVadThreshold()); this.applyCommands(this.transition({type: 'remote.tick', identity: handle.identity, rms, threshold, nowMs})); handle.nextTickAtMs = nowMs + REMOTE_SPEAKING_ANALYSER_INTERVAL_MS;