From bcb30eea8842d564358ba4aabeb1af607f9725eb Mon Sep 17 00:00:00 2001 From: Hampus Date: Sun, 11 Oct 2026 19:27:24 +0200 Subject: [PATCH] fix(voice): one-way audio and stream loss in p2p calls (#3405) --- .../components/useVoiceCallTracksAndLayout.ts | 8 +++-- .../voice/engine/MediaEngineFacade.ts | 22 +++++++++++-- .../voice/engine/VoiceRoomEventBinder.ts | 22 ++++++++++--- .../voice/engine/mesh/VoiceMeshEngine.ts | 33 ++++++++++++------- .../VoiceEngineV2AppConnectionHostAdapter.ts | 11 ++++++- 5 files changed, 73 insertions(+), 23 deletions(-) diff --git a/fluxer_app/src/features/voice/components/useVoiceCallTracksAndLayout.ts b/fluxer_app/src/features/voice/components/useVoiceCallTracksAndLayout.ts index 73783591e..9a8931712 100644 --- a/fluxer_app/src/features/voice/components/useVoiceCallTracksAndLayout.ts +++ b/fluxer_app/src/features/voice/components/useVoiceCallTracksAndLayout.ts @@ -468,9 +468,11 @@ export function useVoiceCallTracksAndLayout({channel, expandedUserIds}: UseVoice useEffect(() => { if (!pinnedParticipantIdentity) return; if (cameraTracksAll.length === 0) return; - const stillExists = cameraTracksAll.some((tr) => tr.participant.identity === pinnedParticipantIdentity); - if (!stillExists) VoiceCallLayoutCommands.setPinnedParticipant(null); - }, [cameraTracksAll, pinnedParticipantIdentity]); + if (cameraTracksAll.some((tr) => tr.participant.identity === pinnedParticipantIdentity)) return; + const {connectionId} = parseVoiceParticipantIdentity(pinnedParticipantIdentity); + if (connectionVoiceStates[connectionId]?.channel_id === channel.id) return; + VoiceCallLayoutCommands.setPinnedParticipant(null); + }, [cameraTracksAll, channel.id, connectionVoiceStates, pinnedParticipantIdentity]); const { mainTrack: focusMainTrack, carouselTracks, diff --git a/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts b/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts index 6b94b3678..ded3d9171 100644 --- a/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts +++ b/fluxer_app/src/features/voice/engine/MediaEngineFacade.ts @@ -801,6 +801,7 @@ class MediaEngineFacade extends Store { restoreStream: boolean; screenShareSnapshot: ScreenShareReconnectSnapshot | null; toastWhenStreamNotRestored: boolean; + sendUpdateWhenStreamNotRestored: boolean; }): Promise { assert.equal(typeof args.restoreCamera, 'boolean', 'restoreLocalMedia requires a camera flag'); assert.equal(typeof args.restoreStream, 'boolean', 'restoreLocalMedia requires a stream flag'); @@ -828,7 +829,9 @@ class MediaEngineFacade extends Store { } if (restored) return; if (streamRestoreFailed || args.screenShareSnapshot !== null) { - voiceEngineV2AppMediaStateAdapter.applyScreenShareState(false, {sendUpdate: false}); + voiceEngineV2AppMediaStateAdapter.applyScreenShareState(false, { + sendUpdate: args.sendUpdateWhenStreamNotRestored, + }); } if (args.toastWhenStreamNotRestored && this.i18n) { ToastCommands.createToast({ @@ -1690,7 +1693,7 @@ class MediaEngineFacade extends Store { } voiceEngineV2AppMediaExecutionAdapter.resetMicrophoneFailureLatch(); this.reconcileLocalAudioStateInBackground('region hot-swap complete'); - if (isConversion) void this.reapplyVideoQualityLimits(newRoom); + if (isConversion) void this.finishMeshConversion(currentRoom, newRoom); }, (_guildId, _channelId, _connectionId, _attemptId, error) => { logger.warn('LiveKit connect failed, clearing gateway voice state', {error}); @@ -2715,6 +2718,19 @@ class MediaEngineFacade extends Store { } } + private async finishMeshConversion(meshRoom: Room, room: Room): Promise { + await this.restoreLocalMedia({ + reason: 'mesh conversion', + room, + restoreCamera: false, + restoreStream: false, + screenShareSnapshot: voiceEngineV2AppScreenShareExecutionAdapter.prepareScreenShareReconnect(meshRoom), + toastWhenStreamNotRestored: true, + sendUpdateWhenStreamNotRestored: true, + }); + await this.reapplyVideoQualityLimits(room); + } + private async restoreLocalMediaState(roomOverride?: Room | null): Promise { const room = roomOverride ?? voiceEngineV2AppConnectionHostAdapter.room; const participant = room?.localParticipant ?? null; @@ -2728,6 +2744,7 @@ class MediaEngineFacade extends Store { restoreStream: LocalVoiceState.getSelfStream() && !participant.isScreenShareEnabled, screenShareSnapshot: pendingScreenShareReconnect, toastWhenStreamNotRestored: false, + sendUpdateWhenStreamNotRestored: false, }); } @@ -2744,6 +2761,7 @@ class MediaEngineFacade extends Store { restoreStream: pendingRestore.restoreStream, screenShareSnapshot: null, toastWhenStreamNotRestored: true, + sendUpdateWhenStreamNotRestored: false, }); } diff --git a/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts b/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts index cf38b49cf..719bd9d5a 100644 --- a/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts +++ b/fluxer_app/src/features/voice/engine/VoiceRoomEventBinder.ts @@ -284,6 +284,20 @@ export function bindRoomEvents( participantSpeakingDisposers.get(identity)?.dispose(); participantSpeakingDisposers.delete(identity); }; + const observeRemoteScreenSharePublication = (participant: Participant, pub: RemoteTrackPublication): void => { + reportScreenSharePublicationObserved(participant.identity, pub.trackSid ?? null); + const streamKey = getStreamKeyForParticipantIdentity(guildId, channelId, participant.identity); + if (streamKey) { + cancelDeferredStopWatchingStreamKey(streamKey); + } + }; + const observeExistingRemoteScreenSharePublications = (): void => { + room.remoteParticipants.forEach((participant) => { + for (const pub of participant.trackPublications.values()) { + if (pub.source === Track.Source.ScreenShare) observeRemoteScreenSharePublication(participant, pub); + } + }); + }; const endScreenShareIfPublicationDidNotReturn = (): void => { const publication = screenSharePublicationAwaitingReconnect; screenSharePublicationAwaitingReconnect = null; @@ -305,6 +319,7 @@ export function bindRoomEvents( bindParticipantSpeakingEvents(room.localParticipant); room.remoteParticipants.forEach((participant) => bindParticipantSpeakingEvents(participant)); dependencies.remoteSpeaking.hydrateFromRoom(room); + observeExistingRemoteScreenSharePublications(); bindScreenShareNegotiation(); dependencies.permissions.applyDeafen(room, getEffectiveAudioState().effectiveDeaf); dependencies.connection.markConnected(); @@ -596,11 +611,7 @@ export function bindRoomEvents( pub.setSubscribed(true); } if (pub.source === Track.Source.ScreenShare && !participant.isLocal) { - reportScreenSharePublicationObserved(participant.identity, pub.trackSid ?? null); - const streamKey = getStreamKeyForParticipantIdentity(guildId, channelId, participant.identity); - if (streamKey) { - cancelDeferredStopWatchingStreamKey(streamKey); - } + observeRemoteScreenSharePublication(participant, pub); dependencies.subscriptions.reattachScreenShareAfterPublish(participant.identity, pub); dependencies.subscriptions.reconcileSubscriptions(); } @@ -615,6 +626,7 @@ export function bindRoomEvents( }), ); if (room.state === ConnectionState.Connected) { + guard(attemptId, observeExistingRemoteScreenSharePublications)(); guard(attemptId, bindScreenShareNegotiation)(); } } diff --git a/fluxer_app/src/features/voice/engine/mesh/VoiceMeshEngine.ts b/fluxer_app/src/features/voice/engine/mesh/VoiceMeshEngine.ts index 66f3054d5..eb59103ca 100644 --- a/fluxer_app/src/features/voice/engine/mesh/VoiceMeshEngine.ts +++ b/fluxer_app/src/features/voice/engine/mesh/VoiceMeshEngine.ts @@ -320,6 +320,7 @@ export class VoiceMeshEngine extends RTCEngine { handlePeerStatus(peer: VoiceMeshPeer): void { if (this.peers.get(peer.connectionId) !== peer) return; const {status} = peer; + if (status === 'failed' && this.ownP2p() === false) return; VoiceMeshPeers.setStatus(peer.connectionId, status); if (status === 'connecting') return; this.reportPeer(peer, status); @@ -395,28 +396,30 @@ export class VoiceMeshEngine extends RTCEngine { const serverMuted = src === 'microphone' && voiceEngineV2AppVoiceStateAdapter.getVoiceStateByConnectionId(peer.connectionId)?.mute === true; - if (receiver && receiver.track.readyState === 'live' && publication.isDesired && !serverMuted) { - if (publication.track?.mediaStreamTrack !== receiver.track) { - participant.addSubscribedMediaTrack(receiver.track, info.sid, new MediaStream([receiver.track]), receiver); - } - return; + const live = receiver !== null && receiver.track.readyState === 'live' && publication.isDesired && !serverMuted; + if (live && publication.track?.mediaStreamTrack === receiver.track) return; + if (publication.track) { + publication.track.stop(); + publication.setTrack(undefined); + } + if (live) { + participant.addSubscribedMediaTrack(receiver.track, info.sid, new MediaStream([receiver.track]), receiver); } - if (!publication.track) return; - publication.track.stop(); - publication.setTrack(undefined); } private readonly syncRoster = (): void => { if (!this.room) return; const {connectionId, channelId, guildId} = this.target; - const own = voiceEngineV2AppVoiceStateAdapter.getVoiceStateByConnectionId(connectionId); - const ownP2p = own?.channel_id === channelId ? own.p2p : undefined; + const ownP2p = this.ownP2p(); this.syncConversionWatchdog(ownP2p === false && !this.conversionHeld); const roster = new Map(); - if (ownP2p === true) { + if (ownP2p !== undefined) { const states = voiceEngineV2AppVoiceStateAdapter.getAllVoiceStatesInChannel(guildId ?? ME, channelId); for (const state of Object.values(states)) { - if (state.p2p === true && state.connection_id !== connectionId) roster.set(state.connection_id, state); + if (state.connection_id === connectionId) continue; + if (this.peers.has(state.connection_id) || (ownP2p && state.p2p === true)) { + roster.set(state.connection_id, state); + } } } const before = this.peers.size; @@ -431,6 +434,12 @@ export class VoiceMeshEngine extends RTCEngine { for (const peer of this.peers.values()) this.syncPeerReceiver(peer, 'microphone'); }; + private ownP2p(): boolean | undefined { + const {connectionId, channelId} = this.target; + const own = voiceEngineV2AppVoiceStateAdapter.getVoiceStateByConnectionId(connectionId); + return own?.channel_id === channelId ? own.p2p : undefined; + } + private addPeer(state: NormalizedVoiceState): void { const peer = new VoiceMeshPeer(this, state.connection_id, state.user_id, this.target.connectionId); this.peers.set(peer.connectionId, peer); diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.ts index 7d86b80a2..fcefd44db 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppConnectionHostAdapter.ts @@ -3,7 +3,11 @@ import assert from 'node:assert/strict'; import Authentication from '@app/features/auth/state/Authentication'; import {Logger} from '@app/features/platform/utils/AppLogger'; -import {createVoiceMeshRoom, holdVoiceMeshConversion} from '@app/features/voice/engine/mesh/VoiceMeshRoom'; +import { + createVoiceMeshRoom, + holdVoiceMeshConversion, + isVoiceMeshRoom, +} from '@app/features/voice/engine/mesh/VoiceMeshRoom'; import {Store} from '@app/features/voice/engine/Store'; import {sendVoiceStateDisconnect} from '@app/features/voice/engine/VoiceChannelConnector'; import { @@ -785,8 +789,13 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store { logger.warn('Region hot-swap: missing participant for track republishing'); return; } + const isMeshConversion = isVoiceMeshRoom(oldRoom); const publications = selectLocalMediaPublicationsForConnectionRepublish( Array.from(oldParticipant.trackPublications.values()), + ).filter( + (publication) => + !isMeshConversion || + (publication.source !== Track.Source.ScreenShare && publication.source !== Track.Source.ScreenShareAudio), ); const errors: Array<{source: string; error: unknown}> = []; let codecPolicyFailure: Error | null = null;