diff --git a/fluxer_app/src/env.d.ts b/fluxer_app/src/env.d.ts index 809fdae8b..c88ec00a4 100644 --- a/fluxer_app/src/env.d.ts +++ b/fluxer_app/src/env.d.ts @@ -16,6 +16,8 @@ interface FluxerDebugApi { getClientInfoSync?: () => string; getClientInfoObject?: () => Promise; getClientInfoObjectSync?: () => unknown; + getVoiceSubscriptionDebug?: () => Promise; + getVoiceSubscriptionDebugJson?: () => Promise; } type FluxerDebugGlobal = Record & FluxerDebugApi; diff --git a/fluxer_app/src/features/platform/utils/ClientInfo.ts b/fluxer_app/src/features/platform/utils/ClientInfo.ts index 2e9488f13..4d9f5fd4c 100644 --- a/fluxer_app/src/features/platform/utils/ClientInfo.ts +++ b/fluxer_app/src/features/platform/utils/ClientInfo.ts @@ -3,6 +3,7 @@ import i18n from '@app/app/I18n'; import Config from '@app/features/app/config/Config'; import {Logger} from '@app/features/platform/utils/AppLogger'; +import {getFluxerDebugObject} from '@app/features/platform/utils/FluxerDebugGlobal'; import {getElectronAPI, isDesktop} from '@app/features/ui/utils/NativeUtils'; import type {DesktopInfo} from '@app/types/electron.d'; import Bowser from 'bowser'; @@ -325,29 +326,6 @@ export async function getFormattedClientInfo(): Promise { return formatClientBuildInfo(await getClientInfo()); } -interface FluxerDebugApi { - getClientInfo?: () => Promise; - getClientInfoSync?: () => string; - getClientInfoObject?: () => Promise; - getClientInfoObjectSync?: () => ClientInfo; -} - -function getFluxerDebugObject(): (Record & FluxerDebugApi) | null { - if (typeof window === 'undefined') { - return null; - } - const win = window as Window & { - __FLUXER_DEBUG__?: Record & FluxerDebugApi; - }; - if (win.__FLUXER_DEBUG__ === undefined || win.__FLUXER_DEBUG__ === null) { - win.__FLUXER_DEBUG__ = {}; - } - if (typeof win.__FLUXER_DEBUG__ !== 'object' || Array.isArray(win.__FLUXER_DEBUG__)) { - return null; - } - return win.__FLUXER_DEBUG__; -} - export function installFluxerConfigDebugApi(): void { const config = getFluxerDebugObject(); if (!config) { diff --git a/fluxer_app/src/features/platform/utils/FluxerDebugGlobal.ts b/fluxer_app/src/features/platform/utils/FluxerDebugGlobal.ts new file mode 100644 index 000000000..d5b50b2ef --- /dev/null +++ b/fluxer_app/src/features/platform/utils/FluxerDebugGlobal.ts @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +type FluxerDebugObject = NonNullable; + +export function getFluxerDebugObject(): FluxerDebugObject | null { + if (typeof window === 'undefined') { + return null; + } + const existing = window.__FLUXER_DEBUG__; + if (existing === undefined || existing === null) { + const created: FluxerDebugObject = {}; + window.__FLUXER_DEBUG__ = created; + return created; + } + if (typeof existing !== 'object' || Array.isArray(existing)) { + return null; + } + return existing; +} diff --git a/fluxer_app/src/features/voice/diagnostics/VoiceSubscriptionDebugApi.ts b/fluxer_app/src/features/voice/diagnostics/VoiceSubscriptionDebugApi.ts new file mode 100644 index 000000000..7ce9f46ef --- /dev/null +++ b/fluxer_app/src/features/voice/diagnostics/VoiceSubscriptionDebugApi.ts @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {Logger} from '@app/features/platform/utils/AppLogger'; +import {getFluxerDebugObject} from '@app/features/platform/utils/FluxerDebugGlobal'; +import {loadLazyModule} from '@app/features/platform/utils/LazyModuleLoader'; + +const logger = new Logger('VoiceSubscriptionDebugApi'); + +async function collectReport(): Promise { + const {collectVoiceSubscriptionDebugReport} = await loadLazyModule( + () => import('@app/features/voice/diagnostics/VoiceSubscriptionDebugReport'), + ); + return collectVoiceSubscriptionDebugReport(); +} + +export function installVoiceSubscriptionDebugApi(): void { + const debugObject = getFluxerDebugObject(); + if (!debugObject) { + return; + } + try { + debugObject.getVoiceSubscriptionDebug = collectReport; + debugObject.getVoiceSubscriptionDebugJson = async () => JSON.stringify(await collectReport(), null, 2); + } catch (error) { + logger.warn('Failed to install __FLUXER_DEBUG__ voice subscription helpers', error); + } +} diff --git a/fluxer_app/src/features/voice/diagnostics/VoiceSubscriptionDebugReport.ts b/fluxer_app/src/features/voice/diagnostics/VoiceSubscriptionDebugReport.ts new file mode 100644 index 000000000..3137912bf --- /dev/null +++ b/fluxer_app/src/features/voice/diagnostics/VoiceSubscriptionDebugReport.ts @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import Config from '@app/features/app/config/Config'; +import { + summarizeVoiceSubscriptionDebug, + type VoiceSubscriptionDebugMediaStreamTrack, + type VoiceSubscriptionDebugParticipant, + type VoiceSubscriptionDebugPublication, + type VoiceSubscriptionDebugSubscription, + type VoiceSubscriptionDebugSummary, + type VoiceSubscriptionDebugTransceiver, +} from '@app/features/voice/diagnostics/VoiceSubscriptionDebugSummary'; +import MediaEngine from '@app/features/voice/engine/MediaEngineFacade'; +import type {VoiceMediaGraphSubscriptionCommand} from '@app/features/voice/engine/VoiceMediaGraph'; +import {voiceMediaGraphStore} from '@app/features/voice/engine/VoiceMediaGraphStore'; +import type {RemoteTrackPublication, Room} from 'livekit-client'; + +interface VoiceSubscriptionDebugFailure { + key: string; + code: number; + reason: string; + reportedAt: number; + streamKey: string | null; + participantIdentity: string | null; + participantSid: string | null; + trackSid: string | null; + source: string | null; + generation: number | null; + error: string | null; +} + +interface VoiceSubscriptionDebugReport { + capturedAt: string; + buildVersion: string; + graphNowMs: number; + connection: { + connected: boolean; + connecting: boolean; + reconnecting: boolean; + guildId: string | null; + channelId: string | null; + connectionId: string | null; + voiceServerEndpoint: string | null; + roomState: string | null; + localIdentity: string | null; + }; + summary: VoiceSubscriptionDebugSummary; + publisherTransceivers: Array; + subscriberTransportPresent: boolean; + remoteParticipants: Array; + subscriptions: Array; + pendingSubscriptionCommands: Array; + watchIntent: {viewerStreamKeys: Array; deferredStopKeys: Array}; + watchAttempts: Array<{ + streamKey: string; + attemptKey: string; + startedAt: number; + hasRenderedVideoFrame: boolean; + generation: number; + }>; + deadlines: Array<{ + key: string; + kind: string; + streamKey: string | null; + subscriptionKey: string | null; + attemptKey: string | null; + generation: number; + dueAt: number; + }>; + failures: Array; +} + +function describeMediaStreamTrack( + track: MediaStreamTrack | null | undefined, +): VoiceSubscriptionDebugMediaStreamTrack | null { + if (!track) { + return null; + } + return { + id: track.id, + kind: track.kind, + enabled: track.enabled, + muted: track.muted, + readyState: track.readyState, + }; +} + +function describeTransceivers(room: Room | null): Array { + const publisher = room?.engine?.pcManager?.publisher; + if (!publisher) { + return []; + } + return publisher.getTransceivers().map((transceiver) => ({ + mid: transceiver.mid, + direction: transceiver.direction, + currentDirection: transceiver.currentDirection, + receiver: describeMediaStreamTrack(transceiver.receiver?.track), + sender: describeMediaStreamTrack(transceiver.sender?.track), + })); +} + +function describePublication(publication: RemoteTrackPublication): VoiceSubscriptionDebugPublication { + const dimensions = publication.dimensions; + return { + trackSid: publication.trackSid, + trackName: publication.trackName, + kind: publication.kind, + source: publication.source, + mimeType: publication.mimeType ?? null, + dimensions: dimensions ? {width: dimensions.width, height: dimensions.height} : null, + simulcasted: publication.simulcasted ?? null, + muted: publication.isMuted, + enabled: publication.isEnabled, + subscribed: publication.isSubscribed, + desired: publication.isDesired, + subscriptionStatus: publication.subscriptionStatus, + permissionStatus: publication.permissionStatus, + videoQuality: publication.videoQuality ?? null, + track: describeMediaStreamTrack(publication.track?.mediaStreamTrack), + }; +} + +function describeRemoteParticipants(room: Room | null): Array { + if (!room) { + return []; + } + return Array.from(room.remoteParticipants.values()).map((participant) => ({ + identity: participant.identity, + sid: participant.sid, + publications: Array.from(participant.trackPublications.values()).map(describePublication), + })); +} + +function describeError(error: unknown): string | null { + if (error == null) { + return null; + } + if (error instanceof Error) { + return `${error.name}: ${error.message}`; + } + return String(error); +} + +export function collectVoiceSubscriptionDebugReport(): VoiceSubscriptionDebugReport { + const room = MediaEngine.room; + const snapshot = voiceMediaGraphStore.getGraphSnapshot(); + const transceivers = describeTransceivers(room); + const remoteParticipants = describeRemoteParticipants(room); + const subscriptions = Array.from(snapshot.subscriptionsByKey.entries()).map(([key, entry]) => ({ + key, + participantIdentity: entry.participantIdentity, + source: entry.source, + desired: { + enabled: entry.desired.enabled, + quality: entry.desired.quality, + context: entry.desired.context, + isIntersecting: entry.desired.isIntersecting, + hasObservedElement: entry.desired.observedElement != null, + }, + actual: { + subscribed: entry.actual.subscribed, + enabled: entry.actual.enabled, + quality: entry.actual.quality, + lastCommandAt: entry.actual.lastCommandAt, + lastError: entry.actual.lastError, + }, + publication: { + available: entry.publication.available, + trackSid: entry.publication.trackSid, + observedAt: entry.publication.observedAt, + }, + firstFrame: {renderedAt: entry.firstFrame.renderedAt}, + subscribed: entry.subscribed, + })); + const pendingSubscriptionCommands = Array.from(snapshot.subscriptionCommands); + return { + capturedAt: new Date().toISOString(), + buildVersion: Config.PUBLIC_BUILD_VERSION ?? 'dev', + graphNowMs: voiceMediaGraphStore.nowMs(), + connection: { + connected: MediaEngine.connected, + connecting: MediaEngine.connecting, + reconnecting: MediaEngine.reconnecting, + guildId: MediaEngine.guildId, + channelId: MediaEngine.channelId, + connectionId: MediaEngine.connectionId, + voiceServerEndpoint: MediaEngine.voiceServerEndpoint, + roomState: room?.state ?? null, + localIdentity: room?.localParticipant.identity ?? null, + }, + summary: summarizeVoiceSubscriptionDebug( + transceivers, + remoteParticipants, + subscriptions, + pendingSubscriptionCommands.length, + ), + publisherTransceivers: transceivers, + subscriberTransportPresent: room?.engine?.pcManager?.subscriber != null, + remoteParticipants, + subscriptions, + pendingSubscriptionCommands, + watchIntent: { + viewerStreamKeys: Array.from(snapshot.watchIntent.viewerStreamKeys), + deferredStopKeys: Array.from(snapshot.watchIntent.deferredStopKeys), + }, + watchAttempts: Array.from(snapshot.attemptsByStreamKey.entries()).map(([streamKey, attempt]) => ({ + streamKey, + attemptKey: attempt.attemptKey, + startedAt: attempt.startedAt, + hasRenderedVideoFrame: attempt.hasRenderedVideoFrame, + generation: attempt.generation, + })), + deadlines: Array.from(snapshot.deadlinesByKey.entries()).map(([key, deadline]) => ({ + key, + kind: deadline.kind, + streamKey: deadline.streamKey, + subscriptionKey: deadline.subscriptionKey, + attemptKey: deadline.attemptKey, + generation: deadline.generation, + dueAt: deadline.dueAt, + })), + failures: Array.from(snapshot.failuresByKey.entries()).map(([key, failure]) => ({ + key, + code: failure.code, + reason: failure.reason, + reportedAt: failure.reportedAt, + streamKey: failure.streamKey ?? null, + participantIdentity: failure.participantIdentity ?? null, + participantSid: failure.participantSid ?? null, + trackSid: failure.trackSid ?? null, + source: failure.source ?? null, + generation: failure.generation ?? null, + error: describeError(failure.error), + })), + }; +} diff --git a/fluxer_app/src/features/voice/diagnostics/VoiceSubscriptionDebugSummary.test.ts b/fluxer_app/src/features/voice/diagnostics/VoiceSubscriptionDebugSummary.test.ts new file mode 100644 index 000000000..a4e546a51 --- /dev/null +++ b/fluxer_app/src/features/voice/diagnostics/VoiceSubscriptionDebugSummary.test.ts @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import { + summarizeVoiceSubscriptionDebug, + type VoiceSubscriptionDebugParticipant, + type VoiceSubscriptionDebugPublication, + type VoiceSubscriptionDebugSubscription, + type VoiceSubscriptionDebugTransceiver, +} from '@app/features/voice/diagnostics/VoiceSubscriptionDebugSummary'; +import {describe, expect, it} from 'vitest'; + +function transceiver(overrides: Partial): VoiceSubscriptionDebugTransceiver { + return { + mid: '0', + direction: 'recvonly', + currentDirection: 'recvonly', + receiver: {id: 'receiver', kind: 'video', enabled: true, muted: false, readyState: 'live'}, + sender: null, + ...overrides, + }; +} + +function publication(overrides: Partial): VoiceSubscriptionDebugPublication { + return { + trackSid: 'TR_1', + trackName: 'screen', + kind: 'video', + source: 'screen_share', + mimeType: 'video/VP9', + dimensions: {width: 1920, height: 1080}, + simulcasted: false, + muted: false, + enabled: true, + subscribed: false, + desired: true, + subscriptionStatus: 'desired', + permissionStatus: 'allowed', + videoQuality: 2, + track: null, + ...overrides, + }; +} + +function participant(publications: Array): VoiceSubscriptionDebugParticipant { + return {identity: 'user_1_connection-a', sid: 'PA_1', publications}; +} + +function subscription(overrides: Partial): VoiceSubscriptionDebugSubscription { + return { + key: 'user_1_connection-a:screen_share', + participantIdentity: 'user_1_connection-a', + source: 'screen_share', + desired: {enabled: true, quality: 'high', context: 'focused', isIntersecting: true, hasObservedElement: false}, + actual: {subscribed: null, enabled: null, quality: null, lastCommandAt: null, lastError: null}, + publication: {available: true, trackSid: 'TR_1', observedAt: 1000}, + firstFrame: {renderedAt: null}, + subscribed: true, + ...overrides, + }; +} + +describe('summarizeVoiceSubscriptionDebug', () => { + it('reports zero issued commands when the client never asked for the stream', () => { + const summary = summarizeVoiceSubscriptionDebug([], [participant([publication({})])], [subscription({})], 0); + expect(summary.remoteScreenSharePublications).toBe(1); + expect(summary.screenShareSubscriptions).toBe(1); + expect(summary.screenShareSubscriptionsWithCommandIssued).toBe(0); + expect(summary.videoReceiveTransceivers).toBe(0); + expect(summary.negotiatedVideoReceiveTransceivers).toBe(0); + expect(summary.screenSharePublicationsWithTrack).toBe(0); + }); + + it('counts a negotiated receive transceiver once the answer applies it', () => { + const summary = summarizeVoiceSubscriptionDebug( + [transceiver({}), transceiver({mid: '1', currentDirection: null})], + [participant([publication({subscribed: true, track: null})])], + [subscription({actual: {subscribed: true, enabled: true, quality: 'high', lastCommandAt: 5, lastError: null}})], + 2, + ); + expect(summary.videoReceiveTransceivers).toBe(2); + expect(summary.negotiatedVideoReceiveTransceivers).toBe(1); + expect(summary.subscribedScreenSharePublications).toBe(1); + expect(summary.screenShareSubscriptionsWithCommandIssued).toBe(1); + expect(summary.pendingSubscriptionCommands).toBe(2); + }); + + it('ignores send-only video, audio receivers, and non-screen-share sources', () => { + const summary = summarizeVoiceSubscriptionDebug( + [ + transceiver({direction: 'sendonly', currentDirection: 'sendonly'}), + transceiver({ + mid: '1', + receiver: {id: 'audio', kind: 'audio', enabled: true, muted: false, readyState: 'live'}, + }), + ], + [ + participant([ + publication({source: 'camera'}), + publication({trackSid: 'TR_2', kind: 'audio', source: 'screen_share_audio'}), + ]), + ], + [subscription({source: 'camera'})], + 0, + ); + expect(summary.videoReceiveTransceivers).toBe(0); + expect(summary.negotiatedVideoReceiveTransceivers).toBe(0); + expect(summary.remoteScreenSharePublications).toBe(0); + expect(summary.screenShareSubscriptions).toBe(0); + }); + + it('counts a rendered first frame separately from an attached track', () => { + const summary = summarizeVoiceSubscriptionDebug( + [transceiver({})], + [ + participant([ + publication({ + subscribed: true, + track: {id: 'track', kind: 'video', enabled: true, muted: false, readyState: 'live'}, + }), + ]), + ], + [subscription({firstFrame: {renderedAt: 42}})], + 0, + ); + expect(summary.screenSharePublicationsWithTrack).toBe(1); + expect(summary.screenShareSubscriptionsWithFirstFrame).toBe(1); + }); +}); diff --git a/fluxer_app/src/features/voice/diagnostics/VoiceSubscriptionDebugSummary.ts b/fluxer_app/src/features/voice/diagnostics/VoiceSubscriptionDebugSummary.ts new file mode 100644 index 000000000..bab6934d9 --- /dev/null +++ b/fluxer_app/src/features/voice/diagnostics/VoiceSubscriptionDebugSummary.ts @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {VOICE_MEDIA_GRAPH_SCREEN_SHARE_SOURCE} from '@app/features/voice/engine/VoiceMediaGraph'; + +export interface VoiceSubscriptionDebugMediaStreamTrack { + id: string; + kind: string; + enabled: boolean; + muted: boolean; + readyState: string; +} + +export interface VoiceSubscriptionDebugTransceiver { + mid: string | null; + direction: string; + currentDirection: string | null; + receiver: VoiceSubscriptionDebugMediaStreamTrack | null; + sender: VoiceSubscriptionDebugMediaStreamTrack | null; +} + +export interface VoiceSubscriptionDebugPublication { + trackSid: string; + trackName: string; + kind: string; + source: string; + mimeType: string | null; + dimensions: {width: number; height: number} | null; + simulcasted: boolean | null; + muted: boolean; + enabled: boolean; + subscribed: boolean; + desired: boolean; + subscriptionStatus: string; + permissionStatus: string; + videoQuality: number | null; + track: VoiceSubscriptionDebugMediaStreamTrack | null; +} + +export interface VoiceSubscriptionDebugParticipant { + identity: string; + sid: string; + publications: Array; +} + +export interface VoiceSubscriptionDebugSubscription { + key: string; + participantIdentity: string; + source: string; + desired: { + enabled: boolean; + quality: string; + context: string; + isIntersecting: boolean; + hasObservedElement: boolean; + }; + actual: { + subscribed: boolean | null; + enabled: boolean | null; + quality: string | null; + lastCommandAt: number | null; + lastError: {code: number; reason: string; at: number} | null; + }; + publication: {available: boolean; trackSid: string | null; observedAt: number | null}; + firstFrame: {renderedAt: number | null}; + subscribed: boolean; +} + +export interface VoiceSubscriptionDebugSummary { + videoReceiveTransceivers: number; + negotiatedVideoReceiveTransceivers: number; + remoteScreenSharePublications: number; + subscribedScreenSharePublications: number; + screenSharePublicationsWithTrack: number; + screenShareSubscriptions: number; + screenShareSubscriptionsWithCommandIssued: number; + screenShareSubscriptionsWithFirstFrame: number; + pendingSubscriptionCommands: number; +} + +function isReceiveDirection(direction: string | null): boolean { + return direction === 'recvonly' || direction === 'sendrecv'; +} + +export function summarizeVoiceSubscriptionDebug( + transceivers: ReadonlyArray, + participants: ReadonlyArray, + subscriptions: ReadonlyArray, + pendingSubscriptionCommands: number, +): VoiceSubscriptionDebugSummary { + const videoTransceivers = transceivers.filter((transceiver) => transceiver.receiver?.kind === 'video'); + const screenSharePublications = participants + .flatMap((participant) => participant.publications) + .filter( + (publication) => publication.source === VOICE_MEDIA_GRAPH_SCREEN_SHARE_SOURCE && publication.kind === 'video', + ); + const screenShareSubscriptions = subscriptions.filter( + (subscription) => subscription.source === VOICE_MEDIA_GRAPH_SCREEN_SHARE_SOURCE, + ); + return { + videoReceiveTransceivers: videoTransceivers.filter((transceiver) => isReceiveDirection(transceiver.direction)) + .length, + negotiatedVideoReceiveTransceivers: videoTransceivers.filter((transceiver) => + isReceiveDirection(transceiver.currentDirection), + ).length, + remoteScreenSharePublications: screenSharePublications.length, + subscribedScreenSharePublications: screenSharePublications.filter((publication) => publication.subscribed).length, + screenSharePublicationsWithTrack: screenSharePublications.filter((publication) => publication.track != null).length, + screenShareSubscriptions: screenShareSubscriptions.length, + screenShareSubscriptionsWithCommandIssued: screenShareSubscriptions.filter( + (subscription) => subscription.actual.lastCommandAt != null, + ).length, + screenShareSubscriptionsWithFirstFrame: screenShareSubscriptions.filter( + (subscription) => subscription.firstFrame.renderedAt != null, + ).length, + pendingSubscriptionCommands, + }; +} diff --git a/fluxer_app/src/index.tsx b/fluxer_app/src/index.tsx index c4b488878..466014b03 100644 --- a/fluxer_app/src/index.tsx +++ b/fluxer_app/src/index.tsx @@ -30,6 +30,7 @@ import { } from '@app/features/platform/utils/ClientInfo'; import {loadLazyModule} from '@app/features/platform/utils/LazyModuleLoader'; import {scheduleNonLatinScriptFaces} from '@app/features/theme/fonts/ScriptFontLoader'; +import {installVoiceSubscriptionDebugApi} from '@app/features/voice/diagnostics/VoiceSubscriptionDebugApi'; import {i18n} from '@lingui/core'; import {I18nProvider} from '@lingui/react'; import {configure} from 'mobx'; @@ -48,6 +49,7 @@ if (typeof window !== 'undefined' && window.history) { } installFluxerConfigDebugApi(); +installVoiceSubscriptionDebugApi(); function createRoot(): ReactDOM.Root { const container = document.getElementById('root');