refactor(voice): remove heartbeat and debug logging sessions (#2121)

This commit is contained in:
Hampus authored and GitHub committed 2026-08-30 21:08:46 +02:00
1 parent 36b85512c6
commit b4d9cdc584
33 files changed
+566 -3783

No files matched your search

@@ -83,9 +83,6 @@ export const Endpoints = {
CHANNEL_CALL: (channelId: string) => `/channels/${channelId}/call`,
CHANNEL_CALL_RING: (channelId: string) => `/channels/${channelId}/call/ring`,
CHANNEL_CALL_STOP_RINGING: (channelId: string) => `/channels/${channelId}/call/stop-ringing`,
CHANNEL_VOICE_DEBUG_LOGGING_SESSION: (channelId: string) => `/channels/${channelId}/voice-debug-logging/session`,
CHANNEL_VOICE_DEBUG_LOGGING_EVENTS: (channelId: string) => `/channels/${channelId}/voice-debug-logging/events`,
CHANNEL_VOICE_PRESENCE_HEARTBEAT: (channelId: string) => `/channels/${channelId}/voice-presence/heartbeat`,
GUILDS: '/guilds',
GUILD: (guildId: string) => `/guilds/${guildId}`,
GUILD_CHANNELS: (guildId: string) => `/guilds/${guildId}/channels`,
@@ -9,7 +9,7 @@ import {
renderVoiceDebugStatsHtml,
renderVoiceDebugStatsUnavailableHtml,
} from '@app/features/voice/diagnostics/VoiceDebugStatsHtml';
import voiceEngineV2AppDebugLoggingHostAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppDebugLoggingHostAdapter';
import voiceEngineV2AppDebugEventSinkHostAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppDebugEventSinkHostAdapter';
import {collectStatsForNerdsSnapshot} from '@app/features/voice/utils/StatsForNerdsCopy';
export function canOpenVoiceDebugEventSinkPopout(): boolean {
@@ -41,5 +41,5 @@ function publishBrowserOpeningStatsSnapshot(): void {
export async function openVoiceDebugEventSinkPopout(): Promise<void> {
publishBrowserOpeningStatsSnapshot();
await voiceEngineV2AppDebugLoggingHostAdapter.openEventSinkPopout();
await voiceEngineV2AppDebugEventSinkHostAdapter.openEventSinkPopout();
}
@@ -1,72 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Endpoints} from '@app/features/app/constants/Endpoints';
import {http} from '@app/features/platform/transport/RestTransport';
import type {
VoiceDebugLoggingEventsResponse,
VoiceDebugLoggingStatusResponse,
} from '@fluxer/schema/src/domains/channel/ChannelSchemas';
export interface VoiceDebugLoggingEvent {
type: string;
timestamp_ns: string;
monotonic_ns?: string;
data?: Record<string, unknown>;
}
export async function fetchStatus(channelId: string): Promise<VoiceDebugLoggingStatusResponse> {
const response = await http.get<VoiceDebugLoggingStatusResponse>(
Endpoints.CHANNEL_VOICE_DEBUG_LOGGING_SESSION(channelId),
{retries: 1},
);
return (
response.body ?? {
active: false,
session_id: null,
activated_by_user_id: null,
started_at_ms: null,
expires_at_ms: null,
poll_interval_ms: 10000,
upload_interval_ms: 2000,
}
);
}
export async function setEnabled(channelId: string, enabled: boolean): Promise<VoiceDebugLoggingStatusResponse> {
const response = await http.put<VoiceDebugLoggingStatusResponse>(
Endpoints.CHANNEL_VOICE_DEBUG_LOGGING_SESSION(channelId),
{body: {enabled}, retries: 1},
);
const fallback = response.body ?? {
active: false,
session_id: null,
activated_by_user_id: null,
started_at_ms: null,
expires_at_ms: null,
poll_interval_ms: 10000,
upload_interval_ms: 2000,
};
return await fetchStatus(channelId).catch(() => fallback);
}
export async function uploadEvents(params: {
channelId: string;
sessionId: string;
connectionId: string | null;
participantIdentity: string | null;
events: Array<VoiceDebugLoggingEvent>;
}): Promise<VoiceDebugLoggingEventsResponse> {
const response = await http.post<VoiceDebugLoggingEventsResponse>(
Endpoints.CHANNEL_VOICE_DEBUG_LOGGING_EVENTS(params.channelId),
{
body: {
session_id: params.sessionId,
connection_id: params.connectionId ?? undefined,
participant_identity: params.participantIdentity ?? undefined,
events: params.events,
},
retries: 1,
},
);
return response.body ?? {accepted: false, active: false, stored_event_count: 0};
}
@@ -1,51 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Endpoints} from '@app/features/app/constants/Endpoints';
import {http} from '@app/features/platform/transport/RestTransport';
import type {
VoicePresenceHeartbeatEndResponse,
VoicePresenceHeartbeatResponse,
} from '@fluxer/schema/src/domains/channel/ChannelSchemas';
export async function heartbeat(params: {
channelId: string;
connectionId: string;
}): Promise<VoicePresenceHeartbeatResponse> {
const response = await http.post<VoicePresenceHeartbeatResponse>(
Endpoints.CHANNEL_VOICE_PRESENCE_HEARTBEAT(params.channelId),
{
body: {connection_id: params.connectionId},
mode: 'silent',
retries: 1,
timeoutMs: 5000,
},
);
if (response.ok && response.body && typeof response.body.ok === 'boolean') {
return response.body;
}
return {
ok: false,
heartbeat_interval_ms: 15000,
heartbeat_ttl_ms: 45000,
expires_at_ms: 0,
};
}
export async function end(params: {
channelId: string;
connectionId: string;
}): Promise<VoicePresenceHeartbeatEndResponse> {
const response = await http.delete<VoicePresenceHeartbeatEndResponse>(
Endpoints.CHANNEL_VOICE_PRESENCE_HEARTBEAT(params.channelId),
{
body: {connection_id: params.connectionId},
mode: 'silent',
retries: 1,
timeoutMs: 5000,
},
);
if (response.ok && response.body && typeof response.body.ok === 'boolean') {
return response.body;
}
return {ok: false};
}
@@ -647,9 +647,10 @@ export const VoiceMoreOptionsMenu: React.FC<VoiceMoreOptionsMenuProps> = observe
const layoutMode = VoiceCallLayout.layoutMode;
const isGrid = layoutMode === 'grid';
const connectedChannelId = MediaEngine.channelId;
const canControlDebugLogging = connectedChannelId != null && (Users.currentUser?.isStaff() ?? false);
const canOpenDebugEventSink =
canControlDebugLogging && VoiceDebugEventSinkCommands.canOpenVoiceDebugEventSinkPopout();
connectedChannelId != null &&
(Users.currentUser?.isStaff() ?? false) &&
VoiceDebugEventSinkCommands.canOpenVoiceDebugEventSinkPopout();
const isDmVoiceCall = connectedChannelId != null && (MediaEngine.guildId ?? null) === null;
const currentRegion =
isDmVoiceCall && connectedChannelId
@@ -841,25 +842,6 @@ export const VoiceMoreOptionsMenu: React.FC<VoiceMoreOptionsMenuProps> = observe
>
{i18n._(PRIORITIZE_SPEAKERS_DESCRIPTOR)}
</CheckboxItem>
{canControlDebugLogging && (
<CheckboxItem
icon={
<ChartBarIcon
weight="fill"
className={styles.icon}
data-flx="voice.voice-settings-menus.voice-more-options-menu.icon.debug-logging"
/>
}
checked={MediaEngine.voiceDebugLoggingActive}
disabled={MediaEngine.voiceDebugLoggingToggleInFlight}
onCheckedChange={(checked) => {
void MediaEngine.setVoiceDebugLoggingEnabled(checked);
}}
data-flx="voice.voice-settings-menus.voice-more-options-menu.checkbox-item.debug-logging"
>
<Trans>Start debug logging session</Trans>
</CheckboxItem>
)}
{canOpenDebugEventSink && (
<MenuItem
icon={
@@ -59,11 +59,7 @@ import {
saveCurrentVoiceSessionRestoreSnapshot,
type VoiceSessionRestoreSyncHandle,
} from '@app/features/voice/engine/media_engine_facade/VoiceSessionRestoreSync';
import {getScreenShareCaptureDiagnosticSnapshot} from '@app/features/voice/engine/ScreenShareCaptureDiagnostics';
import ScreenShareCodecNegotiation, {
buildLocalCodecAdvertisements,
getScreenShareCodecPreferenceOrder,
} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import ScreenShareCodecNegotiation from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import ScreenSharePublicationMigration from '@app/features/voice/engine/ScreenSharePublicationMigration';
import {Store, useStoreVersion} from '@app/features/voice/engine/Store';
import {shouldMoveToAfkOnTick} from '@app/features/voice/engine/VoiceAfkTracking';
@@ -118,7 +114,7 @@ import {
createVoiceEngineV2AppControllerHost,
type VoiceEngineV2AppControllerHost,
} from '@app/features/voice/engine/v2/VoiceEngineV2AppControllerHost';
import voiceEngineV2AppDebugLoggingHostAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppDebugLoggingHostAdapter';
import voiceEngineV2AppDebugEventSinkHostAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppDebugEventSinkHostAdapter';
import {createVoiceEngineV2AppEventLogSpillLoggerSink} from '@app/features/voice/engine/v2/VoiceEngineV2AppEventLogSpillLoggerSink';
import {createVoiceEngineV2AppIngestionPort} from '@app/features/voice/engine/v2/VoiceEngineV2AppHostPorts';
import type {VoiceEngineV2AppLifecycleDisposable} from '@app/features/voice/engine/v2/VoiceEngineV2AppLifecycleAdapter';
@@ -153,18 +149,9 @@ import VoiceCallLayout from '@app/features/voice/state/VoiceCallLayout';
import VoiceRegionTeleport from '@app/features/voice/state/VoiceRegionTeleport';
import VoiceSessionRestore, {type VoiceSessionRestoreSnapshot} from '@app/features/voice/state/VoiceSessionRestore';
import VoiceSettings from '@app/features/voice/state/VoiceSettings';
import {type CodecPreference, getCodecCapabilityReport} from '@app/features/voice/utils/CodecCapabilityDetector';
import {getGpuEncoderReportSync, loadGpuEncoderReport} from '@app/features/voice/utils/GpuEncoderCapabilities';
import {
getNativeAudioCaptureDiagnosticState,
setNativeAudioCaptureBridgeLifecycleBridge,
} from '@app/features/voice/utils/NativeAudioCaptureBridge';
import {getOpenH264StatusSync, loadOpenH264Status} from '@app/features/voice/utils/OpenH264Status';
import type {CodecPreference} from '@app/features/voice/utils/CodecCapabilityDetector';
import {setNativeAudioCaptureBridgeLifecycleBridge} from '@app/features/voice/utils/NativeAudioCaptureBridge';
import {areOrderedStringArraysEqual} from '@app/features/voice/utils/StringArrayUtils';
import {
getVideoDecoderExclusionsSync,
loadVideoDecoderExclusions,
} from '@app/features/voice/utils/VideoDecoderCapabilities';
import {buildVoiceParticipantIdentity} from '@app/features/voice/utils/VoiceParticipantIdentity';
import {
isVoiceServerMuteActive,
@@ -212,8 +199,6 @@ import {
import {makeObservable, observable} from 'mobx';
const logger = new Logger('MediaEngineFacade');
const MAX_DIAGNOSTIC_LINUX_AUDIO_TARGETS = 200;
const MAX_DIAGNOSTIC_PIPEWIRE_GRAPH_RECORDS = 500;
function isCameraPermissionDeniedCommandFailure(error: unknown): boolean {
if (!(error instanceof Error)) return false;
@@ -318,137 +303,6 @@ function buildRoomEventDependencies(facade: {
};
}
function voiceDiagnosticError(error: unknown): Record<string, unknown> {
if (error instanceof Error) {
return {
name: error.name,
message: error.message,
stack: error.stack,
};
}
return {message: String(error)};
}
function limitLinuxAudioTargets(result: unknown): unknown {
if (!result || typeof result !== 'object' || !('targets' in result)) return result;
const record = result as {targets?: unknown};
if (!Array.isArray(record.targets)) return result;
return {
...result,
targets: record.targets.slice(0, MAX_DIAGNOSTIC_LINUX_AUDIO_TARGETS),
targetCount: record.targets.length,
truncated: record.targets.length > MAX_DIAGNOSTIC_LINUX_AUDIO_TARGETS,
};
}
function limitGraphArray(record: Record<string, unknown>, key: string): Record<string, unknown> {
const value = record[key];
if (!Array.isArray(value)) return record;
return {
...record,
[key]: value.slice(0, MAX_DIAGNOSTIC_PIPEWIRE_GRAPH_RECORDS),
[`${key}Count`]: value.length,
[`${key}Truncated`]: value.length > MAX_DIAGNOSTIC_PIPEWIRE_GRAPH_RECORDS,
};
}
function limitPipewireRoutingGraph(graph: unknown): unknown {
if (!graph || typeof graph !== 'object') return graph;
let limited = graph as Record<string, unknown>;
limited = limitGraphArray(limited, 'nodes');
limited = limitGraphArray(limited, 'ports');
limited = limitGraphArray(limited, 'ownedLinks');
return limited;
}
function limitPipewireRoutingGraphResult(result: unknown): unknown {
if (!result || typeof result !== 'object') return result;
const record = result as Record<string, unknown>;
if ('graph' in record) {
return {...record, graph: limitPipewireRoutingGraph(record.graph)};
}
if (Array.isArray(record.graphs)) {
return {
...record,
graphs: record.graphs.map((entry) =>
entry && typeof entry === 'object'
? {...entry, graph: limitPipewireRoutingGraph((entry as Record<string, unknown>).graph)}
: entry,
),
};
}
return result;
}
function hasPipewireRoutingGraph(result: unknown): boolean {
if (!result || typeof result !== 'object') return false;
const record = result as Record<string, unknown>;
if (record.ok !== true) return false;
if (record.graph && typeof record.graph === 'object') return true;
return (
Array.isArray(record.graphs) &&
record.graphs.some((entry) => {
if (!entry || typeof entry !== 'object') return false;
const graph = (entry as Record<string, unknown>).graph;
return Boolean(graph && typeof graph === 'object');
})
);
}
async function createVoiceAudioDiagnosticsSnapshot(): Promise<Record<string, unknown>> {
const electron = getElectronAPI();
const [
nativeAudioAvailability,
nativeAudioApplications,
nativeAudioRoutingGraph,
virtmicAvailability,
virtmicTargets,
virtmicRoutingGraph,
] = await Promise.all([
electron?.nativeAudio
? electron.nativeAudio.getAvailability().catch((error) => ({error: voiceDiagnosticError(error)}))
: Promise.resolve(null),
electron?.nativeAudio
? electron.nativeAudio.listAudibleApplications().catch((error) => ({error: voiceDiagnosticError(error)}))
: Promise.resolve(null),
electron?.nativeAudio
? electron.nativeAudio.getRoutingGraph().catch((error) => ({error: voiceDiagnosticError(error)}))
: Promise.resolve(null),
electron?.virtmic
? electron.virtmic.getAvailability().catch((error) => ({error: voiceDiagnosticError(error)}))
: Promise.resolve(null),
electron?.virtmic
? electron.virtmic.listTargets({granular: true}).catch((error) => ({error: voiceDiagnosticError(error)}))
: Promise.resolve(null),
electron?.virtmic
? electron.virtmic.getRoutingGraph().catch((error) => ({error: voiceDiagnosticError(error)}))
: Promise.resolve(null),
]);
return {
platform: electron?.platform ?? null,
nativeAudio: {
availability: nativeAudioAvailability,
audibleApplications: nativeAudioApplications,
capture: getNativeAudioCaptureDiagnosticState(),
pipewireRoutingGraph: limitPipewireRoutingGraphResult(nativeAudioRoutingGraph),
},
linuxRouting: {
virtmicAvailability,
pipewireNodeInventory: limitLinuxAudioTargets(virtmicTargets),
pipewireRoutingGraph: limitPipewireRoutingGraphResult(virtmicRoutingGraph),
activeLinkGraphExported:
hasPipewireRoutingGraph(virtmicRoutingGraph) || hasPipewireRoutingGraph(nativeAudioRoutingGraph),
routingSettings: {
screenShareAudioSourceMode: VoiceSettings.getScreenShareAudioSourceMode(),
screenShareAudioIncludeSources: VoiceSettings.getScreenShareAudioIncludeSources(),
screenShareAudioExcludeSources: VoiceSettings.getScreenShareAudioExcludeSources(),
shareAppAudio: VoiceSettings.getShareAppAudio(),
shareDesktopAudio: VoiceSettings.getShareDesktopAudio(),
},
},
};
}
interface ConnectToVoiceChannelOptions {
skipChannelGate?: boolean;
deferNavigationUntilConnected?: boolean;
@@ -656,7 +510,6 @@ class MediaEngineFacade extends Store {
voiceEngineV2AppScreenShareExecutionAdapter.subscribe(forwardChange);
VoiceEngineV2AppSubscriptionAdapter.subscribe(forwardChange);
VoiceEngineV2AppPermissionAdapter.subscribe(forwardChange);
voiceEngineV2AppDebugLoggingHostAdapter.subscribe(forwardChange);
ScreenSharePublicationMigration.subscribe(forwardChange);
}
@@ -817,18 +670,6 @@ class MediaEngineFacade extends Store {
return voiceEngineV2AppConnectionHostAdapter.voiceServerEndpoint;
}
get voiceDebugLoggingActive(): boolean {
return voiceEngineV2AppDebugLoggingHostAdapter.active;
}
get voiceDebugLoggingToggleInFlight(): boolean {
return voiceEngineV2AppDebugLoggingHostAdapter.toggleInFlight;
}
async setVoiceDebugLoggingEnabled(enabled: boolean): Promise<void> {
await voiceEngineV2AppDebugLoggingHostAdapter.setEnabled(enabled);
}
get connectionContext(): VoiceEngineConnectionContext {
return {
guildId: this.guildId,
@@ -2522,19 +2363,18 @@ class MediaEngineFacade extends Store {
this.startAfkTracking();
const channelId = voiceEngineV2AppConnectionHostAdapter.channelId;
if (channelId) {
void voiceEngineV2AppDebugLoggingHostAdapter.start({
voiceEngineV2AppDebugEventSinkHostAdapter.start({
guildId: voiceEngineV2AppConnectionHostAdapter.guildId,
channelId,
connectionId: voiceEngineV2AppConnectionHostAdapter.connectionId,
room,
collectSnapshot: () => this.createVoiceDiagnosticsSnapshot(),
});
}
logger.info('All tracking started');
}
private stopTracking(): void {
void voiceEngineV2AppDebugLoggingHostAdapter.stop('tracking-stopped');
voiceEngineV2AppDebugEventSinkHostAdapter.stop('tracking-stopped');
this.statsHostAdapter.stopLatencyTracking();
this.statsHostAdapter.stopStatsTracking();
VoiceEngineV2AppRemoteSpeakingAdapter.clear();
@@ -2555,83 +2395,6 @@ class MediaEngineFacade extends Store {
logger.info('All tracking stopped');
}
private async createVoiceDiagnosticsSnapshot(): Promise<Record<string, unknown>> {
await Promise.allSettled([loadGpuEncoderReport(), loadOpenH264Status(), loadVideoDecoderExclusions()]);
return {
connection: {
guildId: voiceEngineV2AppConnectionHostAdapter.guildId,
channelId: voiceEngineV2AppConnectionHostAdapter.channelId,
connectionId: voiceEngineV2AppConnectionHostAdapter.connectionId,
connected: voiceEngineV2AppConnectionHostAdapter.connected,
connecting: voiceEngineV2AppConnectionHostAdapter.connecting,
reconnecting: voiceEngineV2AppConnectionHostAdapter.reconnecting,
disconnecting: voiceEngineV2AppConnectionHostAdapter.disconnecting,
voiceServerEndpoint: voiceEngineV2AppConnectionHostAdapter.voiceServerEndpoint,
regionHotSwapInProgress: voiceEngineV2AppConnectionHostAdapter.regionHotSwapInProgress,
reconnectAttempts: voiceEngineV2AppConnectionHostAdapter.reconnectAttempts,
},
localVoiceState: {
selfMute: LocalVoiceState.getSelfMute(),
selfDeaf: LocalVoiceState.getSelfDeaf(),
selfVideo: LocalVoiceState.getSelfVideo(),
selfStream: LocalVoiceState.getSelfStream(),
selfStreamAudio: LocalVoiceState.getSelfStreamAudio(),
selfStreamAudioMute: LocalVoiceState.getSelfStreamAudioMute(),
viewerStreamKeys: LocalVoiceState.getViewerStreamKeys(),
hasUserSetMute: LocalVoiceState.getHasUserSetMute(),
hasUserSetDeaf: LocalVoiceState.getHasUserSetDeaf(),
},
effectiveAudioState: getEffectiveAudioState(),
connectionVoiceStates: voiceEngineV2AppVoiceStateAdapter.getConnectionVoiceStates(),
participants: Object.values(this.voiceEngineV2Participants.participants),
stats: {
currentLatency: this.currentLatency,
averageLatency: this.averageLatency,
displayLatency: this.displayLatency,
estimatedLatency: this.estimatedLatency,
reconnectionCount: this.reconnectionCount,
voiceStats: this.voiceStats,
perTrackStats: this.perTrackStats,
statsTimeSeriesTail: this.statsTimeSeries.slice(-30),
latencyHistoryTail: this.latencyHistory.slice(-30),
publisherTransport: this.publisherTransport,
subscriberTransport: this.subscriberTransport,
},
screenShare: {
selectedCodec: ScreenShareCodecNegotiation.getSelectedCodec(),
preferenceOrder: getScreenShareCodecPreferenceOrder(),
capture: await getScreenShareCaptureDiagnosticSnapshot(),
audioCapture: await createVoiceAudioDiagnosticsSnapshot(),
codecCapabilities: {
localAdvertisements: buildLocalCodecAdvertisements(),
report: getCodecCapabilityReport(),
gpuEncoderReport: getGpuEncoderReportSync(),
openH264Status: getOpenH264StatusSync(),
decoderExclusions: getVideoDecoderExclusionsSync(),
},
adaptiveQuality: AdaptiveScreenShareEngine.qualitySnapshot,
settings: {
preferredScreenShareCodec: VoiceSettings.getPreferredScreenShareCodec(),
screenShareEncoderMode: VoiceSettings.getScreenShareEncoderMode(),
screenShareSoftwareQuality: VoiceSettings.getScreenShareSoftwareQuality(),
screenShareScalabilityMode: VoiceSettings.getScreenShareScalabilityMode(),
screenShareBackupCodecMode: VoiceSettings.getScreenShareBackupCodecMode(),
screenShareMaxBitrateMbps: VoiceSettings.getScreenShareMaxBitrateMbps(),
adaptiveScreenShareQuality: VoiceSettings.getAdaptiveScreenShareQuality(),
screenshareResolution: VoiceSettings.getScreenshareResolution(),
videoFrameRate: VoiceSettings.getVideoFrameRate(),
streamingMode: VoiceSettings.getStreamingMode(),
shareAppAudio: VoiceSettings.getShareAppAudio(),
shareDesktopAudio: VoiceSettings.getShareDesktopAudio(),
shareDeviceAudio: VoiceSettings.getShareDeviceAudio(),
screenShareAudioSourceMode: VoiceSettings.getScreenShareAudioSourceMode(),
screenShareAudioIncludeSources: VoiceSettings.getScreenShareAudioIncludeSources(),
screenShareAudioExcludeSources: VoiceSettings.getScreenShareAudioExcludeSources(),
},
},
};
}
private abortVoiceConnection(): void {
this.transitionFacadeState({type: 'screenShareReconnect.clear'});
voiceEngineV2AppConnectionHostAdapter.abortConnection();
@@ -1,226 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {getElectronAPI} from '@app/features/ui/utils/NativeUtils';
import type {
NativeScreenCaptureAvailability,
NativeScreenCaptureDiagnostics,
NativeScreenCaptureSourceKind,
} from '@app/types/electron.d';
export type ScreenShareCaptureMethod =
| 'display-media'
| 'native-screen-capture'
| 'native-voice-engine-screen-capture'
| 'device-media';
interface ScreenShareCaptureState {
active: boolean;
generation: number;
method: ScreenShareCaptureMethod;
startedAtMs: number;
updatedAtMs: number;
endedAtMs?: number;
endReason?: string;
platform?: string | null;
sourceId?: string | null;
desktopSourceId?: string | null;
sourceKind?: NativeScreenCaptureSourceKind | null;
captureId?: string | null;
displayShareEnvironment?: string | null;
displayMediaSettings?: Record<string, unknown> | null;
device?: {
videoDeviceId?: string;
audioDeviceId?: string;
};
nativeAvailability?: NativeScreenCaptureAvailability | null;
nativeDiagnostics?: NativeScreenCaptureDiagnostics | null;
}
interface CaptureStateUpdate {
method: ScreenShareCaptureMethod;
sourceId?: string | null;
desktopSourceId?: string | null;
sourceKind?: NativeScreenCaptureSourceKind | null;
captureId?: string | null;
displayShareEnvironment?: string | null;
displayMediaSettings?: Record<string, unknown> | null;
device?: {
videoDeviceId?: string;
audioDeviceId?: string;
};
nativeAvailability?: NativeScreenCaptureAvailability | null;
nativeDiagnostics?: NativeScreenCaptureDiagnostics | null;
}
interface DisplayMediaTrackSettingsUpdate {
sourceId?: string | null;
desktopSourceId?: string | null;
displayShareEnvironment?: string | null;
}
interface MediaTrackSettingsReader {
getSettings(): MediaTrackSettings;
}
const MAX_CAPTURE_HISTORY = 20;
let activeCapture: ScreenShareCaptureState | null = null;
let lastCapture: ScreenShareCaptureState | null = null;
let captureGeneration = 0;
const captureHistory: Array<ScreenShareCaptureState | null> = new Array(MAX_CAPTURE_HISTORY).fill(null);
let captureHistoryHead = 0;
let captureHistoryLength = 0;
function nowMs(): number {
return Date.now();
}
function readPlatform(): string | null {
return getElectronAPI()?.platform ?? null;
}
function cloneCaptureState(state: ScreenShareCaptureState | null): ScreenShareCaptureState | null {
if (!state) return null;
const cloned = {...state};
if (state.device) {
cloned.device = {...state.device};
}
return cloned;
}
function cloneCaptureHistoryTail(limit: number): Array<ScreenShareCaptureState | null> {
const tailCount = Math.min(captureHistoryLength, limit);
const tail = new Array<ScreenShareCaptureState | null>(tailCount);
const startIndex = captureHistoryLength - tailCount;
for (let i = 0; i < tailCount; i += 1) {
const index = (captureHistoryHead + startIndex + i) % MAX_CAPTURE_HISTORY;
tail[i] = cloneCaptureState(captureHistory[index] ?? null);
}
return tail;
}
function pushHistory(state: ScreenShareCaptureState): void {
const cloned = cloneCaptureState(state);
if (!cloned) return;
if (captureHistoryLength < MAX_CAPTURE_HISTORY) {
const index = (captureHistoryHead + captureHistoryLength) % MAX_CAPTURE_HISTORY;
captureHistory[index] = cloned;
captureHistoryLength += 1;
return;
}
captureHistory[captureHistoryHead] = cloned;
captureHistoryHead = (captureHistoryHead + 1) % MAX_CAPTURE_HISTORY;
}
export function markScreenShareCaptureActive(update: CaptureStateUpdate): void {
const currentTime = nowMs();
const canUpdateCurrent =
activeCapture != null &&
activeCapture.method === update.method &&
(update.captureId == null || activeCapture.captureId === update.captureId);
const base = canUpdateCurrent ? activeCapture : null;
const device = update.device ?? base?.device;
const nextCapture: ScreenShareCaptureState = {
active: true,
generation: base?.generation ?? ++captureGeneration,
method: update.method,
startedAtMs: base?.startedAtMs ?? currentTime,
updatedAtMs: currentTime,
platform: readPlatform(),
sourceId: update.sourceId ?? base?.sourceId ?? null,
desktopSourceId: update.desktopSourceId ?? base?.desktopSourceId ?? null,
sourceKind: update.sourceKind ?? base?.sourceKind ?? null,
captureId: update.captureId ?? base?.captureId ?? null,
displayShareEnvironment: update.displayShareEnvironment ?? base?.displayShareEnvironment ?? null,
displayMediaSettings: update.displayMediaSettings ?? base?.displayMediaSettings ?? null,
nativeAvailability: update.nativeAvailability ?? base?.nativeAvailability ?? null,
nativeDiagnostics: update.nativeDiagnostics ?? base?.nativeDiagnostics ?? null,
};
if (device) {
nextCapture.device = {...device};
}
activeCapture = nextCapture;
}
export function updateScreenShareDisplayMediaSettings(
track: MediaTrackSettingsReader,
extra?: DisplayMediaTrackSettingsUpdate,
): void {
const settings = track.getSettings();
markScreenShareCaptureActive({
method: 'display-media',
...extra,
displayMediaSettings: {
width: settings.width,
height: settings.height,
frameRate: settings.frameRate,
aspectRatio: settings.aspectRatio,
deviceId: settings.deviceId,
displaySurface: (settings as MediaTrackSettings & {displaySurface?: string}).displaySurface,
cursor: (settings as MediaTrackSettings & {cursor?: string}).cursor,
logicalSurface: (settings as MediaTrackSettings & {logicalSurface?: boolean}).logicalSurface,
},
});
}
export function markScreenShareCaptureEnded(reason: string): void {
if (!activeCapture) return;
const ended = {
...activeCapture,
active: false,
updatedAtMs: nowMs(),
endedAtMs: nowMs(),
endReason: reason,
};
lastCapture = ended;
pushHistory(ended);
activeCapture = null;
}
function inferWindowsCaptureMethod(
capture: ScreenShareCaptureState | null,
diagnostics: NativeScreenCaptureDiagnostics | null,
): string | null {
if (!capture || capture.platform !== 'win32') return null;
if (capture.method === 'display-media') return 'chromium-getDisplayMedia-desktopCapturer';
if (capture.method === 'device-media') return 'getUserMedia-device';
if (capture.method !== 'native-screen-capture' && capture.method !== 'native-voice-engine-screen-capture') {
return null;
}
if (diagnostics?.activeStrategy) return diagnostics.activeStrategy;
if (capture.sourceKind === 'game') return 'wgc';
if (capture.sourceKind === 'screen') return 'wgc';
if (capture.sourceKind === 'window') return 'dxgi-duplication';
return 'native-screen-capture';
}
async function readNativeAvailability(): Promise<NativeScreenCaptureAvailability | null> {
const api = getElectronAPI()?.nativeScreenCapture;
if (!api) return null;
return api.getAvailability().catch(() => null);
}
async function readNativeDiagnostics(
captureId: string | null | undefined,
): Promise<NativeScreenCaptureDiagnostics | null> {
const api = getElectronAPI()?.nativeScreenCapture;
if (!api || !captureId) return null;
return api.getDiagnostics(captureId).catch(() => null);
}
export async function getScreenShareCaptureDiagnosticSnapshot(): Promise<Record<string, unknown>> {
const active = cloneCaptureState(activeCapture);
const nativeAvailability = await readNativeAvailability();
const nativeDiagnostics = await readNativeDiagnostics(active?.captureId);
if (active) {
active.nativeAvailability = nativeAvailability ?? active.nativeAvailability ?? null;
active.nativeDiagnostics = nativeDiagnostics ?? active.nativeDiagnostics ?? null;
}
return {
active,
last: cloneCaptureState(lastCapture),
historyTail: cloneCaptureHistoryTail(5),
nativeAvailability,
windowsCaptureMethod: inferWindowsCaptureMethod(active, nativeDiagnostics ?? active?.nativeDiagnostics ?? null),
};
}
@@ -14,7 +14,6 @@ import {
hasAnyTerminalTransport,
isMutedOrDeafened,
isPermissionDeniedError,
isPresenceConnectionReady,
isReadyToRepublishTrack,
} from './VoiceEngineV2AppAdapterAssertions';
@@ -188,26 +187,6 @@ describe('VoiceEngineV2AppAdapterAssertions', () => {
});
});
describe('isPresenceConnectionReady', () => {
it('returns true when connected and ids are present', () => {
expect(isPresenceConnectionReady(true, 'channel', 'connection')).toBe(true);
});
it('returns false when disconnected', () => {
expect(isPresenceConnectionReady(false, 'channel', 'connection')).toBe(false);
});
it('returns false on missing channelId', () => {
expect(isPresenceConnectionReady(true, null, 'connection')).toBe(false);
expect(isPresenceConnectionReady(true, '', 'connection')).toBe(false);
});
it('returns false on missing connectionId', () => {
expect(isPresenceConnectionReady(true, 'channel', null)).toBe(false);
expect(isPresenceConnectionReady(true, 'channel', '')).toBe(false);
});
});
describe('hasAnyTerminalTransport', () => {
it('returns true when any slot is set', () => {
const sentinel = {};
@@ -144,17 +144,6 @@ export function assertDisconnectReason(
assert.fail(`${fieldName} must be one of 'user' | 'error' | 'server'`);
}
export function isPresenceConnectionReady(
connected: boolean,
channelId: string | null,
connectionId: string | null,
): boolean {
if (!connected) return false;
if (channelId === null || channelId.length === 0) return false;
if (connectionId === null || connectionId.length === 0) return false;
return true;
}
export interface TerminalTransportSnapshot {
readonly current: {readonly room?: unknown};
readonly hotSwap: {readonly pendingRoom?: unknown; readonly previousRoom?: unknown};
@@ -3,7 +3,6 @@
import assert from 'node:assert/strict';
import {isElectronPlatform} from '@app/features/platform/types/Platform';
import {Logger} from '@app/features/platform/utils/AppLogger';
import * as VoicePresenceHeartbeatCommands from '@app/features/voice/commands/VoicePresenceHeartbeatCommands';
import {Store} from '@app/features/voice/engine/Store';
import {sendVoiceStateDisconnect} from '@app/features/voice/engine/VoiceChannelConnector';
import {
@@ -35,7 +34,6 @@ import {
assertOptionalNonEmptyString,
assertVoiceServerUpdateShape,
hasAnyTerminalTransport,
isPresenceConnectionReady,
isReadyToRepublishTrack,
} from '@app/features/voice/engine/v2/VoiceEngineV2AppAdapterAssertions';
import {VoiceEngineV2AppReconnectPolicy} from '@app/features/voice/engine/v2/VoiceEngineV2AppReconnectPolicy';
@@ -60,7 +58,6 @@ import {timer} from 'rxjs';
const logger = new Logger('VoiceEngineV2AppConnectionHostAdapter');
const VOICE_SERVER_TIMEOUT_MS = 5000;
const VIDEO_DECODER_EXCLUSION_TIMEOUT_MS = 500;
const VOICE_PRESENCE_HEARTBEAT_INTERVAL_MS = 15000;
export interface VoiceServerUpdateData {
token: string;
@@ -133,19 +130,6 @@ async function getRoomVideoDecoderExclusions(): Promise<RoomOptions['subscriberV
}
}
function isSamePresenceConnection(
activeSub: Subscription | null,
activeConnection: {channelId: string; connectionId: string} | null,
channelId: string,
connectionId: string,
): boolean {
if (!activeSub) return false;
if (!activeConnection) return false;
if (activeConnection.channelId !== channelId) return false;
if (activeConnection.connectionId !== connectionId) return false;
return true;
}
function createWebAudioMixOption(): RoomOptions['webAudioMix'] {
const audioContext = getSharedVoiceAudioContext();
if (audioContext) {
@@ -210,8 +194,6 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
private reconnect = new VoiceEngineV2AppReconnectPolicy();
private voiceServerTimeoutSub: Subscription | null = null;
private hotSwapTimeoutSub: Subscription | null = null;
private voicePresenceHeartbeatSub: Subscription | null = null;
private voicePresenceHeartbeatConnection: {channelId: string; connectionId: string} | null = null;
private isLocalDisconnecting = false;
private hotSwapOperationQueue: Array<HotSwapQueuedOperation> = [];
@@ -338,71 +320,6 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
});
}
private startVoicePresenceHeartbeatForCurrentConnection(): void {
const {channelId, connectionId, connected} = this.connectionState;
if (!isPresenceConnectionReady(connected, channelId, connectionId)) {
this.stopVoicePresenceHeartbeat();
return;
}
const presenceChannelId = channelId as string;
const presenceConnectionId = connectionId as string;
if (
isSamePresenceConnection(
this.voicePresenceHeartbeatSub,
this.voicePresenceHeartbeatConnection,
presenceChannelId,
presenceConnectionId,
)
) {
return;
}
this.stopVoicePresenceHeartbeat();
this.voicePresenceHeartbeatConnection = {channelId: presenceChannelId, connectionId: presenceConnectionId};
this.voicePresenceHeartbeatSub = timer(0, VOICE_PRESENCE_HEARTBEAT_INTERVAL_MS).subscribe(() => {
void this.sendVoicePresenceHeartbeat(presenceChannelId, presenceConnectionId, {requireConnected: true});
});
}
private stopVoicePresenceHeartbeat(options: {markEnded?: boolean} = {}): void {
const connection = this.voicePresenceHeartbeatConnection;
if (this.voicePresenceHeartbeatSub) {
this.voicePresenceHeartbeatSub.unsubscribe();
this.voicePresenceHeartbeatSub = null;
}
this.voicePresenceHeartbeatConnection = null;
if (options.markEnded && connection) {
void this.markVoicePresenceHeartbeatEnded(connection);
}
}
private async sendVoicePresenceHeartbeat(
channelId: string,
connectionId: string,
options: {requireConnected: boolean},
): Promise<void> {
const current = this.connectionState;
if (
current.channelId !== channelId ||
current.connectionId !== connectionId ||
(options.requireConnected && !current.connected)
) {
return;
}
try {
await VoicePresenceHeartbeatCommands.heartbeat({channelId, connectionId});
} catch (error) {
logger.warn('Voice presence heartbeat failed', {channelId, connectionId, error});
}
}
private async markVoicePresenceHeartbeatEnded(connection: {channelId: string; connectionId: string}): Promise<void> {
try {
await VoicePresenceHeartbeatCommands.end(connection);
} catch (error) {
logger.warn('Voice presence heartbeat end failed', {...connection, error});
}
}
get lastConnectedChannel(): {
guildId: string;
channelId: string;
@@ -637,7 +554,6 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
this.update(() => {
this.transitionConnection({type: 'connection.failed', reason: 'error'});
});
this.stopVoicePresenceHeartbeat({markEnded: true});
this.throttle.setInFlightConnect(false);
this.reconnect.setReconnectState('error');
void onConnectFailed?.(guildId, resolvedChannelId, connectionId, attemptId, error ?? new Error(message));
@@ -675,7 +591,6 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
} catch (error) {
logger.warn('Failed to disconnect stale room', error);
}
this.stopVoicePresenceHeartbeat({markEnded: true});
return;
}
logger.info('Initializing voice connection');
@@ -691,7 +606,6 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
this.update(() => {
this.transitionConnection({type: 'connection.failed', reason: 'error'});
});
this.stopVoicePresenceHeartbeat({markEnded: true});
this.throttle.setInFlightConnect(false);
this.reconnect.setReconnectState('error');
void onConnectFailed?.(guildId, resolvedChannelId, connectionId, attemptId, error);
@@ -834,7 +748,6 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
this.transitionConnection({type: 'hotSwap.complete', room: newRoom, endpoint, connectionId});
});
this.clearHotSwapTimeout();
this.startVoicePresenceHeartbeatForCurrentConnection();
onHotSwapComplete?.(newRoom, attemptId, guildId, channelId);
await this.drainHotSwapQueue();
try {
@@ -960,7 +873,6 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
this.reconnect.setLastConnectedChannel(guildId, channelId);
this.throttle.setInFlightConnect(false);
this.reconnect.resetOnConnection();
this.startVoicePresenceHeartbeatForCurrentConnection();
assert.ok(this.connectionState.connected, 'markConnected post-condition: connection state reflects connected');
logger.info('Connection established');
}
@@ -973,7 +885,6 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
this.invalidateThrottleAttempt();
this.throttle.setInFlightConnect(false);
this.reconnect.setReconnectState(reason);
this.stopVoicePresenceHeartbeat({markEnded: true});
logger.info('Connection terminated', {reason});
}
@@ -991,7 +902,6 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
this.transitionConnection({type: 'connection.reconnected'});
});
this.reconnect.resetOnConnection();
this.startVoicePresenceHeartbeatForCurrentConnection();
logger.info('Connection reconnected');
}
@@ -1013,7 +923,6 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
});
this.invalidateThrottleAttempt();
this.reconnect.setReconnectState(reason);
this.stopVoicePresenceHeartbeat({markEnded: true});
this.update(() => {
this.isLocalDisconnecting = false;
});
@@ -1034,7 +943,6 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
this.transitionConnection({type: 'connection.disconnectForChannelMove'});
});
this.invalidateThrottleAttempt();
this.stopVoicePresenceHeartbeat({markEnded: true});
logger.info('Disconnected for channel move (preserving connectionId)');
}
@@ -1070,7 +978,6 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
this.disconnectRoomForTerminalUnload(previousRoom, 'previous-hot-swap');
}
this.disconnectRoomForTerminalUnload(room, 'current');
this.stopVoicePresenceHeartbeat({markEnded: true});
this.throttle.setInFlightConnect(false);
this.update(() => {
this.isLocalDisconnecting = false;
@@ -1188,7 +1095,6 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
this.transitionConnection({type: 'connection.reset'});
});
this.invalidateThrottleAttempt();
this.stopVoicePresenceHeartbeat({markEnded: true});
this.throttle.setInFlightConnect(false);
}
@@ -1206,7 +1112,6 @@ export class VoiceEngineV2AppConnectionHostAdapter extends Store {
this.transitionConnection({type: 'connection.abort'});
});
this.invalidateThrottleAttempt();
this.stopVoicePresenceHeartbeat({markEnded: true});
this.throttle.setInFlightConnect(false);
logger.info('Connection aborted due to gateway error');
}
@@ -0,0 +1,545 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import assert from 'node:assert/strict';
import {Logger} from '@app/features/platform/utils/AppLogger';
import {getElectronAPI} from '@app/features/ui/utils/NativeUtils';
import {
appendBrowserVoiceDebugEventSinkEntries,
openBrowserVoiceDebugEventSinkPopout,
} from '@app/features/voice/diagnostics/VoiceDebugBrowserEventSinkPopout';
import {asVoiceTrackSource, VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource';
import type {DesktopVoiceDebugEventSinkEntry} from '@app/types/electron.d';
import type {
LocalTrackPublication,
Participant,
RemoteTrack,
RemoteTrackPublication,
Room,
TrackPublication,
} from 'livekit-client';
import {RoomEvent} from 'livekit-client';
import {assertNonNullObject, assertString} from './VoiceEngineV2AppAdapterAssertions';
const logger = new Logger('VoiceEngineV2AppDebugEventSinkHostAdapter');
export const VOICE_ENGINE_V2_APP_DEBUG_EVENT_SINK_MAX_ENTRIES = 1000;
export const VOICE_ENGINE_V2_APP_DEBUG_EVENT_SINK_MAX_LINE_CHARS = 262_144;
const SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC = 'fluxer.rtc.codec-negotiation.v1';
const TEXT_DECODER = new TextDecoder();
interface VoiceDebugEventSinkEvent {
type: string;
timestamp_ns: string;
monotonic_ns?: string;
data?: Record<string, unknown>;
}
class BoundedRing<T> {
private readonly items: Array<T | null>;
private readonly capacity: number;
private readonly label: string;
private head = 0;
private tail = 0;
private count = 0;
constructor(capacity: number, label: string) {
assert.ok(capacity > 0, `${label} capacity must be positive`);
assertString(label, 'label');
this.capacity = capacity;
this.label = label;
this.items = new Array(capacity).fill(null);
}
get length(): number {
return this.count;
}
pushDropOldest(item: T): T | null {
let dropped: T | null = null;
if (this.count >= this.capacity) {
dropped = this.popFront();
}
this.pushBack(item);
assert.ok(this.count <= this.capacity, `${this.label} must stay bounded`);
return dropped;
}
toArray(): Array<T> {
const out: Array<T> = [];
for (let index = 0; index < this.count; index += 1) {
const slot = (this.head + index) % this.capacity;
const item = this.items[slot];
if (item === null) {
assert.fail(`${this.label} slot must exist`);
}
out.push(item);
}
return out;
}
private pushBack(item: T): void {
assert.ok(this.count < this.capacity, `${this.label} must have free capacity before push`);
this.items[this.tail] = item;
this.tail = (this.tail + 1) % this.capacity;
this.count += 1;
}
private popFront(): T | null {
if (this.count === 0) return null;
const item = this.items[this.head];
assert.notEqual(item, null, `${this.label} front slot must exist`);
this.items[this.head] = null;
this.head = (this.head + 1) % this.capacity;
this.count -= 1;
return item;
}
}
type VoiceDebugEventSinkRoomEventHandler = (...args: Array<never>) => void;
type VoiceDebugEventSinkRoomEventBinding = [RoomEvent, VoiceDebugEventSinkRoomEventHandler];
interface VoiceDebugEventSinkStartOptions {
guildId: string | null;
channelId: string;
connectionId: string | null;
room: Room;
}
interface RoomParticipantSummary {
identity: string;
sid: string;
isLocal: boolean;
name?: string;
metadata?: string;
attributes?: Record<string, string>;
connectionQuality?: string;
isSpeaking?: boolean;
permissions?: unknown;
trackPublications: Array<TrackPublicationSummary>;
}
interface TrackPublicationSummary {
trackSid: string;
trackName?: string;
source?: string;
kind?: string;
mimeType?: string;
isMuted?: boolean;
isSubscribed?: boolean;
isEnabled?: boolean;
dimensions?: {
width: number;
height: number;
};
}
function millisecondsToNanosecondsString(milliseconds: number): string {
if (!Number.isFinite(milliseconds) || milliseconds < 0) return '0';
const wholeMs = Math.trunc(milliseconds);
const fractionalNs = Math.round((milliseconds - wholeMs) * 1000000);
return (BigInt(wholeMs) * 1000000n + BigInt(fractionalNs)).toString();
}
function createDiagnosticEvent(type: string, data?: Record<string, unknown>): VoiceDebugEventSinkEvent {
const monotonicNow = typeof performance !== 'undefined' ? performance.now() : Date.now();
const timeOrigin =
typeof performance !== 'undefined' && Number.isFinite(performance.timeOrigin)
? performance.timeOrigin
: Date.now() - monotonicNow;
return {
type,
timestamp_ns: millisecondsToNanosecondsString(timeOrigin + monotonicNow),
monotonic_ns: millisecondsToNanosecondsString(monotonicNow),
...(data ? {data} : {}),
};
}
function errorToData(error: unknown): Record<string, unknown> {
if (error instanceof Error) {
return {
name: error.name,
message: error.message,
stack: error.stack,
};
}
return {message: String(error)};
}
function truncateEventSinkLine(line: string): string {
assertString(line, 'event sink line');
if (line.length <= VOICE_ENGINE_V2_APP_DEBUG_EVENT_SINK_MAX_LINE_CHARS) return line;
const omittedChars = line.length - VOICE_ENGINE_V2_APP_DEBUG_EVENT_SINK_MAX_LINE_CHARS;
return `${line.slice(0, VOICE_ENGINE_V2_APP_DEBUG_EVENT_SINK_MAX_LINE_CHARS)}... [truncated ${omittedChars} chars]`;
}
function stringifyEventSinkEntry(sequence: number, event: VoiceDebugEventSinkEvent): string {
assert.ok(Number.isSafeInteger(sequence), 'event sink sequence must be a safe integer');
assert.ok(sequence >= 1, 'event sink sequence must be >= 1');
try {
return truncateEventSinkLine(JSON.stringify({sequence, ...event}));
} catch (error) {
return truncateEventSinkLine(
JSON.stringify({
sequence,
type: event.type,
timestamp_ns: event.timestamp_ns,
monotonic_ns: event.monotonic_ns,
stringifyError: errorToData(error),
}),
);
}
}
function createEventSinkEntry(sequence: number, event: VoiceDebugEventSinkEvent): DesktopVoiceDebugEventSinkEntry {
assert.ok(event !== null && typeof event === 'object', 'event sink event must be an object');
assertString(event.type, 'event sink event type');
assertString(event.timestamp_ns, 'event sink event timestamp');
return {
sequence,
line: stringifyEventSinkEntry(sequence, event),
};
}
function getTrackDimensions(publication: TrackPublication): TrackPublicationSummary['dimensions'] | undefined {
const track = publication.track;
if (!track || !('dimensions' in track)) return undefined;
const dimensions = (track as {dimensions?: {width?: number; height?: number}}).dimensions;
if (typeof dimensions?.width !== 'number' || typeof dimensions.height !== 'number') return undefined;
return {
width: dimensions.width,
height: dimensions.height,
};
}
function summarizePublication(publication: TrackPublication): TrackPublicationSummary {
return {
trackSid: publication.trackSid,
trackName: publication.trackName,
source: publication.source,
kind: publication.kind,
mimeType: publication.mimeType,
isMuted: publication.isMuted,
isSubscribed: 'isSubscribed' in publication ? Boolean(publication.isSubscribed) : undefined,
isEnabled: 'isEnabled' in publication ? Boolean(publication.isEnabled) : undefined,
dimensions: getTrackDimensions(publication),
};
}
function summarizeParticipant(participant: Participant | undefined): RoomParticipantSummary | null {
if (!participant) return null;
const trackPublications: Array<TrackPublicationSummary> = [];
participant.trackPublications.forEach((publication) => {
trackPublications.push(summarizePublication(publication));
});
return {
identity: participant.identity,
sid: participant.sid,
isLocal: participant.isLocal,
name: participant.name,
metadata: participant.metadata,
attributes: participant.attributes,
connectionQuality: participant.connectionQuality,
isSpeaking: participant.isSpeaking,
permissions: participant.permissions,
trackPublications,
};
}
function summarizeRoom(room: Room | null): Record<string, unknown> | null {
if (!room) return null;
return {
name: room.name,
state: room.state,
numParticipants: room.numParticipants,
localParticipant: summarizeParticipant(room.localParticipant),
remoteParticipants: Array.from(room.remoteParticipants.values()).map((participant) =>
summarizeParticipant(participant),
),
};
}
function summarizeTrackEvent(
publication: TrackPublication | RemoteTrackPublication | LocalTrackPublication,
participant: Participant | undefined,
): Record<string, unknown> {
return {
publication: summarizePublication(publication as TrackPublication),
participant: summarizeParticipant(participant),
isScreenShare: asVoiceTrackSource(publication.source) === VoiceTrackSource.ScreenShare,
isScreenShareAudio: asVoiceTrackSource(publication.source) === VoiceTrackSource.ScreenShareAudio,
};
}
function summarizeRemoteTrack(track: RemoteTrack): Record<string, unknown> {
return {
sid: track.sid,
kind: track.kind,
source: track.source,
mediaStreamTrackId: track.mediaStreamTrack?.id,
readyState: track.mediaStreamTrack?.readyState,
muted: track.mediaStreamTrack?.muted,
};
}
function parseAllowedDataMessage(payload: Uint8Array, topic: string | undefined): Record<string, unknown> {
if (topic !== SCREEN_SHARE_CODEC_NEGOTIATION_TOPIC) {
return {
topic: topic ?? null,
payloadBytes: payload.byteLength,
decoded: null,
};
}
try {
return {
topic,
payloadBytes: payload.byteLength,
decoded: JSON.parse(TEXT_DECODER.decode(payload)) as unknown,
};
} catch (error) {
return {
topic,
payloadBytes: payload.byteLength,
decodeError: errorToData(error),
};
}
}
export class VoiceEngineV2AppDebugEventSinkHostAdapter {
private channelId: string | null = null;
private connectionId: string | null = null;
private room: Room | null = null;
private roomDisposer: (() => void) | null = null;
private readonly eventSinkEntries = new BoundedRing<DesktopVoiceDebugEventSinkEntry>(
VOICE_ENGINE_V2_APP_DEBUG_EVENT_SINK_MAX_ENTRIES,
'voice debug event sink history',
);
private eventSinkSequence = 0;
private eventSinkForwardFailureCount = 0;
getEventSinkEntries(): Array<DesktopVoiceDebugEventSinkEntry> {
assert.ok(
this.eventSinkEntries.length <= VOICE_ENGINE_V2_APP_DEBUG_EVENT_SINK_MAX_ENTRIES,
'event sink history must stay bounded',
);
return this.eventSinkEntries.toArray();
}
async openEventSinkPopout(): Promise<void> {
const electron = getElectronAPI();
const entries = this.getEventSinkEntries();
if (electron?.openVoiceDebugEventSinkPopout) {
try {
await electron.openVoiceDebugEventSinkPopout(entries);
return;
} catch (error) {
logger.warn('Failed to open voice debug event sink desktop popout', {error});
}
}
try {
const opened = await openBrowserVoiceDebugEventSinkPopout(entries);
if (!opened) {
logger.warn('Failed to open voice debug event sink browser popout');
}
} catch (error) {
logger.warn('Failed to open voice debug event sink browser popout', {error});
}
}
private isStartIdempotent(options: VoiceDebugEventSinkStartOptions): boolean {
if (this.channelId !== options.channelId) return false;
if (this.connectionId !== options.connectionId) return false;
return this.room === options.room;
}
start(options: VoiceDebugEventSinkStartOptions): void {
assertNonNullObject(options, 'options');
assertString(options.channelId, 'options.channelId');
assert.ok(options.channelId.length > 0, 'options.channelId must not be empty');
assertNonNullObject(options.room, 'options.room');
if (this.isStartIdempotent(options)) return;
this.stop('replaced');
this.channelId = options.channelId;
this.connectionId = options.connectionId;
this.room = options.room;
this.bindRoom(options.room);
this.record('voice.debug_event_sink.tracking_started', {
guildId: options.guildId,
channelId: options.channelId,
connectionId: options.connectionId,
room: summarizeRoom(options.room),
});
}
stop(reason = 'stopped'): void {
assertString(reason, 'reason');
if (this.channelId !== null) {
this.record('voice.debug_event_sink.tracking_stopped', {
reason,
channelId: this.channelId,
connectionId: this.connectionId,
room: summarizeRoom(this.room),
});
}
this.roomDisposer?.();
this.roomDisposer = null;
this.channelId = null;
this.connectionId = null;
this.room = null;
}
private buildRoomLifecycleEventBindings(room: Room): Array<VoiceDebugEventSinkRoomEventBinding> {
return [
[RoomEvent.Connected, () => this.record('livekit.room.connected', {room: summarizeRoom(room)})],
[
RoomEvent.Disconnected,
(reason?: unknown) =>
this.record('livekit.room.disconnected', {reason: String(reason ?? 'unknown'), room: summarizeRoom(room)}),
],
[RoomEvent.Reconnecting, () => this.record('livekit.room.reconnecting', {room: summarizeRoom(room)})],
[RoomEvent.Reconnected, () => this.record('livekit.room.reconnected', {room: summarizeRoom(room)})],
];
}
private buildParticipantEventBindings(): Array<VoiceDebugEventSinkRoomEventBinding> {
return [
[
RoomEvent.ParticipantConnected,
(participant: Participant) =>
this.record('livekit.participant.connected', {participant: summarizeParticipant(participant)}),
],
[
RoomEvent.ParticipantDisconnected,
(participant: Participant) =>
this.record('livekit.participant.disconnected', {participant: summarizeParticipant(participant)}),
],
[
RoomEvent.ActiveSpeakersChanged,
(speakers: Array<Participant>) =>
this.record('livekit.active_speakers.changed', {
speakers: speakers.map((participant) => summarizeParticipant(participant)),
}),
],
];
}
private buildTrackEventBindings(): Array<VoiceDebugEventSinkRoomEventBinding> {
return [
[
RoomEvent.TrackPublished,
(publication: RemoteTrackPublication, participant: Participant) =>
this.record('livekit.track.published', summarizeTrackEvent(publication, participant)),
],
[
RoomEvent.TrackUnpublished,
(publication: RemoteTrackPublication, participant: Participant) =>
this.record('livekit.track.unpublished', summarizeTrackEvent(publication, participant)),
],
[
RoomEvent.TrackSubscribed,
(track: RemoteTrack, publication: RemoteTrackPublication, participant: Participant) =>
this.record('livekit.track.subscribed', {
...summarizeTrackEvent(publication, participant),
track: summarizeRemoteTrack(track),
}),
],
[
RoomEvent.TrackUnsubscribed,
(track: RemoteTrack, publication: RemoteTrackPublication, participant: Participant) =>
this.record('livekit.track.unsubscribed', {
...summarizeTrackEvent(publication, participant),
track: summarizeRemoteTrack(track),
}),
],
[
RoomEvent.TrackMuted,
(publication: TrackPublication, participant: Participant) =>
this.record('livekit.track.muted', summarizeTrackEvent(publication, participant)),
],
[
RoomEvent.TrackUnmuted,
(publication: TrackPublication, participant: Participant) =>
this.record('livekit.track.unmuted', summarizeTrackEvent(publication, participant)),
],
[
RoomEvent.LocalTrackPublished,
(publication: LocalTrackPublication, participant: Participant) =>
this.record('livekit.local_track.published', summarizeTrackEvent(publication, participant)),
],
[
RoomEvent.LocalTrackUnpublished,
(publication: LocalTrackPublication, participant: Participant) =>
this.record('livekit.local_track.unpublished', summarizeTrackEvent(publication, participant)),
],
];
}
private buildDataEventBindings(): Array<VoiceDebugEventSinkRoomEventBinding> {
return [
[
RoomEvent.DataReceived,
(payload: Uint8Array, participant: Participant | undefined, kind: unknown, topic?: string) =>
this.record('livekit.data.received', {
participant: summarizeParticipant(participant),
kind: String(kind),
...parseAllowedDataMessage(payload, topic),
}),
],
];
}
private buildRoomEventBindings(room: Room): Array<VoiceDebugEventSinkRoomEventBinding> {
return [
...this.buildRoomLifecycleEventBindings(room),
...this.buildParticipantEventBindings(),
...this.buildTrackEventBindings(),
...this.buildDataEventBindings(),
];
}
private bindRoom(room: Room): void {
assertNonNullObject(room, 'room');
this.roomDisposer?.();
const bindings = this.buildRoomEventBindings(room);
assert.ok(bindings.length > 0, 'expected at least one room event binding');
for (const [event, handler] of bindings) {
room.on(event, handler);
}
this.roomDisposer = () => {
for (const [event, handler] of bindings) {
room.off(event, handler);
}
};
}
private record(type: string, data?: Record<string, unknown>): void {
const event = createDiagnosticEvent(type, data);
this.appendEventSinkEntry(event);
}
private appendEventSinkEntry(event: VoiceDebugEventSinkEvent): void {
this.eventSinkSequence += 1;
assert.ok(Number.isSafeInteger(this.eventSinkSequence), 'event sink sequence must stay safe');
const entry = createEventSinkEntry(this.eventSinkSequence, event);
this.eventSinkEntries.pushDropOldest(entry);
this.forwardEventSinkEntries([entry]);
}
private forwardEventSinkEntries(entries: Array<DesktopVoiceDebugEventSinkEntry>): void {
assert.ok(entries.length >= 1, 'event sink forward requires at least one entry');
const electron = getElectronAPI();
if (electron?.appendVoiceDebugEventSinkEntries) {
try {
electron.appendVoiceDebugEventSinkEntries(entries);
this.eventSinkForwardFailureCount = 0;
} catch (error) {
this.eventSinkForwardFailureCount += 1;
if (this.eventSinkForwardFailureCount === 1) {
logger.warn('Failed to forward voice debug event sink entries to desktop popout', {error});
}
}
}
appendBrowserVoiceDebugEventSinkEntries(entries);
}
}
export default new VoiceEngineV2AppDebugEventSinkHostAdapter();
File diff suppressed because it is too large. Load diff
@@ -3,7 +3,6 @@
import assert from 'node:assert/strict';
import {Platform} from '@app/features/platform/types/Platform';
import AdaptiveScreenShareEngine from '@app/features/voice/engine/AdaptiveScreenShareEngine';
import {markScreenShareCaptureEnded} from '@app/features/voice/engine/ScreenShareCaptureDiagnostics';
import type {NegotiationReason} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import ScreenSharePublicationMigration from '@app/features/voice/engine/ScreenSharePublicationMigration';
import {updateLocalParticipantFromRoom} from '@app/features/voice/engine/VoiceMediaEngineBridge';
@@ -408,7 +407,6 @@ export class VoiceEngineV2AppScreenShareCodecMigration {
this.adapter.ensureScreenShareKeepAliveSinkInternal(ctx.participant);
} else {
await this.adapter.cleanupLingeringScreenShareTracks(ctx.participant);
markScreenShareCaptureEnded('screen-share-codec-migration-failed');
}
this.adapter.transitionScreenShareLifecycleInternal({
type: 'share.reject',
@@ -3,10 +3,6 @@
import assert from 'node:assert/strict';
import {isDesktop, isNativeMacOS} from '@app/features/ui/utils/NativeUtils';
import AdaptiveScreenShareEngine from '@app/features/voice/engine/AdaptiveScreenShareEngine';
import {
markScreenShareCaptureActive,
markScreenShareCaptureEnded,
} from '@app/features/voice/engine/ScreenShareCaptureDiagnostics';
import {updateLocalParticipantFromRoom} from '@app/features/voice/engine/VoiceMediaEngineBridge';
import {
enforceLocalMediaPublicationCap,
@@ -150,7 +146,6 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
? null
: async () => {
await this.adapter.cleanupLingeringScreenShareTracks(participant, stopCleanupSnapshot ?? undefined);
markScreenShareCaptureEnded('screen-share-disabled');
},
updateLocalParticipant: true,
audioSync: {kind: 'participant-after-watch'},
@@ -179,7 +174,6 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
): Promise<void> {
assert.ok(participant);
const cancelled = isUserCancelledScreenShareError(error);
const endedReason = cancelled ? 'screen-share-cancelled' : 'screen-share-failed';
if (cancelled) {
logger.debug('User cancelled or permission denied', {name: (error as Error).name});
} else {
@@ -192,7 +186,6 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
}
if (!actual) {
await this.adapter.cleanupLingeringScreenShareTracks(participant, stopCleanupSnapshot ?? undefined);
markScreenShareCaptureEnded(endedReason);
}
settleScreenShareFailure({
adapter: this.adapter,
@@ -356,8 +349,7 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
playSound: boolean,
): Promise<void> {
assert.ok(participant);
const {videoDeviceId, audioDeviceId} = options || {};
markScreenShareCaptureActive({method: 'device-media', device: {videoDeviceId, audioDeviceId}});
const {videoDeviceId} = options || {};
await runScreenShareActivationRitual({
adapter: this.adapter,
room,
@@ -409,7 +401,7 @@ export class VoiceEngineV2AppScreenShareLiveKitFlows {
participant,
actual: participant.isScreenShareEnabled,
applyState,
onInactiveAfterSync: () => markScreenShareCaptureEnded('device-screen-share-failed'),
onInactiveAfterSync: null,
monitorEndOnActive: false,
playSound: false,
buildTransition: (actualNow) =>
@@ -1,6 +1,5 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {markScreenShareCaptureActive} from '@app/features/voice/engine/ScreenShareCaptureDiagnostics';
import {
type CapturedScreenShareTracks,
type DeviceScreenShareCaptureOptions,
@@ -134,13 +133,6 @@ export async function createDeviceReplacementTracks(
throw new Error('No video track found in device screen share capture');
}
const audioTrack = stream.getAudioTracks()[0];
markScreenShareCaptureActive({
method: 'device-media',
device: {
videoDeviceId: options?.videoDeviceId,
audioDeviceId: options?.audioDeviceId,
},
});
stopUnselectedStreamTracks(stream, [videoTrack, audioTrack]);
return {
videoTrack,
@@ -1,12 +1,10 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {updateScreenShareDisplayMediaSettings} from '@app/features/voice/engine/ScreenShareCaptureDiagnostics';
import {
type CapturedScreenShareTracks,
stopMediaTrack,
stopUnselectedStreamTracks,
} from '@app/features/voice/engine/voice_screen_share_manager/shared';
import ActiveScreenShareSource from '@app/features/voice/state/ActiveScreenShareSource';
import type {ScreenShareCaptureOptions} from 'livekit-client';
type DisplayMediaVideoConstraints = MediaTrackConstraints & {
@@ -108,9 +106,6 @@ export async function createDisplayScreenShareTracks(
videoTrack.contentHint = options.contentHint;
}
await videoTrack.applyConstraints({colorSpace: 'rec709'} as MediaTrackConstraints).catch(() => undefined);
updateScreenShareDisplayMediaSettings(videoTrack, {
sourceId: ActiveScreenShareSource.getSourceId(),
});
const cursor = resolveCapturedDisplayMediaCursorCapture(videoTrack, options);
if ((videoTrack.getSettings() as DisplayMediaTrackSettings).cursor !== cursor) {
await videoTrack.applyConstraints({cursor} as MediaTrackConstraints).catch(() => undefined);