fix(desktop): capture screen share audio from chosen devices (#3456)

fluxer_desktop/src/common/Types.ts

@@ -27,6 +27,7 @@ import type {
 	NativeAudioAvailability,
 	NativeAudioEndMessage,
 	NativeAudioFrameMessage,
+	NativeAudioPlaybackDevice,
 	NativeAudioStartOptions,
 	NativeAudioStartResult,
 	NativeScreenCaptureLifecycleMessage,
@@ -183,6 +184,8 @@ export interface NativeAudioApi {
 	resolveAudioRootPidForSource: (sourceId: string) => Promise<number | null>;
 	start: (options: NativeAudioStartOptions) => Promise<NativeAudioStartResult>;
 	setRule: (captureId: string, linuxRule: NonNullable<NativeAudioStartOptions['linuxRule']>) => Promise<boolean>;
+	listPlaybackDevices: () => Promise<Array<NativeAudioPlaybackDevice>>;
+	setWinEndpoints: (captureId: string, endpoints: Array<string>) => Promise<boolean>;
 	stop: (captureId: string) => Promise<void>;
 	onFrame: (callback: (message: NativeAudioFrameMessage) => void) => () => void;
 	onEnd: (callback: (message: NativeAudioEndMessage) => void) => () => void;

fluxer_desktop/src/main/NativeAudio.ts

@@ -7,6 +7,7 @@ import type {
 	NativeAudioApplication,
 	NativeAudioAvailability,
 	NativeAudioEndReason,
+	NativeAudioPlaybackDevice,
 	NativeAudioStartOptions,
 	NativeAudioStartResult,
 	VirtmicLinkOptions,
@@ -22,7 +23,9 @@ import {
 	isValidAudioFrame,
 	isValidLinuxRule,
 	isValidTargetPid,
+	isValidWinEndpoints,
 	normalizeTimestampUs,
+	resolveWindowsCaptureScope,
 } from '@electron/main/NativeAudioValidation';
 import {ipcMain} from 'electron';
 
@@ -39,6 +42,7 @@ interface NativeCaptureInstance {
 	start(): Promise<void> | void;
 	stop(): Promise<void> | void;
 	setRoutingRule?: (target: {linuxRule: NonNullable<NativeAudioStartOptions['linuxRule']>}) => boolean;
+	setEndpoints?: (endpoints: Array<string>) => boolean;
 }
 
 interface WindowsNativeAudioModule {
@@ -59,12 +63,14 @@ interface WindowsNativeAudioModule {
 	};
 	pidFromHwnd: (hwnd: bigint) => number;
 	resolveAudioRootPid: (pid: number) => number;
+	listPlaybackDevices?: () => Promise<Array<NativeAudioPlaybackDevice>>;
 	ProcessLoopback: new (
 		pid: number,
 		options?: {
 			includeProcessTree?: boolean;
 			captureScope?: 'process' | 'system' | 'session-mixer';
 			winCaptureScope?: 'process' | 'system' | 'session-mixer';
+			endpoints?: Array<string>;
 		},
 	) => NativeCaptureInstance;
 	loadError?: Error | null;
@@ -270,6 +276,7 @@ function loadNativeAudioAddon(): NativeAddonLoadResult {
 							processInclude: false,
 							processExclude: false,
 							sessionMixer: false,
+							endpointSelect: false,
 							systemLoopbackMode: 'unavailable',
 						},
 					},
@@ -290,6 +297,7 @@ function loadNativeAudioAddon(): NativeAddonLoadResult {
 						processInclude,
 						processExclude,
 						sessionMixer,
+						endpointSelect: sessionMixer,
 						systemLoopbackMode:
 							systemLoopbackMode === 'process-exclude' || systemLoopbackMode === 'session-mixer'
 								? systemLoopbackMode
@@ -510,6 +518,21 @@ async function listNativeAudioApplications(): Promise<Array<NativeAudioApplicati
 	}
 }
 
+async function listWindowsPlaybackDevices(): Promise<Array<NativeAudioPlaybackDevice>> {
+	const loadResult = loadNativeAudioAddon();
+	if (loadResult.platform !== 'win32' || !loadResult.availability.available || !loadResult.addon) {
+		return [];
+	}
+	const addon = loadResult.addon as WindowsNativeAudioModule;
+	if (typeof addon.listPlaybackDevices !== 'function') return [];
+	try {
+		return await addon.listPlaybackDevices();
+	} catch (error) {
+		logger.warn('Failed to list Windows playback devices', error);
+		return [];
+	}
+}
+
 async function resolveAudioRootPidForSource(sourceId: unknown): Promise<number | null> {
 	const windowToken = parseDesktopWindowSourceToken(sourceId);
 	if (!windowToken) {
@@ -646,6 +669,19 @@ function reconfigureCaptureById(
 	}
 }
 
+function reconfigureWindowsEndpointsById(senderId: number, captureId: string, endpoints: Array<string>): boolean {
+	const session = activeSessions.get(captureId);
+	if (!session || session.sender.id !== senderId) return false;
+	if (session.finalized || session.stopping) return false;
+	if (typeof session.capture.setEndpoints !== 'function') return false;
+	try {
+		return session.capture.setEndpoints(endpoints);
+	} catch (error) {
+		logger.warn('Failed to change Windows playback devices in place', {captureId, error});
+		return false;
+	}
+}
+
 async function makeRoomForSenderSession(senderId: number): Promise<void> {
 	const senderSessions = activeSessionIdsBySenderId.get(senderId);
 	if (!senderSessions) return;
@@ -679,6 +715,10 @@ async function startNativeAudioCapture(
 				: 'process';
 	const winSystemCapture =
 		loadResult.platform === 'win32' && (winCaptureScope === 'system' || winCaptureScope === 'session-mixer');
+	const winEndpoints = loadResult.platform === 'win32' ? options?.winEndpoints : undefined;
+	if (winEndpoints !== undefined && !isValidWinEndpoints(winEndpoints)) {
+		throw new Error('Invalid Windows playback device selection');
+	}
 	const targetPid = options?.targetPid;
 	if (!options || (!linuxRule && !macSystemCapture && !winSystemCapture && !isValidTargetPid(targetPid))) {
 		throw new Error('Invalid native audio target PID');
@@ -715,13 +755,7 @@ async function startNativeAudioCapture(
 	if (requestedScope === 'system' && availability.capabilities?.systemExcludesSelf !== true) {
 		throw new Error(`Native ${loadResult.platform} system audio capture does not guarantee Fluxer self-exclusion`);
 	}
-	const effectiveWinCaptureScope =
-		loadResult.platform === 'win32' &&
-		winCaptureScope === 'system' &&
-		availability.capabilities?.processExclude === false &&
-		availability.capabilities?.sessionMixer === true
-			? 'session-mixer'
-			: winCaptureScope;
+	const windowsCapture = resolveWindowsCaptureScope(winCaptureScope, winEndpoints, availability.capabilities);
 	if (sender.isDestroyed()) {
 		throw new Error('Cannot start native audio capture for destroyed renderer');
 	}
@@ -739,7 +773,8 @@ async function startNativeAudioCapture(
 		if (!isValidTargetPid(targetPidForCapture)) throw new Error('Invalid native audio target PID');
 		capture = new (loadResult.addon as WindowsNativeAudioModule).ProcessLoopback(targetPidForCapture, {
 			includeProcessTree: winSystemCapture ? false : (options.includeProcessTree ?? true),
-			captureScope: effectiveWinCaptureScope,
+			captureScope: windowsCapture.captureScope,
+			...(windowsCapture.endpoints ? {endpoints: windowsCapture.endpoints} : {}),
 		});
 	} else if (loadResult.platform === 'darwin') {
 		if (!isValidTargetPid(targetPidForCapture)) throw new Error('Invalid native audio target PID');
@@ -883,6 +918,16 @@ export function registerNativeAudioHandlers(): void {
 			return reconfigureCaptureById(event.sender.id, captureId, linuxRule);
 		},
 	);
+	ipcMain.handle('native-audio:set-win-endpoints', (event, captureId: unknown, endpoints: unknown): boolean => {
+		if (typeof captureId !== 'string' || !isValidWinEndpoints(endpoints)) {
+			return false;
+		}
+		return reconfigureWindowsEndpointsById(event.sender.id, captureId, endpoints);
+	});
+	ipcMain.handle(
+		'native-audio:list-playback-devices',
+		(): Promise<Array<NativeAudioPlaybackDevice>> => listWindowsPlaybackDevices(),
+	);
 	ipcMain.handle('native-audio:stop', async (event, captureId: string): Promise<void> => {
 		const session = activeSessions.get(captureId);
 		if (!session || session.sender.id !== event.sender.id) {
@@ -899,6 +944,8 @@ export function cleanupNativeAudio(): void {
 	ipcMain.removeHandler('native-audio:resolve-root-pid');
 	ipcMain.removeHandler('native-audio:start');
 	ipcMain.removeHandler('native-audio:set-rule');
+	ipcMain.removeHandler('native-audio:set-win-endpoints');
+	ipcMain.removeHandler('native-audio:list-playback-devices');
 	ipcMain.removeHandler('native-audio:stop');
 	handlersRegistered = false;
 	for (const session of [...activeSessions.values()]) {

fluxer_desktop/src/main/NativeAudioValidation.test.mjs

@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import assert from 'node:assert/strict';
+import {describe, test} from 'node:test';
+import {loadTsModule} from './fixtures/TsModuleLoader.mjs';
+
+const {isValidWinEndpoints, resolveWindowsCaptureScope} = loadTsModule('@electron/main/NativeAudioValidation');
+
+const SPEAKERS_ID = '{0.0.0.00000000}.{6f1e8a2c-0000-4000-8000-000000000001}';
+
+describe('isValidWinEndpoints', () => {
+	test('accepts the default entry and device ids', () => {
+		assert.equal(isValidWinEndpoints(['default']), true);
+		assert.equal(isValidWinEndpoints(['default', SPEAKERS_ID]), true);
+		assert.equal(isValidWinEndpoints([`{${'a'.repeat(511)}`]), true);
+	});
+
+	test('rejects malformed selections', () => {
+		assert.equal(isValidWinEndpoints(undefined), false);
+		assert.equal(isValidWinEndpoints('default'), false);
+		assert.equal(isValidWinEndpoints([]), false);
+		assert.equal(isValidWinEndpoints(Array.from({length: 17}, (_, index) => `{${index}}`)), false);
+		assert.equal(isValidWinEndpoints(['speakers']), false);
+		assert.equal(isValidWinEndpoints(['{abc\0}']), false);
+		assert.equal(isValidWinEndpoints([`{${'a'.repeat(512)}`]), false);
+		assert.equal(isValidWinEndpoints([42]), false);
+	});
+});
+
+describe('resolveWindowsCaptureScope', () => {
+	const fullCapabilities = {
+		process: true,
+		system: true,
+		processExclude: true,
+		sessionMixer: true,
+		endpointSelect: true,
+	};
+
+	test('device selection needs the session mixer and the endpoint capability', () => {
+		assert.deepEqual(resolveWindowsCaptureScope('session-mixer', ['default'], fullCapabilities), {
+			captureScope: 'session-mixer',
+			endpoints: ['default'],
+		});
+		assert.throws(() =>
+			resolveWindowsCaptureScope('session-mixer', ['default'], {...fullCapabilities, endpointSelect: false}),
+		);
+		assert.throws(() => resolveWindowsCaptureScope('session-mixer', ['default'], undefined));
+		assert.throws(() => resolveWindowsCaptureScope('system', ['default'], fullCapabilities));
+	});
+
+	test('include-only hosts downgrade entire system audio to the session mixer', () => {
+		assert.deepEqual(resolveWindowsCaptureScope('system', undefined, {...fullCapabilities, processExclude: false}), {
+			captureScope: 'session-mixer',
+		});
+	});
+
+	test('entire system audio keeps process-exclude loopback when it is available', () => {
+		assert.deepEqual(resolveWindowsCaptureScope('system', undefined, fullCapabilities), {captureScope: 'system'});
+		assert.deepEqual(resolveWindowsCaptureScope('process', undefined, fullCapabilities), {captureScope: 'process'});
+	});
+});

fluxer_desktop/src/main/NativeAudioValidation.ts

@@ -2,7 +2,7 @@
 
 import {isPlainRecord as isPlainObject} from '@electron/common/PlainRecord';
 
-import type {NativeAudioStartOptions, VirtmicNode} from '@electron/common/Types';
+import type {NativeAudioAvailability, NativeAudioStartOptions, VirtmicNode} from '@electron/common/Types';
 
 interface InternalAudioFrame {
 	samples: Float32Array;
@@ -20,6 +20,11 @@ const MAX_LINUX_RULE_KEYS_PER_PATTERN = 32;
 const MAX_LINUX_RULE_KEY_LENGTH = 128;
 const MAX_LINUX_RULE_VALUE_LENGTH = 512;
 const MAX_UINT32 = 0xffffffff;
+const DEFAULT_WIN_ENDPOINT = 'default';
+const MAX_WIN_ENDPOINTS = 16;
+const MAX_WIN_ENDPOINT_ID_LENGTH = 512;
+
+type WindowsCaptureScope = NonNullable<NativeAudioStartOptions['winCaptureScope']>;
 
 export function isValidTargetPid(pid: unknown): pid is number {
 	return typeof pid === 'number' && Number.isSafeInteger(pid) && pid > 0 && pid <= MAX_UINT32;
@@ -77,6 +82,38 @@ export function isValidLinuxRule(
 	);
 }
 
+function isValidWinEndpoint(value: unknown): value is string {
+	if (typeof value !== 'string') return false;
+	if (value === DEFAULT_WIN_ENDPOINT) return true;
+	return value.length <= MAX_WIN_ENDPOINT_ID_LENGTH && !value.includes('\0') && value.startsWith('{');
+}
+
+export function isValidWinEndpoints(value: unknown): value is Array<string> {
+	return (
+		Array.isArray(value) && value.length > 0 && value.length <= MAX_WIN_ENDPOINTS && value.every(isValidWinEndpoint)
+	);
+}
+
+export function resolveWindowsCaptureScope(
+	requested: WindowsCaptureScope,
+	winEndpoints: Array<string> | undefined,
+	capabilities: NativeAudioAvailability['capabilities'],
+): {captureScope: WindowsCaptureScope; endpoints?: Array<string>} {
+	if (winEndpoints) {
+		if (requested !== 'session-mixer') {
+			throw new Error('Windows playback device selection requires the session-mixer scope');
+		}
+		if (capabilities?.endpointSelect !== true) {
+			throw new Error('Windows playback device selection is unavailable');
+		}
+		return {captureScope: 'session-mixer', endpoints: winEndpoints};
+	}
+	if (requested === 'system' && capabilities?.processExclude === false && capabilities?.sessionMixer === true) {
+		return {captureScope: 'session-mixer'};
+	}
+	return {captureScope: requested};
+}
+
 export function normalizeTimestampUs(timestampUs: number | bigint): number {
 	const numericTimestamp = typeof timestampUs === 'bigint' ? Number(timestampUs) : timestampUs;
 	if (!Number.isFinite(numericTimestamp)) return 0;

fluxer_desktop/src/preload/index.ts

@@ -36,6 +36,7 @@ import type {
 	NativeAudioAvailability,
 	NativeAudioEndMessage,
 	NativeAudioFrameMessage,
+	NativeAudioPlaybackDevice,
 	NativeAudioStartOptions,
 	NativeAudioStartResult,
 	NativeScreenCaptureLifecycleEventKind,
@@ -779,6 +780,10 @@ const api: ElectronAPI = {
 			ipcRenderer.invoke('native-audio:start', options),
 		setRule: (captureId: string, linuxRule: NonNullable<NativeAudioStartOptions['linuxRule']>): Promise<boolean> =>
 			ipcRenderer.invoke('native-audio:set-rule', captureId, linuxRule),
+		listPlaybackDevices: (): Promise<Array<NativeAudioPlaybackDevice>> =>
+			ipcRenderer.invoke('native-audio:list-playback-devices'),
+		setWinEndpoints: (captureId: string, endpoints: Array<string>): Promise<boolean> =>
+			ipcRenderer.invoke('native-audio:set-win-endpoints', captureId, endpoints),
 		stop: (captureId: string): Promise<void> => ipcRenderer.invoke('native-audio:stop', captureId),
 		onFrame: (callback: (message: NativeAudioFrameMessage) => void): (() => void) => {
 			const handler = (_event: Electron.IpcRendererEvent, message: NativeAudioFrameMessage): void => {