From 8cac3b35c9144ca2ea9dfb2a6e6561ea252123b3 Mon Sep 17 00:00:00 2001 From: Hampus Date: Tue, 13 Oct 2026 17:57:09 +0200 Subject: [PATCH] fix(desktop): reuse window capture items on Windows 10 (#3449) --- .github/workflows/tests.yaml | 3 +- .../native/win-game-capture/Cargo.toml | 2 + .../native/win-game-capture/index.d.ts | 2 + .../native/win-game-capture/index.js | 18 ++ .../native/win-game-capture/package.json | 2 +- .../src/capture_item_reuse.rs | 204 ++++++++++++ .../native/win-game-capture/src/lib.rs | 34 ++ .../native/win-game-capture/src/live_items.rs | 305 ++++++++++++++++++ fluxer_desktop/src/main/DisplayMedia.ts | 2 + fluxer_desktop/src/main/StreamingPriority.ts | 3 + .../src/main/WindowsScreenCaptureGuard.ts | 2 +- .../WindowsWindowCaptureItemReuse.test.mjs | 139 ++++++++ .../src/main/WindowsWindowCaptureItemReuse.ts | 86 +++++ tools/ci/check-desktop-native-workspaces.sh | 28 +- 14 files changed, 824 insertions(+), 6 deletions(-) create mode 100644 fluxer_desktop/native/win-game-capture/src/capture_item_reuse.rs create mode 100644 fluxer_desktop/native/win-game-capture/src/live_items.rs create mode 100644 fluxer_desktop/src/main/WindowsWindowCaptureItemReuse.test.mjs create mode 100644 fluxer_desktop/src/main/WindowsWindowCaptureItemReuse.ts diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 571343715..aeb0f78a1 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -117,6 +117,7 @@ jobs: with: toolchain: "1.98.1" components: clippy, rustfmt + targets: x86_64-pc-windows-msvc - name: Install pnpm uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 @@ -208,7 +209,7 @@ jobs: - name: Clippy (warnings as errors) run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings - - name: Clippy (desktop native workspaces on Linux, warnings as errors) + - name: Clippy (desktop native workspaces on Linux and Windows backends, warnings as errors) run: tools/ci/check-desktop-native-workspaces.sh clippy - name: Verify the source-built ffmpeg CLI is on PATH diff --git a/fluxer_desktop/native/win-game-capture/Cargo.toml b/fluxer_desktop/native/win-game-capture/Cargo.toml index b9a83a37c..ba6f985d2 100644 --- a/fluxer_desktop/native/win-game-capture/Cargo.toml +++ b/fluxer_desktop/native/win-game-capture/Cargo.toml @@ -19,7 +19,9 @@ napi-derive = "3.6.7" windows = {version = "0.62.2", features = [ "Win32_Foundation", "Win32_System_LibraryLoader", + "Win32_System_Memory", "Win32_System_Threading", + "Win32_System_WinRT_Graphics_Capture", ]} [build-dependencies] diff --git a/fluxer_desktop/native/win-game-capture/index.d.ts b/fluxer_desktop/native/win-game-capture/index.d.ts index 3efa35421..b028ea259 100644 --- a/fluxer_desktop/native/win-game-capture/index.d.ts +++ b/fluxer_desktop/native/win-game-capture/index.d.ts @@ -2,4 +2,6 @@ export declare function elevateGpuSchedulingPriority(processId?: number, priorityClass?: 'high' | 'realtime'): boolean; export declare function restoreGpuSchedulingPriority(processId?: number): boolean; +export declare function reuseWindowCaptureItems(): boolean; +export declare function windowCaptureItemCalls(): {forwarded: number; reused: number} | null; export declare const loadError: Error | null; diff --git a/fluxer_desktop/native/win-game-capture/index.js b/fluxer_desktop/native/win-game-capture/index.js index cf25831fe..571182460 100644 --- a/fluxer_desktop/native/win-game-capture/index.js +++ b/fluxer_desktop/native/win-game-capture/index.js @@ -94,9 +94,27 @@ function restoreGpuSchedulingPriority(processId) { } } +function reuseWindowCaptureItems() { + if (!binding) return false; + try { + binding.reuseWindowCaptureItems(); + return true; + } catch (error) { + console.warn('[win-game-capture] reuseWindowCaptureItems failed:', error?.message || error); + return false; + } +} + +function windowCaptureItemCalls() { + if (!binding) return null; + return binding.windowCaptureItemCalls(); +} + module.exports = { elevateGpuSchedulingPriority, restoreGpuSchedulingPriority, + reuseWindowCaptureItems, + windowCaptureItemCalls, get loadError() { return loadError; }, diff --git a/fluxer_desktop/native/win-game-capture/package.json b/fluxer_desktop/native/win-game-capture/package.json index 8e7f50026..c76c41fdf 100644 --- a/fluxer_desktop/native/win-game-capture/package.json +++ b/fluxer_desktop/native/win-game-capture/package.json @@ -1,7 +1,7 @@ { "name": "@fluxer/win-game-capture", "version": "0.0.0", - "description": "Native Windows GPU scheduling priority helpers", + "description": "Native Windows GPU scheduling priority and window capture item helpers", "private": true, "license": "AGPL-3.0-or-later", "os": [ diff --git a/fluxer_desktop/native/win-game-capture/src/capture_item_reuse.rs b/fluxer_desktop/native/win-game-capture/src/capture_item_reuse.rs new file mode 100644 index 000000000..c15e57b86 --- /dev/null +++ b/fluxer_desktop/native/win-game-capture/src/capture_item_reuse.rs @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +use crate::live_items::{LiveItems, WeakItem}; +use std::{ + cell::RefCell, + ffi::c_void, + mem::{offset_of, size_of}, + sync::{ + OnceLock, + atomic::{AtomicPtr, AtomicU64, Ordering}, + }, +}; +use windows::{ + Win32::{ + Foundation::{HMODULE, HWND}, + System::{ + LibraryLoader::{ + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS, GET_MODULE_HANDLE_EX_FLAG_PIN, + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, GetModuleHandleExW, + }, + Memory::{PAGE_PROTECTION_FLAGS, PAGE_READWRITE, VirtualProtect}, + WinRT::{ + Graphics::Capture::{ + IGraphicsCaptureItemInterop, IGraphicsCaptureItemInterop_Vtbl, + }, + RoGetActivationFactory, + }, + }, + }, + core::{GUID, HRESULT, IUnknown, Interface, PCWSTR, Weak, h, w}, +}; + +type CreateForWindow = + unsafe extern "system" fn(*mut c_void, HWND, *const GUID, *mut *mut c_void) -> HRESULT; + +const _: () = assert!( + offset_of!(IGraphicsCaptureItemInterop_Vtbl, CreateForWindow) == 3 * size_of::() +); +const _: () = assert!(size_of::() == size_of::<*mut c_void>()); + +static INSTALLED: OnceLock> = OnceLock::new(); +static CREATE_FOR_WINDOW_ORIGINAL: OnceLock = OnceLock::new(); +static CALLS_FORWARDED: AtomicU64 = AtomicU64::new(0); +static CALLS_REUSED: AtomicU64 = AtomicU64::new(0); + +pub struct CreateForWindowCalls { + pub forwarded: u64, + pub reused: u64, +} + +thread_local! { + static LIVE_ITEMS: RefCell>> = + const { RefCell::new(LiveItems::new()) }; +} + +impl WeakItem for Weak { + type Strong = IUnknown; + + fn upgrade(&self) -> Option { + Weak::upgrade(self) + } +} + +fn live_item(window: isize) -> Option { + LIVE_ITEMS + .try_with(|items| items.try_borrow_mut().ok()?.lookup(window)) + .ok() + .flatten() +} + +fn remember_item(window: isize, item: &IUnknown) { + let Ok(weak) = item.downgrade() else { + return; + }; + let _ = LIVE_ITEMS.try_with(|items| { + if let Ok(mut items) = items.try_borrow_mut() { + items.remember(window, weak); + } + }); +} + +unsafe fn reuse_live_item( + window: isize, + iid: *const GUID, + result: *mut *mut c_void, +) -> Option { + if iid.is_null() { + return None; + } + if result.is_null() { + return None; + } + let item = live_item(window)?; + let reused = unsafe { item.query(iid, result) }; + reused.is_ok().then_some(reused) +} + +unsafe extern "system" fn create_for_window_reusing_live_item( + this: *mut c_void, + window: HWND, + iid: *const GUID, + result: *mut *mut c_void, +) -> HRESULT { + let Some(original) = CREATE_FOR_WINDOW_ORIGINAL.get().copied() else { + unreachable!("the slot is replaced only after the original is recorded"); + }; + let window_key = window.0 as isize; + if let Some(reused) = unsafe { reuse_live_item(window_key, iid, result) } { + CALLS_REUSED.fetch_add(1, Ordering::Relaxed); + return reused; + } + CALLS_FORWARDED.fetch_add(1, Ordering::Relaxed); + let created = unsafe { original(this, window, iid, result) }; + if created.is_err() { + return created; + } + if result.is_null() { + return created; + } + if let Some(item) = unsafe { IUnknown::from_raw_borrowed(&*result) } { + remember_item(window_key, item); + } + created +} + +fn module_at(address: *const c_void, flags: u32) -> Result { + let mut module = HMODULE::default(); + unsafe { + GetModuleHandleExW( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | flags, + PCWSTR(address.cast()), + &mut module, + ) + } + .map_err(|error| format!("GetModuleHandleExW(address): {error}"))?; + Ok(module) +} + +fn pin_modules(slot: *const c_void) -> Result<(), String> { + let mut graphics_capture = HMODULE::default(); + unsafe { + GetModuleHandleExW( + GET_MODULE_HANDLE_EX_FLAG_PIN, + w!("GraphicsCapture.dll"), + &mut graphics_capture, + ) + } + .map_err(|error| format!("GetModuleHandleExW(GraphicsCapture.dll): {error}"))?; + if module_at(slot, GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT)? != graphics_capture { + return Err("capture item factory vtable is outside GraphicsCapture.dll".into()); + } + let replacement: CreateForWindow = create_for_window_reusing_live_item; + module_at(replacement as *const c_void, GET_MODULE_HANDLE_EX_FLAG_PIN)?; + Ok(()) +} + +fn replace_slot(slot: *mut CreateForWindow, original: CreateForWindow) -> Result<(), String> { + let replacement: CreateForWindow = create_for_window_reusing_live_item; + let size = size_of::(); + let mut protection = PAGE_PROTECTION_FLAGS(0); + unsafe { VirtualProtect(slot.cast(), size, PAGE_READWRITE, &mut protection) } + .map_err(|error| format!("VirtualProtect(PAGE_READWRITE): {error}"))?; + let slot_value = unsafe { AtomicPtr::::from_ptr(slot.cast()) }; + slot_value.store(replacement as *mut c_void, Ordering::Release); + let mut replaced = PAGE_PROTECTION_FLAGS(0); + let restored = unsafe { VirtualProtect(slot.cast(), size, protection, &mut replaced) }; + if let Err(error) = restored { + slot_value.store(original as *mut c_void, Ordering::Release); + return Err(format!("VirtualProtect(restore): {error}")); + } + Ok(()) +} + +fn install_once() -> Result<(), String> { + let interop: IGraphicsCaptureItemInterop = + unsafe { RoGetActivationFactory(h!("Windows.Graphics.Capture.GraphicsCaptureItem")) } + .map_err(|error| format!("GraphicsCaptureItem activation factory: {error}"))?; + let vtable = unsafe { + *interop + .as_raw() + .cast::<*mut IGraphicsCaptureItemInterop_Vtbl>() + }; + assert!(!vtable.is_null()); + let slot = unsafe { &raw mut (*vtable).CreateForWindow }; + pin_modules(slot.cast())?; + let original = unsafe { slot.read() }; + let replacement: CreateForWindow = create_for_window_reusing_live_item; + assert!(!std::ptr::fn_addr_eq(original, replacement)); + CREATE_FOR_WINDOW_ORIGINAL + .set(original) + .map_err(|_| "CreateForWindow original already recorded".to_string())?; + replace_slot(slot, original) +} + +pub fn install() -> Result<(), String> { + INSTALLED.get_or_init(install_once).clone() +} + +pub fn create_for_window_calls() -> CreateForWindowCalls { + CreateForWindowCalls { + forwarded: CALLS_FORWARDED.load(Ordering::Relaxed), + reused: CALLS_REUSED.load(Ordering::Relaxed), + } +} diff --git a/fluxer_desktop/native/win-game-capture/src/lib.rs b/fluxer_desktop/native/win-game-capture/src/lib.rs index 89a035bd0..7eed1e439 100644 --- a/fluxer_desktop/native/win-game-capture/src/lib.rs +++ b/fluxer_desktop/native/win-game-capture/src/lib.rs @@ -2,7 +2,11 @@ #![deny(clippy::all)] +#[cfg(target_os = "windows")] +mod capture_item_reuse; mod gpu_priority; +#[cfg(any(target_os = "windows", test))] +mod live_items; use napi::bindgen_prelude::*; use napi_derive::napi; @@ -19,3 +23,33 @@ pub fn elevate_gpu_scheduling_priority( pub fn restore_gpu_scheduling_priority(process_id: Option) -> Result<()> { gpu_priority::restore(process_id).map_err(napi::Error::from_reason) } + +#[napi(object)] +pub struct WindowCaptureItemCalls { + pub forwarded: i64, + pub reused: i64, +} + +#[napi(js_name = "reuseWindowCaptureItems")] +pub fn reuse_window_capture_items() -> Result<()> { + #[cfg(target_os = "windows")] + capture_item_reuse::install().map_err(napi::Error::from_reason)?; + Ok(()) +} + +#[napi(js_name = "windowCaptureItemCalls")] +pub fn window_capture_item_calls() -> WindowCaptureItemCalls { + #[cfg(target_os = "windows")] + { + let calls = capture_item_reuse::create_for_window_calls(); + WindowCaptureItemCalls { + forwarded: i64::try_from(calls.forwarded).unwrap_or(i64::MAX), + reused: i64::try_from(calls.reused).unwrap_or(i64::MAX), + } + } + #[cfg(not(target_os = "windows"))] + WindowCaptureItemCalls { + forwarded: 0, + reused: 0, + } +} diff --git a/fluxer_desktop/native/win-game-capture/src/live_items.rs b/fluxer_desktop/native/win-game-capture/src/live_items.rs new file mode 100644 index 000000000..cc4b56bb0 --- /dev/null +++ b/fluxer_desktop/native/win-game-capture/src/live_items.rs @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +pub const LIVE_ITEMS_MAX: usize = 8; + +pub trait WeakItem { + type Strong; + + fn upgrade(&self) -> Option; +} + +struct LiveItem { + window: isize, + weak: W, + used_at: u64, +} + +pub struct LiveItems { + slots: [Option>; LIVE_ITEMS_MAX], + uses: u64, +} + +impl LiveItems { + pub const fn new() -> Self { + Self { + slots: [const { None }; LIVE_ITEMS_MAX], + uses: 0, + } + } + + pub fn lookup(&mut self, window: isize) -> Option { + let used_at = self.next_use(); + for slot in &mut self.slots { + let Some(item) = slot else { + continue; + }; + if item.window != window { + continue; + } + match item.weak.upgrade() { + Some(strong) => { + item.used_at = used_at; + return Some(strong); + } + None => *slot = None, + } + } + None + } + + pub fn remember(&mut self, window: isize, weak: W) { + let used_at = self.next_use(); + let index = self.vacant_index(); + assert!(index < LIVE_ITEMS_MAX); + self.slots[index] = Some(LiveItem { + window, + weak, + used_at, + }); + } + + fn next_use(&mut self) -> u64 { + assert!(self.uses < u64::MAX); + self.uses += 1; + self.uses + } + + fn vacant_index(&self) -> usize { + let mut least_recently_used = 0; + let mut least_recent_use = u64::MAX; + for (index, slot) in self.slots.iter().enumerate() { + let Some(item) = slot else { + return index; + }; + if item.weak.upgrade().is_none() { + return index; + } + if item.used_at < least_recent_use { + least_recent_use = item.used_at; + least_recently_used = index; + } + } + least_recently_used + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::rc::{Rc, Weak}; + + const FRAMES_IN_HALF_AN_HOUR_AT_30_FPS: u32 = 30 * 60 * 30; + + impl WeakItem for Weak { + type Strong = Rc; + + fn upgrade(&self) -> Option> { + Weak::upgrade(self) + } + } + + struct CaptureService { + items_created: u32, + live: LiveItems>, + } + + impl CaptureService { + fn new() -> Self { + Self { + items_created: 0, + live: LiveItems::new(), + } + } + + fn create_for_window(&mut self, window: isize) -> Rc { + if let Some(item) = self.live.lookup(window) { + return item; + } + self.items_created += 1; + let item = Rc::new(window); + self.live.remember(window, Rc::downgrade(&item)); + item + } + + fn occupied_count(&self) -> usize { + self.live.slots.iter().flatten().count() + } + } + + #[test] + fn per_frame_probes_reuse_the_item_the_share_holds() { + let mut service = CaptureService::new(); + drop(service.create_for_window(0x1234)); + let share_item = service.create_for_window(0x1234); + assert_eq!(service.items_created, 2); + for _ in 0..FRAMES_IN_HALF_AN_HOUR_AT_30_FPS { + let probe = service.create_for_window(0x1234); + assert!(Rc::ptr_eq(&probe, &share_item)); + } + assert_eq!(service.items_created, 2); + assert_eq!(Rc::strong_count(&share_item), 1); + } + + #[test] + fn probes_without_a_held_item_each_create_one() { + let mut service = CaptureService::new(); + for _ in 0..100 { + drop(service.create_for_window(7)); + } + assert_eq!(service.items_created, 100); + assert_eq!(service.occupied_count(), 1); + } + + #[test] + fn a_new_share_of_the_same_window_creates_a_new_item() { + let mut service = CaptureService::new(); + drop(service.create_for_window(9)); + let second_share_item = service.create_for_window(9); + assert_eq!(service.items_created, 2); + drop(second_share_item); + assert!(service.live.lookup(9).is_none()); + assert_eq!(service.occupied_count(), 0); + } + + #[test] + fn windows_never_share_an_item() { + let mut service = CaptureService::new(); + let first = service.create_for_window(1); + let second = service.create_for_window(2); + assert!(!Rc::ptr_eq(&first, &second)); + assert!(Rc::ptr_eq(&service.create_for_window(1), &first)); + assert!(Rc::ptr_eq(&service.create_for_window(2), &second)); + assert_eq!(service.items_created, 2); + } + + #[test] + fn a_dead_slot_is_reused_before_a_live_one_is_evicted() { + let mut service = CaptureService::new(); + let held: Vec<_> = (0..LIVE_ITEMS_MAX as isize) + .map(|window| service.create_for_window(window)) + .collect(); + drop(service.create_for_window(100)); + let newest = service.create_for_window(101); + assert_eq!(service.occupied_count(), LIVE_ITEMS_MAX); + assert!(Rc::ptr_eq(&service.create_for_window(101), &newest)); + for (window, item) in held.iter().enumerate().skip(1) { + assert!(Rc::ptr_eq( + &service.create_for_window(window as isize), + item + )); + } + } + + #[test] + fn a_window_probed_every_frame_outlives_more_held_items_than_slots() { + let mut service = CaptureService::new(); + drop(service.create_for_window(1)); + let editor_item = service.create_for_window(1); + let mut slide_show_items = Vec::new(); + for slide_show in 0..(LIVE_ITEMS_MAX as isize * 3) { + let slide_show_window = 100 + slide_show; + drop(service.create_for_window(slide_show_window)); + let slide_show_item = service.create_for_window(slide_show_window); + for _ in 0..30 { + let probe = service.create_for_window(slide_show_window); + assert!(Rc::ptr_eq(&probe, &slide_show_item)); + } + slide_show_items.push(slide_show_item); + let items_created = service.items_created; + for _ in 0..30 { + let probe = service.create_for_window(1); + assert!(Rc::ptr_eq(&probe, &editor_item)); + } + assert_eq!(service.items_created, items_created); + } + assert_eq!(service.items_created, 2 + 2 * LIVE_ITEMS_MAX as u32 * 3); + assert_eq!(service.occupied_count(), LIVE_ITEMS_MAX); + } + + #[test] + fn the_least_recently_used_held_item_is_the_one_evicted() { + for unused in 0..LIVE_ITEMS_MAX as isize { + let mut service = CaptureService::new(); + let held: Vec<_> = (0..LIVE_ITEMS_MAX as isize) + .map(|window| service.create_for_window(window)) + .collect(); + for window in (0..LIVE_ITEMS_MAX as isize).filter(|window| *window != unused) { + drop(service.create_for_window(window)); + } + let newest = service.create_for_window(100); + assert!(service.live.lookup(unused).is_none()); + assert!(Rc::ptr_eq(&service.create_for_window(100), &newest)); + for window in (0..LIVE_ITEMS_MAX as isize).filter(|window| *window != unused) { + assert!(Rc::ptr_eq( + &service.create_for_window(window), + &held[window as usize] + )); + } + } + } + + #[test] + fn a_newly_held_item_is_evicted_after_the_older_ones() { + let mut service = CaptureService::new(); + let mut held: Vec<_> = (0..LIVE_ITEMS_MAX as isize - 1) + .map(|window| service.create_for_window(window)) + .collect(); + for window in 0..LIVE_ITEMS_MAX as isize - 1 { + drop(service.create_for_window(window)); + } + let newly_held_window = LIVE_ITEMS_MAX as isize - 1; + held.push(service.create_for_window(newly_held_window)); + let over_the_bound = service.create_for_window(100); + assert!(service.live.lookup(0).is_none()); + assert!(Rc::ptr_eq( + &service.create_for_window(newly_held_window), + &held[newly_held_window as usize] + )); + assert!(Rc::ptr_eq(&service.create_for_window(100), &over_the_bound)); + } + + #[test] + fn random_share_sequences_match_a_reference_model() { + for seed in 1..=32u64 { + let mut state = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15); + let mut service = CaptureService::new(); + let mut held: [Option>; LIVE_ITEMS_MAX] = [const { None }; LIVE_ITEMS_MAX]; + let mut items_expected = 0; + for _ in 0..5_000 { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + let window = (state % LIVE_ITEMS_MAX as u64) as usize; + let action = (state >> 32) % 8; + if action == 0 { + held[window] = None; + } else if let Some(share_item) = &held[window] { + let probe = service.create_for_window(window as isize); + assert!(Rc::ptr_eq(&probe, share_item), "seed {seed}"); + } else { + items_expected += 1; + let item = service.create_for_window(window as isize); + assert_eq!(*item, window as isize, "seed {seed}"); + if action == 1 { + held[window] = Some(item); + } + } + assert_eq!(service.items_created, items_expected, "seed {seed}"); + for share_item in held.iter().flatten() { + assert_eq!(Rc::strong_count(share_item), 1, "seed {seed}"); + } + } + } + } + + #[test] + fn the_table_never_grows_past_its_bound() { + let mut service = CaptureService::new(); + let held: Vec<_> = (0..(LIVE_ITEMS_MAX as isize * 4)) + .map(|window| service.create_for_window(window)) + .collect(); + assert_eq!(service.occupied_count(), LIVE_ITEMS_MAX); + drop(held); + assert!(service.live.lookup(0).is_none()); + } +} diff --git a/fluxer_desktop/src/main/DisplayMedia.ts b/fluxer_desktop/src/main/DisplayMedia.ts index 97e08b3a1..e99245bbc 100644 --- a/fluxer_desktop/src/main/DisplayMedia.ts +++ b/fluxer_desktop/src/main/DisplayMedia.ts @@ -8,6 +8,7 @@ import { } from '@electron/main/DisplayMediaValidation'; import {isWaylandSession} from '@electron/main/LinuxSession'; import {startWindowsScreenCaptureGuardForSource} from '@electron/main/WindowsScreenCaptureGuard'; +import {reuseWindowsCaptureItemsForSource} from '@electron/main/WindowsWindowCaptureItemReuse'; import {BrowserWindow, desktopCapturer, ipcMain, screen} from 'electron'; import log from 'electron-log'; @@ -448,6 +449,7 @@ export function registerDisplayMediaHandlers(): void { withAudio: withAudio === true, }); startWindowsScreenCaptureGuardForSource(selectedSource, event.sender); + reuseWindowsCaptureItemsForSource(selectedSource); const streams: Electron.Streams = { video: selectedSource, }; diff --git a/fluxer_desktop/src/main/StreamingPriority.ts b/fluxer_desktop/src/main/StreamingPriority.ts index 13cf975c6..c3a80d4bc 100644 --- a/fluxer_desktop/src/main/StreamingPriority.ts +++ b/fluxer_desktop/src/main/StreamingPriority.ts @@ -3,6 +3,7 @@ import {createRequire} from 'node:module'; import os from 'node:os'; import {stopWindowsScreenCaptureGuard} from '@electron/main/WindowsScreenCaptureGuard'; +import {reportWindowsCaptureItemReuse} from '@electron/main/WindowsWindowCaptureItemReuse'; import {app, powerSaveBlocker} from 'electron'; import log from 'electron-log'; @@ -631,6 +632,7 @@ export function releaseStreamingPriority(): void { restoreProcessPriority(); restoreGpuSchedulingPriority(); stopWindowsScreenCaptureGuard('streaming-priority-release'); + reportWindowsCaptureItemReuse('streaming-priority-release'); log.info('[StreamingPriority] Released'); } } @@ -645,5 +647,6 @@ export function resetStreamingPriority(): void { restoreProcessPriority(); restoreGpuSchedulingPriority(); stopWindowsScreenCaptureGuard('streaming-priority-reset'); + reportWindowsCaptureItemReuse('streaming-priority-reset'); log.info('[StreamingPriority] Reset'); } diff --git a/fluxer_desktop/src/main/WindowsScreenCaptureGuard.ts b/fluxer_desktop/src/main/WindowsScreenCaptureGuard.ts index 4dfbb4b28..94bfd58ae 100644 --- a/fluxer_desktop/src/main/WindowsScreenCaptureGuard.ts +++ b/fluxer_desktop/src/main/WindowsScreenCaptureGuard.ts @@ -21,7 +21,7 @@ function isScreenSource(source: Electron.DesktopCapturerSource): boolean { return source.id.startsWith('screen:'); } -function getWindowsBuildNumber(): number | null { +export function getWindowsBuildNumber(): number | null { const build = Number.parseInt(os.release().split('.')[2] ?? '', 10); return Number.isSafeInteger(build) && build > 0 ? build : null; } diff --git a/fluxer_desktop/src/main/WindowsWindowCaptureItemReuse.test.mjs b/fluxer_desktop/src/main/WindowsWindowCaptureItemReuse.test.mjs new file mode 100644 index 000000000..c295bb7eb --- /dev/null +++ b/fluxer_desktop/src/main/WindowsWindowCaptureItemReuse.test.mjs @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {createRequire} from 'node:module'; +import {describe, test} from 'node:test'; +import {fileURLToPath} from 'node:url'; +import vm from 'node:vm'; + +const require = createRequire(import.meta.url); +const esbuild = require('esbuild'); + +function transform(fileName) { + const sourcePath = fileURLToPath(new URL(fileName, import.meta.url)); + return esbuild.transformSync(readFileSync(sourcePath, 'utf8'), { + loader: 'ts', + format: 'cjs', + platform: 'node', + target: 'node20', + logLevel: 'silent', + }).code; +} + +const reuseSource = transform('./WindowsWindowCaptureItemReuse.ts'); +const nativeAddonSource = transform('./NativeAddon.ts'); + +function loadReuse({platform = 'win32', windowsBuild = 19045, addon}) { + const logged = []; + const calls = {install: 0, loads: 0}; + const counts = {forwarded: 0, reused: 0}; + const fakeAddon = addon ?? { + loadError: null, + reuseWindowCaptureItems: () => { + calls.install += 1; + return true; + }, + windowCaptureItemCalls: () => ({...counts}), + }; + const context = vm.createContext({console, process: {platform, argv: []}}); + const load = (source) => { + const module = {exports: {}}; + vm.runInContext(`(function (module, exports, require) {${source}\n})`, context)( + module, + module.exports, + fakeRequire, + ); + return module.exports; + }; + const fakeRequire = (specifier) => { + if (specifier === 'node:module') { + return { + createRequire: () => (name) => { + assert.equal(name, '@fluxer/win-game-capture'); + calls.loads += 1; + return fakeAddon; + }, + }; + } + if (specifier === '@electron/main/NativeAddon') return load(nativeAddonSource); + if (specifier === '@electron/main/LaunchOptions') return {isSafeModeLaunch: () => false}; + if (specifier === '@electron/main/WindowsScreenCaptureGuard') return {getWindowsBuildNumber: () => windowsBuild}; + if (specifier === 'electron-log') { + const record = + (level) => + (...args) => + logged.push([level, ...args]); + return {info: record('info'), warn: record('warn')}; + } + throw new Error(`unexpected require ${specifier}`); + }; + return {...load(reuseSource), calls, counts, logged}; +} + +const windowSource = {id: 'window:1234:0', name: 'Notepad'}; + +describe('Windows window capture item reuse', () => { + test('installs once, only for window shares on Windows 10 2004 to 22H2', () => { + const cases = [ + [17763, 0], + [18362, 0], + [19040, 0], + [19041, 1], + [19044, 1], + [19045, 1], + [19046, 0], + [20348, 0], + [22000, 0], + [26100, 0], + [null, 0], + ]; + for (const [windowsBuild, installs] of cases) { + const reuse = loadReuse({windowsBuild}); + reuse.reuseWindowsCaptureItemsForSource({id: 'screen:0:0', name: 'Screen 1'}); + assert.equal(reuse.calls.loads, 0, `build ${windowsBuild} screen share`); + reuse.reuseWindowsCaptureItemsForSource(windowSource); + reuse.reuseWindowsCaptureItemsForSource({id: 'window:5678:0', name: 'Explorer'}); + assert.equal(reuse.calls.install, installs, `build ${windowsBuild}`); + assert.equal(reuse.calls.loads, installs, `build ${windowsBuild}`); + } + for (const platform of ['darwin', 'linux']) { + const reuse = loadReuse({platform}); + reuse.reuseWindowsCaptureItemsForSource(windowSource); + assert.equal(reuse.calls.loads, 0, platform); + } + }); + + test('reports the capture item requests made since the previous report', () => { + const reuse = loadReuse({}); + reuse.reuseWindowsCaptureItemsForSource(windowSource); + Object.assign(reuse.counts, {forwarded: 3, reused: 53999}); + reuse.reportWindowsCaptureItemReuse('streaming-priority-release'); + reuse.reportWindowsCaptureItemReuse('streaming-priority-reset'); + Object.assign(reuse.counts, {forwarded: 6, reused: 54009}); + reuse.reportWindowsCaptureItemReuse('streaming-priority-release'); + const reports = reuse.logged + .filter(([, message]) => message.includes('since the last report')) + .map(([, , fields]) => ({...fields})); + assert.deepEqual(reports, [ + {reason: 'streaming-priority-release', forwardedToCaptureService: 3, reusedFromLiveShare: 53999}, + {reason: 'streaming-priority-release', forwardedToCaptureService: 3, reusedFromLiveShare: 10}, + ]); + }); + + test('a failing addon never throws into the share request', () => { + const reuse = loadReuse({ + addon: { + loadError: null, + reuseWindowCaptureItems: () => { + throw new Error('binding exploded'); + }, + windowCaptureItemCalls: () => null, + }, + }); + assert.doesNotThrow(() => reuse.reuseWindowsCaptureItemsForSource(windowSource)); + assert.doesNotThrow(() => reuse.reuseWindowsCaptureItemsForSource(windowSource)); + assert.doesNotThrow(() => reuse.reportWindowsCaptureItemReuse('streaming-priority-release')); + assert.equal(reuse.logged.filter(([level]) => level === 'warn').length, 1); + }); +}); diff --git a/fluxer_desktop/src/main/WindowsWindowCaptureItemReuse.ts b/fluxer_desktop/src/main/WindowsWindowCaptureItemReuse.ts new file mode 100644 index 000000000..7ff34cc80 --- /dev/null +++ b/fluxer_desktop/src/main/WindowsWindowCaptureItemReuse.ts @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {createRequire} from 'node:module'; +import {isSafeModeLaunch} from '@electron/main/LaunchOptions'; +import {loadOptionalNativeAddon} from '@electron/main/NativeAddon'; +import {getWindowsBuildNumber} from '@electron/main/WindowsScreenCaptureGuard'; +import log from 'electron-log'; + +const WINDOWS_10_2004_BUILD = 19041; +const WINDOWS_10_22H2_BUILD = 19045; + +const requireModule = createRequire(import.meta.url); + +interface WindowCaptureItemCalls { + forwarded: number; + reused: number; +} + +interface WindowCaptureItemReuseModule { + reuseWindowCaptureItems: () => boolean; + windowCaptureItemCalls: () => WindowCaptureItemCalls | null; + loadError?: Error | null; +} + +const loadWindowCaptureItemReuseModule = loadOptionalNativeAddon({ + platform: 'win32', + disabled: () => isSafeModeLaunch(), + load: () => requireModule('@fluxer/win-game-capture'), + logger: log, + level: 'warn', + unavailableMessage: + '[WindowsWindowCaptureItemReuse] Failed to load @fluxer/win-game-capture; window shares create a capture item per frame', + disabledMessage: + '[WindowsWindowCaptureItemReuse] Disabled by safe mode; window shares create a capture item per frame', +}); + +let reuseRequested = false; +let reusingModule: WindowCaptureItemReuseModule | null = null; +let reportedCalls: WindowCaptureItemCalls = {forwarded: 0, reused: 0}; + +function reusesCaptureItemsOnWindowsBuild(windowsBuild: number | null): windowsBuild is number { + if (windowsBuild === null) return false; + return windowsBuild >= WINDOWS_10_2004_BUILD && windowsBuild <= WINDOWS_10_22H2_BUILD; +} + +export function reportWindowsCaptureItemReuse(reason: string): void { + const calls = reusingModule?.windowCaptureItemCalls(); + if (!calls) return; + const forwarded = calls.forwarded - reportedCalls.forwarded; + const reused = calls.reused - reportedCalls.reused; + if (forwarded === 0 && reused === 0) return; + reportedCalls = calls; + log.info('[WindowsWindowCaptureItemReuse] Capture item requests since the last report', { + reason, + forwardedToCaptureService: forwarded, + reusedFromLiveShare: reused, + }); +} + +export function reuseWindowsCaptureItemsForSource(source: Electron.DesktopCapturerSource): void { + try { + reuseCaptureItemsForSource(source); + } catch (error) { + log.warn('[WindowsWindowCaptureItemReuse] Window shares create a capture item per frame', {error}); + } +} + +function reuseCaptureItemsForSource(source: Electron.DesktopCapturerSource): void { + if (process.platform !== 'win32') return; + if (!source.id.startsWith('window:')) return; + if (reuseRequested) { + reportWindowsCaptureItemReuse('window-share-start'); + return; + } + const windowsBuild = getWindowsBuildNumber(); + if (!reusesCaptureItemsOnWindowsBuild(windowsBuild)) return; + reuseRequested = true; + const addon = loadWindowCaptureItemReuseModule(); + if (!addon) return; + if (!addon.reuseWindowCaptureItems()) { + log.warn('[WindowsWindowCaptureItemReuse] Window shares create a capture item per frame', {windowsBuild}); + return; + } + reusingModule = addon; + log.info('[WindowsWindowCaptureItemReuse] Window shares ask for their live capture item first', {windowsBuild}); +} diff --git a/tools/ci/check-desktop-native-workspaces.sh b/tools/ci/check-desktop-native-workspaces.sh index f4766b29e..7c6ffd75e 100755 --- a/tools/ci/check-desktop-native-workspaces.sh +++ b/tools/ci/check-desktop-native-workspaces.sh @@ -9,6 +9,7 @@ native_root="$repo_root/fluxer_desktop/native" cargo_bin="${CARGO_BIN:-cargo}" cargo_deny_bin="${CARGO_DENY_BIN:-cargo-deny}" native_target_dir="${DESKTOP_NATIVE_CARGO_TARGET_DIR:-$repo_root/target/desktop-native-workspaces}" +windows_backend_target="x86_64-pc-windows-msvc" mode="${1:-}" linux_workspaces=( @@ -130,6 +131,21 @@ run_clippy() { -D warnings } +run_clippy_windows_backend() { + local manifest="$1" + + CARGO_TARGET_DIR="$native_target_dir" \ + "$cargo_bin" clippy \ + --manifest-path "$manifest" \ + --workspace \ + --all-targets \ + --all-features \ + --locked \ + --target "$windows_backend_target" \ + -- \ + -D warnings +} + run_tests() { local manifest="$1" @@ -190,6 +206,7 @@ if [ "$mode" = clippy ] || [ "$mode" = test ]; then fi checked_count=0 +windows_backend_count=0 cd "$native_root" @@ -210,9 +227,14 @@ while IFS= read -r manifest; do case "$category" in linux) printf 'Clippy full Linux/shared coverage: %s\n' "$relative_manifest" ;; macos) printf 'Clippy Linux stub/common coverage; macOS backend not validated: %s\n' "$relative_manifest" ;; - windows) printf 'Clippy Linux stub/common coverage; Windows backend not validated: %s\n' "$relative_manifest" ;; + windows) printf 'Clippy Linux stub/common coverage: %s\n' "$relative_manifest" ;; esac run_clippy "$manifest" + if [ "$category" = windows ]; then + printf 'Clippy Windows backend coverage for %s: %s\n' "$windows_backend_target" "$relative_manifest" + run_clippy_windows_backend "$manifest" + windows_backend_count=$((windows_backend_count + 1)) + fi ;; test) case "$category" in @@ -227,5 +249,5 @@ while IFS= read -r manifest; do done <"$manifest_list" discovered_count=$((linux_count + macos_count + windows_count)) -printf 'Desktop native workspace categories: discovered=%s linux-full=%s macos-stub-common=%s windows-stub-common=%s\n' "$discovered_count" "$linux_count" "$macos_count" "$windows_count" -printf 'Desktop native %s summary: checked=%s\n' "$mode" "$checked_count" +printf 'Desktop native workspace categories: discovered=%s linux-full=%s macos-stub-common=%s windows=%s\n' "$discovered_count" "$linux_count" "$macos_count" "$windows_count" +printf 'Desktop native %s summary: checked=%s windows-backend=%s\n' "$mode" "$checked_count" "$windows_backend_count"