From 4ce4d4e09a79892087e6bf942a71aac75be29c4d Mon Sep 17 00:00:00 2001 From: Hampus Date: Fri, 9 Oct 2026 02:31:37 +0200 Subject: [PATCH] feat(ci): bundle the renderer in the desktop shell (#3339) --- .github/workflows/build-desktop.yaml | 10 + fluxer_desktop/electron-builder.config.cjs | 1 + fluxer_desktop/scripts/build.mjs | 21 +- tools/ci/src/desktop.rs | 257 +++++++++++++++++---- 4 files changed, 240 insertions(+), 49 deletions(-) diff --git a/.github/workflows/build-desktop.yaml b/.github/workflows/build-desktop.yaml index 60b9cf6aa..5ca7ab1d9 100644 --- a/.github/workflows/build-desktop.yaml +++ b/.github/workflows/build-desktop.yaml @@ -591,6 +591,16 @@ jobs: cargo run --locked --quiet --manifest-path ${{ github.workspace }}/_ci/tools/ci/Cargo.toml -- build-desktop --step build_app_linux + - name: Verify the packaged shell boots its bundled renderer (Linux) + if: matrix.platform == 'linux' && matrix.arch == 'x64' + working-directory: ${{ env.WORKDIR }}/fluxer_desktop + env: + BUILD_CHANNEL: ${{ env.BUILD_CHANNEL }} + BUILD_VERSION: ${{ env.VERSION }} + run: >- + cargo run --locked --quiet --manifest-path ${{ github.workspace }}/_ci/tools/ci/Cargo.toml -- build-desktop + --step verify_bundled_renderer_linux + - name: Verify signed Windows artifacts if: matrix.platform == 'windows' working-directory: ${{ env.WORKDIR }}/fluxer_desktop diff --git a/fluxer_desktop/electron-builder.config.cjs b/fluxer_desktop/electron-builder.config.cjs index 31f26aa18..dda59ceb6 100644 --- a/fluxer_desktop/electron-builder.config.cjs +++ b/fluxer_desktop/electron-builder.config.cjs @@ -1596,6 +1596,7 @@ module.exports = { smartUnpack: false, unpack: [ '**/*.node', + 'dist/renderer/**/*', 'node_modules/@fluxer/win-process-loopback/*.node', ...winGameCaptureTargetArchs.map( (arch) => `node_modules/@fluxer/win-game-capture/win-game-capture.win32-${arch}-msvc.node`, diff --git a/fluxer_desktop/scripts/build.mjs b/fluxer_desktop/scripts/build.mjs index a0ab69ad6..4654b5e59 100644 --- a/fluxer_desktop/scripts/build.mjs +++ b/fluxer_desktop/scripts/build.mjs @@ -22,6 +22,7 @@ const SPLASH_SRC_DIR = path.join(SRC_DIR, 'splash'); const SPLASH_DIST_DIR = path.join(DIST_DIR, 'splash'); const SPLASH_PRELOAD_FILE_NAME = 'splash.cjs'; const REQUIRED_RENDERER_ENTRIES = Object.freeze(['index.html', 'assets']); +const BUNDLED_RENDERER_ENTRIES = Object.freeze([...REQUIRED_RENDERER_ENTRIES, 'version.json']); const FORBIDDEN_RENDERER_ENTRIES = Object.freeze(['sw.js', 'sw.js.map']); const SUPPORTED_BUILD_ARGUMENTS = Object.freeze(['--shared-assets', '--use-shared-renderer']); const MAIN_BOOTSTRAP_ENTRY_NAME = 'index'; @@ -89,10 +90,10 @@ class RendererOutputIncompleteError extends Error { } } -class PackedRendererNotPrunedError extends Error { - constructor(location) { - super(`The renderer is a required module, so ${location} must be empty after the prune step.`); - this.name = 'PackedRendererNotPrunedError'; +class BundledRendererMissingError extends Error { + constructor(location, missing) { + super(`The shell bundles its renderer, so ${location} must still hold ${missing.join(', ')} after the prune step.`); + this.name = 'BundledRendererMissingError'; } } @@ -839,14 +840,14 @@ function ensureBuildChannelFile() { } function prunePackedRendererModules() { - console.log('Pruning module-owned assets from the packed renderer...'); + console.log('Pruning on-demand module assets from the bundled renderer...'); const startedAt = Date.now(); - runDesktopBuildStep('strip_shell_renderer'); - const survivors = findMissingEntries(REQUIRED_RENDERER_ENTRIES, RENDERER_DIST_DIR); - if (survivors.length !== REQUIRED_RENDERER_ENTRIES.length) { - throw new PackedRendererNotPrunedError(RENDERER_DIST_DIR); + runDesktopBuildStep('prune_shell_renderer'); + const missing = findMissingEntries(BUNDLED_RENDERER_ENTRIES, RENDERER_DIST_DIR); + if (missing.length > 0) { + throw new BundledRendererMissingError(RENDERER_DIST_DIR, missing); } - console.log(` Packed renderer pruned in ${Date.now() - startedAt}ms`); + console.log(` Bundled renderer pruned in ${Date.now() - startedAt}ms`); } async function build() { diff --git a/tools/ci/src/desktop.rs b/tools/ci/src/desktop.rs index adcb0d5f0..248b3223a 100644 --- a/tools/ci/src/desktop.rs +++ b/tools/ci/src/desktop.rs @@ -89,6 +89,8 @@ const DESKTOP_DICTIONARY_MODULE_PREFIX: &str = "fluxer_dict_"; const DESKTOP_DICTIONARY_PACKAGE_PREFIX: &str = "dictionary-"; const DESKTOP_DICTIONARY_FILE_NAMES: &[&str] = &["index.aff", "index.dic"]; const DESKTOP_MODULE_MINIMUM_SHELL_VERSION: &str = "0.0.0"; +const DESKTOP_CONTENT_MODULE_BUILD_VERSION: &str = "0.0.0"; +const DESKTOP_RENDERER_VERSION_FILE_NAME: &str = "version.json"; const DESKTOP_REQUIRED_MODULES: &[&str] = &[DESKTOP_RENDERER_MODULE]; const DESKTOP_MODULES_ENV: &str = "FLUXER_MODULES"; @@ -140,7 +142,8 @@ enum DesktopStep { RestoreSharedAssets, SplitModules, PackModules, - StripShellRenderer, + PruneShellRenderer, + VerifyBundledRendererLinux, BuildElectronMain, InstallVelopackCli, BuildAppMacos, @@ -251,7 +254,8 @@ pub async fn run(args: BuildDesktopArgs) -> Result<()> { DesktopStep::RestoreSharedAssets => restore_shared_assets_step(), DesktopStep::SplitModules => split_modules_step(), DesktopStep::PackModules => pack_modules_step(), - DesktopStep::StripShellRenderer => strip_shell_renderer_step(), + DesktopStep::PruneShellRenderer => prune_shell_renderer_step(), + DesktopStep::VerifyBundledRendererLinux => verify_bundled_renderer_linux_step(), DesktopStep::BuildElectronMain => build_electron_main_step(), DesktopStep::InstallVelopackCli => install_velopack_cli_step(), DesktopStep::BuildAppMacos => build_app_step(DesktopBuildPlatform::Macos), @@ -826,6 +830,7 @@ DPkg::Lock::Timeout "120"; "libclang-dev", "clang", "libpulse-dev", + "xvfb", ])?; install_linux_pipewire_headers().await?; install_linux_libfido2().await?; @@ -1544,6 +1549,12 @@ fn restore_shared_assets_step() -> Result<()> { remove_dir_if_exists(&renderer_dir)?; copy_dir_contents(&payload_dir, &renderer_dir)?; ensure_renderer_assets_ready(&renderer_dir)?; + let bundled_version = read_bundled_renderer_version(&renderer_dir)?; + ensure!( + bundled_version == build_version, + "The shared renderer in {} reports version {bundled_version}, this shell is {build_version}", + renderer_dir.display() + ); println!( "Restored {} shared renderer file(s) into {}", manifest.files.len(), @@ -1817,44 +1828,110 @@ fn classify_renderer_files(renderer_dir: &Path) -> Result bool { + module != DESKTOP_RENDERER_MODULE && module != DESKTOP_SOURCEMAP_MODULE +} + +fn desktop_content_module_source_sha(files: &[DesktopSharedAssetFile]) -> String { + let mut entries = files + .iter() + .map(|entry| (entry.path.as_str(), entry.sha256.as_str(), entry.bytes)) + .collect::>(); + entries.sort(); + let mut digest = Sha256::new(); + for (path, sha256, bytes) in entries { + digest.update(path.as_bytes()); + digest.update([0]); + digest.update(sha256.as_bytes()); + digest.update([0]); + digest.update(bytes.to_string().as_bytes()); + digest.update(b"\n"); + } + hex::encode(digest.finalize())[..40].to_string() +} + +fn desktop_module_manifest( + module: &str, + build_version: &str, + release_channel: &str, + source_sha: &str, + files: Vec, +) -> DesktopModuleManifest { + if is_content_addressed_desktop_module(module) { + return DesktopModuleManifest { + module: module.to_string(), + build_version: DESKTOP_CONTENT_MODULE_BUILD_VERSION.to_string(), + release_channel: release_channel.to_string(), + source_sha: desktop_content_module_source_sha(&files), + files, + }; + } + DesktopModuleManifest { + module: module.to_string(), + build_version: build_version.to_string(), + release_channel: release_channel.to_string(), + source_sha: source_sha.to_string(), + files, + } +} + +fn read_bundled_renderer_version(renderer_dir: &Path) -> Result { + let path = renderer_dir.join(DESKTOP_RENDERER_VERSION_FILE_NAME); + let bytes = fs::read(&path).with_context(|| { + format!( + "Failed to read {}, the shell picks between its bundled renderer and an installed module by this version", + path.display() + ) + })?; + let value: Value = serde_json::from_slice(&bytes) + .with_context(|| format!("Failed to parse {}", path.display()))?; + let version = value + .get("version") + .and_then(Value::as_str) + .map(str::trim) + .filter(|version| !version.is_empty()) + .with_context(|| format!("{} names no version", path.display()))?; + Ok(version.to_string()) +} + +fn prune_on_demand_modules_from_renderer_tree( renderer_dir: &Path, ) -> Result>> { ensure_renderer_assets_ready(renderer_dir)?; let mut removed: BTreeMap> = BTreeMap::new(); + let mut kept = 0usize; for file in classify_renderer_files(renderer_dir)? { + if file.module == DESKTOP_RENDERER_MODULE { + kept += 1; + continue; + } fs::remove_file(&file.source) .with_context(|| format!("Failed to remove {}", file.source.display()))?; removed.entry(file.module).or_default().push(file.entry); } ensure!( - removed - .get(DESKTOP_RENDERER_MODULE) - .is_some_and(|entries| !entries.is_empty()), - "{DESKTOP_RENDERER_MODULE} claimed no file in {}, refusing to publish a shell whose renderer module is empty", - renderer_dir.display() - ); - let left = count_files(renderer_dir)?; - ensure!( - left == 0, - "{left} renderer file(s) survived the strip in {}, every packed file must belong to a module", + kept > 0, + "{DESKTOP_RENDERER_MODULE} claimed no file in {}, refusing to package a shell with no bundled renderer", renderer_dir.display() ); remove_empty_dirs_below(renderer_dir)?; - let surviving_entries = fs::read_dir(renderer_dir) - .with_context(|| format!("Failed to read {}", renderer_dir.display()))? - .count(); + ensure_renderer_assets_ready(renderer_dir)?; + let survivors = classify_renderer_files(renderer_dir)?; ensure!( - surviving_entries == 0, - "{surviving_entries} entries survived the strip in {}, the packed renderer directory must be left empty", - renderer_dir.display() + survivors.len() == kept + && survivors + .iter() + .all(|file| file.module == DESKTOP_RENDERER_MODULE), + "{} keeps {} file(s) after the prune, {kept} belong to {DESKTOP_RENDERER_MODULE}", + renderer_dir.display(), + survivors.len() ); Ok(removed) } -fn strip_shell_renderer_step() -> Result<()> { +fn prune_shell_renderer_step() -> Result<()> { let renderer_dir = desktop_renderer_dir(); - let removed = strip_renderer_tree_owned_by_modules(&renderer_dir)?; + let removed = prune_on_demand_modules_from_renderer_tree(&renderer_dir)?; let mut total_files = 0usize; let mut total_bytes = 0u64; for (module, entries) in &removed { @@ -1866,9 +1943,47 @@ fn strip_shell_renderer_step() -> Result<()> { entries.len() ); } + let bundled_version = read_bundled_renderer_version(&renderer_dir)?; println!( - "Stripped {total_files} module file(s), {total_bytes} bytes from {}, the shell now ships no renderer", - renderer_dir.display() + "Pruned {total_files} on-demand module file(s), {total_bytes} bytes from {}, the shell bundles renderer {bundled_version} in {} file(s)", + renderer_dir.display(), + count_files(&renderer_dir)? + ); + Ok(()) +} + +fn verify_bundled_renderer_linux_step() -> Result<()> { + let channel = require_env("BUILD_CHANNEL")?; + let version = require_env("BUILD_VERSION")?; + let dist = Path::new("dist-electron"); + let unpacked = sorted_child_directories(dist)? + .into_iter() + .find(|entry| { + file_name_string(entry) + .is_ok_and(|name| name.starts_with("linux") && name.ends_with("unpacked")) + }) + .with_context(|| format!("No unpacked Linux app found in {}", dist.display()))?; + run_command( + CommandSpec::new("xvfb-run") + .args(["-a", "node", "scripts/release-check.mjs", "--app"]) + .arg(unpacked.as_os_str()) + .args([ + "--channel", + channel.as_str(), + "--expect-renderer", + version.as_str(), + "--expect-source", + "bundled", + "--package-origin", + "http://127.0.0.1:9", + "--app-arg=--no-sandbox", + "--app-arg=--disable-gpu", + "--timeout-seconds", + "180", + ]), + )?; + println!( + "The packaged {channel} shell {version} boots its bundled renderer with no package feed" ); Ok(()) } @@ -1922,13 +2037,13 @@ fn split_modules_step() -> Result<()> { } write_json_pretty( &module_dir.join(DESKTOP_MODULE_FILE_LIST_NAME), - &DesktopModuleManifest { - module: module.clone(), - build_version: build_version.clone(), - release_channel: release_channel.clone(), - source_sha: source_sha.clone(), - files: entries.iter().map(|(_, entry)| entry.clone()).collect(), - }, + &desktop_module_manifest( + module, + &build_version, + &release_channel, + &source_sha, + entries.iter().map(|(_, entry)| entry.clone()).collect(), + ), )?; summaries.push(DesktopModuleClassificationSummary { module: module.clone(), @@ -2071,13 +2186,8 @@ fn write_desktop_dictionary_modules( .len(), }); } - let manifest = DesktopModuleManifest { - module, - build_version: build_version.to_string(), - release_channel: release_channel.to_string(), - source_sha: source_sha.to_string(), - files, - }; + let manifest = + desktop_module_manifest(&module, build_version, release_channel, source_sha, files); write_json_pretty(&module_dir.join(DESKTOP_MODULE_FILE_LIST_NAME), &manifest)?; manifests.push(manifest); } @@ -5151,9 +5261,14 @@ fn read_packed_desktop_modules( manifest.module, manifest.release_channel ); + let expected_version = if is_content_addressed_desktop_module(&manifest.module) { + DESKTOP_CONTENT_MODULE_BUILD_VERSION + } else { + build_version + }; ensure!( - manifest.build_version == build_version, - "Desktop module {} was packed for version {}, expected {build_version}", + manifest.build_version == expected_version, + "Desktop module {} was packed for version {}, expected {expected_version}", manifest.module, manifest.build_version ); @@ -7030,6 +7145,70 @@ export const CHANNEL_DISPLAY_NAME = BUILD_CHANNEL;\n" assert_eq!(steps[manifest + 1], "Prepare GitHub release assets"); } + #[test] + fn pruning_the_shell_renderer_keeps_the_renderer_and_drops_on_demand_modules() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("renderer"); + write_file(&root.join("index.html"), ""); + write_file(&root.join("version.json"), "{\"version\":\"2026.1009.1\"}"); + write_file(&root.join("assets/main.js"), "main"); + write_file(&root.join("assets/main.js.map"), "{}"); + write_file(&root.join("assets/fluxer_fonts_jp/a.woff2"), "font"); + write_file(&root.join("assets/fluxer_grammar_rust/b.wasm"), "wasm"); + + let removed = prune_on_demand_modules_from_renderer_tree(&root).unwrap(); + + assert_eq!( + removed.keys().map(String::as_str).collect::>(), + vec![ + "fluxer_fonts_jp", + "fluxer_grammar_rust", + "fluxer_sourcemaps" + ] + ); + assert!(root.join("index.html").is_file()); + assert!(root.join("assets/main.js").is_file()); + assert!(!root.join("assets/main.js.map").exists()); + assert!(!root.join("assets/fluxer_fonts_jp").exists()); + assert_eq!(read_bundled_renderer_version(&root).unwrap(), "2026.1009.1"); + } + + #[test] + fn on_demand_module_manifests_do_not_change_between_builds() { + let files = vec![DesktopSharedAssetFile { + path: "assets/fluxer_fonts_jp/a.woff2".to_string(), + sha256: "a".repeat(64), + bytes: 4, + }]; + let first = desktop_module_manifest( + "fluxer_fonts_jp", + "2026.1009.1", + "canary", + &"1".repeat(40), + files.clone(), + ); + let second = desktop_module_manifest( + "fluxer_fonts_jp", + "2026.1010.7", + "canary", + &"2".repeat(40), + files.clone(), + ); + assert_eq!(first, second); + assert_eq!(first.build_version, DESKTOP_CONTENT_MODULE_BUILD_VERSION); + assert_eq!(first.source_sha.len(), 40); + + let renderer = desktop_module_manifest( + DESKTOP_RENDERER_MODULE, + "2026.1009.1", + "canary", + &"1".repeat(40), + files, + ); + assert_eq!(renderer.build_version, "2026.1009.1"); + assert_eq!(renderer.source_sha, "1".repeat(40)); + } + #[test] fn renderer_files_split_by_module_segment_and_source_map() { let temp = tempfile::tempdir().unwrap();