pepepper/fluxer
1
0 Fork 0
mirror of https://github.com/fluxerapp/fluxer synced 2026-10-17 04:55:42 +09:00
Code Issues Packages Projects Releases Wiki Activity

perf(repo): parallelise api tests and cache image builds in ghcr (#1984)

Browse Source
This commit is contained in:
Hampus authored and GitHub committed 2026-08-24 21:24:39 +02:00
1 parent 3afad20e36
commit c226eb7211
38 files changed
+701 -264
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/ci/src/app_proxy.rs
+12 -4
View File
@@ -147,13 +147,16 @@ fn build_and_extract_command() -> Result<CommandSpec> {
)
.env(
"CACHE_FROM",
env::var("CACHE_FROM")
.unwrap_or_else(|_| "type=gha,scope=fluxer-app-proxy".to_string()),
env::var("CACHE_FROM").unwrap_or_else(|_| default_app_proxy_cache_ref()),
)
.env(
"CACHE_TO",
env::var("CACHE_TO")
.unwrap_or_else(|_| "type=gha,scope=fluxer-app-proxy,mode=max".to_string()),
env::var("CACHE_TO").unwrap_or_else(|_| {
format!(
"{},mode=max,image-manifest=true,oci-mediatypes=true,ignore-error=true",
default_app_proxy_cache_ref()
)
}),
)
.env(
"DOCKER_BUILD_SUMMARY",
@@ -165,6 +168,11 @@ fn build_and_extract_command() -> Result<CommandSpec> {
))
}
fn default_app_proxy_cache_ref() -> String {
let owner = ghcr_owner().unwrap_or_else(|_| "fluxerapp".to_string());
format!("type=registry,ref=ghcr.io/{owner}/fluxer-app-proxy:buildcache-amd64")
}
fn ghcr_owner() -> Result<String> {
for key in ["GHCR_OWNER", "GITHUB_REPOSITORY_OWNER", "OWNER"] {
if let Ok(value) = env::var(key) {
tools/ci/src/app_wasm.rs
+1 -2
View File
@@ -142,8 +142,7 @@ fn build_markdown_parser_wasm(app_dir: &Path) -> Result<()> {
let rust_source_dir = app_dir.join("../packages/markdown_parser/rust");
let bytes_path =
app_dir.join("src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts");
let temp = TempDir::new().context("Failed to create source temp directory")?;
let target_dir = temp.path().join("target");
let target_dir = rust_source_dir.join("target");
run_command(apply_wasm_c_toolchain(
CommandSpec::new("cargo")
tools/ci/src/ci_workflow.rs
+110 -62
View File
@@ -6,6 +6,7 @@ use crate::gateway::{GatewayStep, run_gateway_step};
use anyhow::{Context, Result};
use clap::{Args, ValueEnum};
use std::env;
use std::ffi::OsString;
use std::path::{Path, PathBuf};
#[derive(Debug, Args, Clone)]
@@ -27,17 +28,10 @@ enum CiStep {
GatewayEunit,
}
#[derive(Debug, Args, Clone)]
pub struct CiScriptsArgs {
#[arg(long, value_enum)]
step: CiScriptsStep,
}
#[derive(Debug, Clone, Copy, ValueEnum)]
#[clap(rename_all = "snake_case")]
enum CiScriptsStep {
Sync,
Test,
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum AppWasm {
Build,
ReuseIfPresent,
}
pub async fn run_ci(args: CiArgs) -> Result<()> {
@@ -51,7 +45,7 @@ pub async fn run_ci(args: CiArgs) -> Result<()> {
CiStep::Typecheck => {
ensure_desktop_build_channel_file(&root)?;
run_generators(&root, true)?;
run_app_test_artifact_generators(&root)?;
run_app_test_artifact_generators(&root, AppWasm::Build)?;
run_command(
CommandSpec::new("pnpm")
.args(["-r", "--if-present", "typecheck"])
@@ -60,7 +54,7 @@ pub async fn run_ci(args: CiArgs) -> Result<()> {
}
CiStep::Test => {
run_generators(&root, false)?;
run_app_test_artifact_generators(&root)?;
run_app_test_artifact_generators(&root, AppWasm::ReuseIfPresent)?;
run_workspace_tests(&root)?;
run_command(with_test_env(
CommandSpec::new("pnpm")
@@ -69,7 +63,7 @@ pub async fn run_ci(args: CiArgs) -> Result<()> {
))
}
CiStep::Knip => {
run_app_test_artifact_generators(&root)?;
run_app_test_artifact_generators(&root, AppWasm::ReuseIfPresent)?;
ensure_desktop_build_channel_file(&root)?;
run_fluxer_app_script(&root, "i18n:compile")?;
run_command(
@@ -98,11 +92,43 @@ fn ensure_desktop_build_channel_file(root: &Path) -> Result<()> {
write_build_channel_file(&root.join("fluxer_desktop"), &channel)
}
fn run_app_test_artifact_generators(root: &Path) -> Result<()> {
run_fluxer_app_script(root, "wasm:codegen")?;
fn run_app_test_artifact_generators(root: &Path, wasm: AppWasm) -> Result<()> {
match (wasm, missing_app_wasm_artifact(root)) {
(AppWasm::ReuseIfPresent, None) => {
println!("Reusing restored fluxer_app wasm artifacts");
}
(AppWasm::ReuseIfPresent, Some(missing)) => {
println!(
"Rebuilding fluxer_app wasm artifacts: {} is missing",
missing.display()
);
run_fluxer_app_script(root, "wasm:codegen")?;
}
(AppWasm::Build, _) => run_fluxer_app_script(root, "wasm:codegen")?,
}
run_fluxer_app_script(root, "generate:masks")
}
fn app_wasm_artifacts(root: &Path) -> Vec<PathBuf> {
let app_dir = root.join("fluxer_app");
vec![
app_dir.join("pkgs/libfluxcore/libfluxcore.js"),
app_dir.join("pkgs/libfluxcore/libfluxcore.d.ts"),
app_dir.join("pkgs/libfluxcore/libfluxcore_bindgen.js"),
app_dir.join("pkgs/libfluxcore/libfluxcore_bindgen.d.ts"),
app_dir.join("pkgs/libfluxcore/libfluxcore_bg.wasm"),
app_dir.join("pkgs/libfluxcore/libfluxcore_bg.wasm.d.ts"),
app_dir.join("pkgs/libfluxcore/package.json"),
app_dir.join("src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts"),
]
}
fn missing_app_wasm_artifact(root: &Path) -> Option<PathBuf> {
app_wasm_artifacts(root)
.into_iter()
.find(|path| !path.exists())
}
fn run_fluxer_app_script(root: &Path, script: &str) -> Result<()> {
run_command(
CommandSpec::new("pnpm")
@@ -111,27 +137,6 @@ fn run_fluxer_app_script(root: &Path, script: &str) -> Result<()> {
)
}
pub async fn run_ci_scripts(args: CiScriptsArgs) -> Result<()> {
let root = repo_root()?;
match args.step {
CiScriptsStep::Sync => run_command(
CommandSpec::new("cargo")
.args([
"fetch",
"--locked",
"--manifest-path",
"tools/ci/Cargo.toml",
])
.current_dir(root),
),
CiScriptsStep::Test => run_command(
CommandSpec::new("cargo")
.args(["test", "--locked", "--manifest-path", "tools/ci/Cargo.toml"])
.current_dir(root),
),
}
}
fn run_generators(root: &Path, for_typecheck: bool) -> Result<()> {
for command in generator_commands(for_typecheck) {
run_command(command.current_dir(root))?;
@@ -155,28 +160,41 @@ fn generator_commands(for_typecheck: bool) -> Vec<CommandSpec> {
commands
}
fn workspace_test_args(concurrency: Option<&str>) -> Vec<OsString> {
let mut args = vec![OsString::from("-r")];
if let Some(concurrency) = concurrency {
args.push(OsString::from(format!(
"--workspace-concurrency={concurrency}"
)));
}
args.extend(
[
"--filter",
"!fluxer_api",
"--filter",
"!fluxer",
"--filter",
"!fluxer_desktop",
"--if-present",
"test",
]
.into_iter()
.map(OsString::from),
);
args
}
fn run_workspace_tests(root: &Path) -> Result<()> {
let workspace_concurrency =
env::var("PNPM_TEST_WORKSPACE_CONCURRENCY").unwrap_or_else(|_| "2".to_string());
let concurrency = env::var("PNPM_TEST_WORKSPACE_CONCURRENCY").ok();
run_command(with_test_env(
CommandSpec::new("pnpm")
.args([
"-r",
&format!("--workspace-concurrency={workspace_concurrency}"),
"--filter",
"!fluxer_api",
"--filter",
"!fluxer",
"--if-present",
"test",
])
.args(workspace_test_args(concurrency.as_deref()))
.current_dir(root),
))
}
fn with_test_env(spec: CommandSpec) -> CommandSpec {
let nats_url = env::var("FLUXER_NATS_URL").unwrap_or_else(|_| default_test_nats_url());
let api_workers = env::var("API_TEST_MAX_WORKERS").unwrap_or_else(|_| "2".to_string());
spec.env("FLUXER_NATS_URL", &nats_url)
.env(
"FLUXER_NATS_CORE_URL",
@@ -186,11 +204,6 @@ fn with_test_env(spec: CommandSpec) -> CommandSpec {
"FLUXER_NATS_JETSTREAM_URL",
env::var("FLUXER_NATS_JETSTREAM_URL").unwrap_or_else(|_| nats_url.clone()),
)
.env("API_TEST_MAX_WORKERS", &api_workers)
.env(
"API_TEST_MAX_CONCURRENCY",
env::var("API_TEST_MAX_CONCURRENCY").unwrap_or(api_workers),
)
}
fn default_test_nats_url() -> String {
@@ -211,7 +224,6 @@ fn repo_root() -> Result<PathBuf> {
#[cfg(test)]
mod tests {
use super::*;
use std::ffi::OsString;
#[test]
fn generator_commands_include_i18n_types_only_for_typecheck() {
@@ -237,7 +249,7 @@ mod tests {
}
#[test]
fn with_test_env_sets_all_nats_urls_and_concurrency() {
fn with_test_env_sets_all_nats_urls_and_leaves_worker_counts_to_vitest() {
let spec = with_test_env(CommandSpec::new("pnpm"));
let env = spec
.env
@@ -257,13 +269,49 @@ mod tests {
env.get(&OsString::from("FLUXER_NATS_JETSTREAM_URL")),
Some(&default_nats_url)
);
assert_eq!(
env.get(&OsString::from("API_TEST_MAX_WORKERS")),
Some(&OsString::from("2"))
assert!(!env.contains_key(&OsString::from("API_TEST_MAX_WORKERS")));
assert!(!env.contains_key(&OsString::from("API_TEST_MAX_CONCURRENCY")));
}
#[test]
fn workspace_tests_exclude_desktop_and_leave_concurrency_to_pnpm() {
let args = workspace_test_args(None);
assert_eq!(args[0], OsString::from("-r"));
assert!(
!args
.iter()
.any(|arg| arg.to_string_lossy().starts_with("--workspace-concurrency"))
);
assert!(args.windows(2).any(|pair| pair
== [
OsString::from("--filter"),
OsString::from("!fluxer_desktop")
]));
}
#[test]
fn workspace_tests_forward_an_explicit_concurrency_override() {
let args = workspace_test_args(Some("4"));
assert_eq!(args[1], OsString::from("--workspace-concurrency=4"));
}
#[test]
fn missing_app_wasm_artifact_reports_the_first_absent_output() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path();
assert_eq!(
env.get(&OsString::from("API_TEST_MAX_CONCURRENCY")),
Some(&OsString::from("2"))
missing_app_wasm_artifact(root),
Some(root.join("fluxer_app/pkgs/libfluxcore/libfluxcore.js"))
);
for path in app_wasm_artifacts(root) {
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, "").unwrap();
}
assert_eq!(missing_app_wasm_artifact(root), None);
}
}
tools/ci/src/desktop.rs
+10 -10
View File
@@ -122,14 +122,14 @@ const PLATFORMS: &[Platform] = &[
platform: "windows",
arch: "x64",
desktop_variant: DEFAULT_DESKTOP_VARIANT,
os: "blacksmith-32vcpu-windows-2025",
os: "windows-2025",
electron_arch: "x64",
},
Platform {
platform: "windows",
arch: "arm64",
desktop_variant: DEFAULT_DESKTOP_VARIANT,
os: "blacksmith-32vcpu-windows-2025",
os: "windows-2025",
electron_arch: "arm64",
},
Platform {
@@ -143,14 +143,14 @@ const PLATFORMS: &[Platform] = &[
platform: "linux",
arch: "x64",
desktop_variant: DEFAULT_DESKTOP_VARIANT,
os: "blacksmith-32vcpu-ubuntu-2404",
os: "ubuntu-24.04",
electron_arch: "x64",
},
Platform {
platform: "linux",
arch: "arm64",
desktop_variant: DEFAULT_DESKTOP_VARIANT,
os: "blacksmith-32vcpu-ubuntu-2404-arm",
os: "ubuntu-24.04-arm",
electron_arch: "arm64",
},
];
@@ -3520,9 +3520,9 @@ mod tests {
assert_eq!(
selected,
vec![
"{\"platform\":\"windows\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"blacksmith-32vcpu-windows-2025\",\"electron_arch\":\"arm64\"}",
"{\"platform\":\"linux\",\"arch\":\"x64\",\"desktop_variant\":\"default\",\"os\":\"blacksmith-32vcpu-ubuntu-2404\",\"electron_arch\":\"x64\"}",
"{\"platform\":\"linux\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"blacksmith-32vcpu-ubuntu-2404-arm\",\"electron_arch\":\"arm64\"}",
"{\"platform\":\"windows\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"windows-2025\",\"electron_arch\":\"arm64\"}",
"{\"platform\":\"linux\",\"arch\":\"x64\",\"desktop_variant\":\"default\",\"os\":\"ubuntu-24.04\",\"electron_arch\":\"x64\"}",
"{\"platform\":\"linux\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"ubuntu-24.04-arm\",\"electron_arch\":\"arm64\"}",
]
);
}
@@ -3560,9 +3560,9 @@ mod tests {
assert_eq!(
selected,
vec![
"{\"platform\":\"windows\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"blacksmith-32vcpu-windows-2025\",\"electron_arch\":\"arm64\"}",
"{\"platform\":\"linux\",\"arch\":\"x64\",\"desktop_variant\":\"default\",\"os\":\"blacksmith-32vcpu-ubuntu-2404\",\"electron_arch\":\"x64\"}",
"{\"platform\":\"linux\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"blacksmith-32vcpu-ubuntu-2404-arm\",\"electron_arch\":\"arm64\"}",
"{\"platform\":\"windows\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"windows-2025\",\"electron_arch\":\"arm64\"}",
"{\"platform\":\"linux\",\"arch\":\"x64\",\"desktop_variant\":\"default\",\"os\":\"ubuntu-24.04\",\"electron_arch\":\"x64\"}",
"{\"platform\":\"linux\",\"arch\":\"arm64\",\"desktop_variant\":\"default\",\"os\":\"ubuntu-24.04-arm\",\"electron_arch\":\"arm64\"}",
]
);
}
tools/ci/src/lib.rs
-2
View File
@@ -34,7 +34,6 @@ enum Command {
BuildDesktopNativeAddon(desktop_native::BuildDesktopNativeAddonArgs),
BuildGatewayNifs(gateway::BuildGatewayNifsArgs),
Ci(ci_workflow::CiArgs),
CiScripts(ci_workflow::CiScriptsArgs),
CleanSchemaGeneratedFiles(schema::CleanSchemaGeneratedFilesArgs),
Gateway(gateway::GatewayArgs),
Release(release::ReleaseArgs),
@@ -53,7 +52,6 @@ pub async fn run() -> Result<()> {
}
Command::BuildGatewayNifs(args) => gateway::run_build_gateway_nifs(args),
Command::Ci(args) => ci_workflow::run_ci(args).await,
Command::CiScripts(args) => ci_workflow::run_ci_scripts(args).await,
Command::CleanSchemaGeneratedFiles(args) => schema::run_clean_generated_files(args),
Command::Gateway(args) => gateway::run_gateway(args),
Command::Release(args) => release::run(args).await,