mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-07 19:22:14 +09:00
feat(ci): publish linux repositories from the desktop release (#2847)
This commit is contained in:
@@ -11,11 +11,6 @@ on:
|
||||
- stable
|
||||
- canary
|
||||
default: stable
|
||||
test_build:
|
||||
description: Stash artifacts under desktop-test/ instead of desktop/ (API will not pick these up as a release).
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
build_version:
|
||||
description: Explicit Fluxer CalVer build version (YYYY.MDD.MICRO, UTC HHMMSS without leading zeroes) to use instead of automatic UTC clock allocation.
|
||||
required: false
|
||||
@@ -32,13 +27,12 @@ permissions:
|
||||
actions: read
|
||||
|
||||
concurrency:
|
||||
group: desktop-${{ inputs.channel }}-${{ inputs.test_build && 'test' || 'release' }}
|
||||
group: desktop-${{ inputs.channel }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CHANNEL: ${{ inputs.channel }}
|
||||
BUILD_CHANNEL: ${{ inputs.channel == 'canary' && 'canary' || 'stable' }}
|
||||
TEST_BUILD: ${{ inputs.test_build && 'true' || 'false' }}
|
||||
|
||||
jobs:
|
||||
meta:
|
||||
@@ -53,8 +47,6 @@ jobs:
|
||||
pub_date: ${{ steps.meta.outputs.pub_date }}
|
||||
channel: ${{ steps.meta.outputs.channel }}
|
||||
build_channel: ${{ steps.meta.outputs.build_channel }}
|
||||
test_build: ${{ steps.meta.outputs.test_build }}
|
||||
s3_prefix: ${{ steps.meta.outputs.s3_prefix }}
|
||||
source_sha: ${{ steps.meta.outputs.source_sha }}
|
||||
steps:
|
||||
- name: Checkout source
|
||||
@@ -85,7 +77,6 @@ jobs:
|
||||
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
|
||||
--step set_metadata
|
||||
--channel "${{ inputs.channel }}"
|
||||
--test-build "${{ inputs.test_build }}"
|
||||
|
||||
matrix:
|
||||
name: Resolve build matrix
|
||||
@@ -113,7 +104,7 @@ jobs:
|
||||
--skip-targets "${{ inputs.skip_targets }}"
|
||||
|
||||
build:
|
||||
name: Build ${{ matrix.platform }} (${{ matrix.arch }}, ${{ matrix.desktop_variant }})
|
||||
name: Build ${{ matrix.platform }} (${{ matrix.arch }})
|
||||
needs:
|
||||
- meta
|
||||
- matrix
|
||||
@@ -137,15 +128,8 @@ jobs:
|
||||
PUBLIC_BUILD_VERSION: ${{ needs.meta.outputs.version }}
|
||||
PUB_DATE: ${{ needs.meta.outputs.pub_date }}
|
||||
SOURCE_SHA: ${{ needs.meta.outputs.source_sha }}
|
||||
S3_DESKTOP_PREFIX: ${{ needs.meta.outputs.s3_prefix }}
|
||||
DESKTOP_HANDOFF_PREFIX: _handoff/desktop/${{ needs.meta.outputs.build_channel }}/${{ needs.meta.outputs.version }}/${{ needs.meta.outputs.source_sha }}
|
||||
S3_ENDPOINT: ${{ vars.DOWNLOADS_S3_ENDPOINT }}
|
||||
S3_BUCKET: ${{ vars.DOWNLOADS_S3_BUCKET }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.DOWNLOADS_AWS_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.DOWNLOADS_AWS_SECRET_ACCESS_KEY || secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
DESKTOP_PLATFORM: ${{ matrix.platform }}
|
||||
DESKTOP_ARCH: ${{ matrix.arch }}
|
||||
DESKTOP_VARIANT: ${{ matrix.desktop_variant }}
|
||||
PLATFORM: ${{ matrix.platform }}
|
||||
ARCH: ${{ matrix.arch }}
|
||||
ELECTRON_ARCH: ${{ matrix.electron_arch }}
|
||||
@@ -495,13 +479,23 @@ jobs:
|
||||
cargo run --locked --quiet --manifest-path ${{ github.workspace }}/_ci/tools/ci/Cargo.toml -- build-desktop
|
||||
--step generate_checksums_windows
|
||||
|
||||
- name: Upload artifacts to S3 handoff
|
||||
- name: Stage build artifacts
|
||||
id: handoff
|
||||
run: >-
|
||||
cargo run --locked --quiet --manifest-path ${{ github.workspace }}/_ci/tools/ci/Cargo.toml -- build-desktop
|
||||
--step upload_handoff
|
||||
--step stage_handoff
|
||||
|
||||
- name: Upload build artifacts
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
|
||||
with:
|
||||
name: ${{ steps.handoff.outputs.artifact_name }}
|
||||
path: upload_staging
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
compression-level: 0
|
||||
|
||||
upload:
|
||||
name: Upload to S3
|
||||
name: Assemble desktop release assets
|
||||
if: ${{ !cancelled() && needs.build.result == 'success' }}
|
||||
needs:
|
||||
- meta
|
||||
@@ -520,17 +514,8 @@ jobs:
|
||||
BUILD_VERSION: ${{ needs.meta.outputs.version }}
|
||||
PUBLIC_BUILD_VERSION: ${{ needs.meta.outputs.version }}
|
||||
PUB_DATE: ${{ needs.meta.outputs.pub_date }}
|
||||
TEST_BUILD: ${{ needs.meta.outputs.test_build }}
|
||||
SOURCE_SHA: ${{ needs.meta.outputs.source_sha }}
|
||||
S3_DESKTOP_PREFIX: ${{ needs.meta.outputs.s3_prefix }}
|
||||
DESKTOP_HANDOFF_PREFIX: _handoff/desktop/${{ needs.meta.outputs.build_channel }}/${{ needs.meta.outputs.version }}/${{ needs.meta.outputs.source_sha }}
|
||||
DESKTOP_RELEASE_ASSETS_PREFIX: _handoff/desktop-release-assets/${{ needs.meta.outputs.build_channel }}/${{ needs.meta.outputs.version }}/${{ needs.meta.outputs.source_sha }}
|
||||
DESKTOP_METADATA_PREFIX: _handoff/desktop-metadata/${{ needs.meta.outputs.build_channel }}/${{ needs.meta.outputs.version }}/${{ needs.meta.outputs.source_sha }}
|
||||
S3_ENDPOINT: ${{ vars.DOWNLOADS_S3_ENDPOINT }}
|
||||
S3_BUCKET: ${{ vars.DOWNLOADS_S3_BUCKET }}
|
||||
PUBLIC_DL_BASE: https://api.fluxer.app/dl
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.DOWNLOADS_AWS_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.DOWNLOADS_AWS_SECRET_ACCESS_KEY || secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
steps:
|
||||
- name: Checkout source
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
@@ -542,12 +527,13 @@ jobs:
|
||||
with:
|
||||
toolchain: "1.98.1"
|
||||
|
||||
- name: Download S3 handoff artifacts
|
||||
run: >-
|
||||
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
|
||||
--step download_handoff
|
||||
- name: Download build artifacts
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
|
||||
with:
|
||||
path: artifacts
|
||||
pattern: fluxer-desktop-${{ needs.meta.outputs.build_channel }}-*
|
||||
|
||||
- name: Build S3 payload layout (+ manifest.json)
|
||||
- name: Build payload layout (+ manifest.json)
|
||||
env:
|
||||
VERSION: ${{ needs.meta.outputs.version }}
|
||||
PUB_DATE: ${{ needs.meta.outputs.pub_date }}
|
||||
@@ -556,42 +542,27 @@ jobs:
|
||||
--step build_payload
|
||||
|
||||
- name: Prepare GitHub release assets
|
||||
if: needs.meta.outputs.test_build != 'true'
|
||||
run: >-
|
||||
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
|
||||
--step prepare_release_assets
|
||||
|
||||
- name: Publish GitHub release descriptor
|
||||
if: needs.meta.outputs.test_build != 'true'
|
||||
run: >-
|
||||
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
|
||||
--step publish_release_descriptor
|
||||
|
||||
- name: Upload payload to S3
|
||||
run: >-
|
||||
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
|
||||
--step upload_payload
|
||||
|
||||
- name: Upload GitHub release asset handoff
|
||||
if: needs.meta.outputs.test_build != 'true'
|
||||
run: >-
|
||||
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
|
||||
--step upload_release_assets
|
||||
- name: Upload GitHub release assets
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
|
||||
with:
|
||||
name: fluxer-desktop-release-assets
|
||||
path: release_assets
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
compression-level: 0
|
||||
|
||||
- name: Build summary
|
||||
run: >-
|
||||
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
|
||||
--step build_summary
|
||||
|
||||
- name: Cleanup S3 handoff
|
||||
if: ${{ success() }}
|
||||
run: >-
|
||||
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
|
||||
--step cleanup_handoff
|
||||
|
||||
publish_release:
|
||||
name: Publish GitHub desktop release
|
||||
if: ${{ !cancelled() && needs.upload.result == 'success' && needs.meta.outputs.test_build != 'true' }}
|
||||
if: ${{ !cancelled() && needs.upload.result == 'success' }}
|
||||
needs:
|
||||
- meta
|
||||
- upload
|
||||
@@ -603,13 +574,6 @@ jobs:
|
||||
env:
|
||||
CHANNEL: ${{ needs.meta.outputs.build_channel }}
|
||||
VERSION: ${{ needs.meta.outputs.version }}
|
||||
S3_DESKTOP_PREFIX: ${{ needs.meta.outputs.s3_prefix }}
|
||||
DESKTOP_RELEASE_ASSETS_PREFIX: _handoff/desktop-release-assets/${{ needs.meta.outputs.build_channel }}/${{ needs.meta.outputs.version }}/${{ needs.meta.outputs.source_sha }}
|
||||
DESKTOP_METADATA_PREFIX: _handoff/desktop-metadata/${{ needs.meta.outputs.build_channel }}/${{ needs.meta.outputs.version }}/${{ needs.meta.outputs.source_sha }}
|
||||
S3_ENDPOINT: ${{ vars.DOWNLOADS_S3_ENDPOINT }}
|
||||
S3_BUCKET: ${{ vars.DOWNLOADS_S3_BUCKET }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.DOWNLOADS_AWS_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.DOWNLOADS_AWS_SECRET_ACCESS_KEY || secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
steps:
|
||||
- name: Checkout source
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
@@ -622,9 +586,10 @@ jobs:
|
||||
toolchain: "1.98.1"
|
||||
|
||||
- name: Download GitHub release assets
|
||||
run: >-
|
||||
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
|
||||
--step download_release_assets
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
|
||||
with:
|
||||
name: fluxer-desktop-release-assets
|
||||
path: release_assets
|
||||
|
||||
- name: Create token
|
||||
id: create-token
|
||||
@@ -656,15 +621,3 @@ jobs:
|
||||
release_args+=(--prerelease)
|
||||
fi
|
||||
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- "${release_args[@]}"
|
||||
|
||||
- name: Publish GitHub release readiness marker
|
||||
env:
|
||||
SOURCE_SHA: ${{ needs.meta.outputs.source_sha }}
|
||||
run: >-
|
||||
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
|
||||
--step publish_release_marker
|
||||
|
||||
- name: Publish payload metadata to S3
|
||||
run: >-
|
||||
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
|
||||
--step publish_payload_metadata
|
||||
|
||||
@@ -442,25 +442,6 @@ pub(crate) async fn s3_client(default_endpoint: Option<&str>) -> Result<S3Client
|
||||
Ok(S3Client::from_conf(s3_config.build()))
|
||||
}
|
||||
|
||||
pub(crate) async fn upload_directory_to_s3<F>(
|
||||
client: &S3Client,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
root: &Path,
|
||||
include: F,
|
||||
) -> Result<()>
|
||||
where
|
||||
F: Fn(&Path) -> bool,
|
||||
{
|
||||
let plan = directory_upload_plan(prefix, root, include)?;
|
||||
let stats = upload_s3_plan_append_only(client, bucket, plan).await?;
|
||||
println!(
|
||||
"Append-only upload complete for s3://{bucket}/{prefix}: uploaded {}, skipped existing {}",
|
||||
stats.uploaded, stats.skipped_existing
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn upload_s3_plan_append_only(
|
||||
client: &S3Client,
|
||||
bucket: &str,
|
||||
@@ -528,47 +509,6 @@ pub(crate) async fn upload_s3_plan_append_only(
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
pub(crate) async fn upload_s3_plan_overwrite(
|
||||
client: &S3Client,
|
||||
bucket: &str,
|
||||
plan: Vec<S3UploadPlanItem>,
|
||||
) -> Result<S3UploadStats> {
|
||||
ensure_unique_s3_keys(&plan)?;
|
||||
let concurrency = s3_write_concurrency();
|
||||
let semaphore = Arc::new(Semaphore::new(concurrency));
|
||||
let bucket = bucket.to_string();
|
||||
let mut tasks = JoinSet::new();
|
||||
for item in plan {
|
||||
let permit = semaphore
|
||||
.clone()
|
||||
.acquire_owned()
|
||||
.await
|
||||
.context("S3 upload semaphore closed")?;
|
||||
let client = client.clone();
|
||||
let bucket = bucket.clone();
|
||||
tasks.spawn(async move {
|
||||
let _permit = permit;
|
||||
put_file_to_s3_overwrite(&client, &bucket, &item)
|
||||
.await
|
||||
.with_context(|| {
|
||||
format!("Failed overwrite upload for s3://{}/{}", bucket, item.key)
|
||||
})?;
|
||||
Ok::<_, anyhow::Error>(S3UploadDisposition::Uploaded)
|
||||
});
|
||||
}
|
||||
|
||||
let mut stats = S3UploadStats::default();
|
||||
while let Some(result) = tasks.join_next().await {
|
||||
match result.context("S3 upload task failed")?? {
|
||||
S3UploadDisposition::Uploaded => stats.uploaded += 1,
|
||||
S3UploadDisposition::SkippedExisting => stats.skipped_existing += 1,
|
||||
S3UploadDisposition::MetadataRepaired => stats.metadata_repaired += 1,
|
||||
}
|
||||
}
|
||||
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct S3ObjectMetadata {
|
||||
pub(crate) e_tag: Option<String>,
|
||||
@@ -862,69 +802,6 @@ async fn repair_existing_s3_object_metadata(
|
||||
unreachable!("S3 retry attempts are always greater than zero")
|
||||
}
|
||||
|
||||
async fn put_file_to_s3_overwrite(
|
||||
client: &S3Client,
|
||||
bucket: &str,
|
||||
item: &S3UploadPlanItem,
|
||||
) -> Result<()> {
|
||||
println!(
|
||||
"Overwriting {} -> s3://{bucket}/{}",
|
||||
item.path.display(),
|
||||
item.key
|
||||
);
|
||||
let identity = s3_file_identity(&item.path)?;
|
||||
let attempts = s3_retry_attempts();
|
||||
for attempt in 1..=attempts {
|
||||
let body = ByteStream::from_path(&item.path)
|
||||
.await
|
||||
.with_context(|| format!("Failed to read {}", item.path.display()))?;
|
||||
let mut request = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&item.key)
|
||||
.content_md5(identity.md5_base64.clone())
|
||||
.body(body);
|
||||
if let Some(content_type) = &item.content_type {
|
||||
request = request.content_type(content_type);
|
||||
}
|
||||
if let Some(cache_control) = &item.cache_control {
|
||||
request = request.cache_control(cache_control);
|
||||
}
|
||||
match request.send().await {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(error) => {
|
||||
let code = error
|
||||
.as_service_error()
|
||||
.and_then(|error| error.code())
|
||||
.map(ToOwned::to_owned);
|
||||
let status = error
|
||||
.raw_response()
|
||||
.map(|response| response.status().as_u16());
|
||||
if is_retryable_s3_error(code.as_deref(), status) && attempt < attempts {
|
||||
sleep_before_s3_retry(
|
||||
"overwrite upload",
|
||||
&format!("s3://{bucket}/{}", item.key),
|
||||
attempt,
|
||||
attempts,
|
||||
code.as_deref(),
|
||||
status,
|
||||
)
|
||||
.await;
|
||||
continue;
|
||||
}
|
||||
let summary = s3_error_summary(code.as_deref(), status);
|
||||
return Err(error).with_context(|| {
|
||||
format!(
|
||||
"Failed to overwrite upload s3://{bucket}/{}{summary}",
|
||||
item.key
|
||||
)
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
unreachable!("S3 retry attempts are always greater than zero")
|
||||
}
|
||||
|
||||
fn is_existing_object_error(code: Option<&str>, status: Option<u16>) -> bool {
|
||||
matches!(
|
||||
code,
|
||||
@@ -991,27 +868,6 @@ fn s3_write_concurrency() -> usize {
|
||||
.unwrap_or(DEFAULT_S3_WRITE_CONCURRENCY)
|
||||
}
|
||||
|
||||
pub(crate) fn directory_upload_plan<F>(
|
||||
prefix: &str,
|
||||
root: &Path,
|
||||
include: F,
|
||||
) -> Result<Vec<S3UploadPlanItem>>
|
||||
where
|
||||
F: Fn(&Path) -> bool,
|
||||
{
|
||||
Ok(collect_files(root)?
|
||||
.into_iter()
|
||||
.filter_map(|file| {
|
||||
let relative = file.strip_prefix(root).ok()?;
|
||||
if !include(relative) {
|
||||
return None;
|
||||
}
|
||||
let key = join_s3_key(prefix, &path_to_s3_key(relative));
|
||||
Some(S3UploadPlanItem::new(file, key).with_detected_content_type())
|
||||
})
|
||||
.collect::<Vec<_>>())
|
||||
}
|
||||
|
||||
pub(crate) fn s3_content_type_for_key(key: &str) -> Option<&'static str> {
|
||||
let ext = key.rsplit('.').next()?.to_ascii_lowercase();
|
||||
match ext.as_str() {
|
||||
@@ -1045,49 +901,6 @@ pub(crate) fn s3_content_type_for_key(key: &str) -> Option<&'static str> {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn download_s3_prefix(
|
||||
client: &S3Client,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
target: &Path,
|
||||
) -> Result<()> {
|
||||
let list_prefix = s3_directory_prefix(prefix);
|
||||
let keys = list_s3_keys(client, bucket, &list_prefix).await?;
|
||||
for key in keys {
|
||||
let relative = key
|
||||
.strip_prefix(&list_prefix)
|
||||
.unwrap_or(&key)
|
||||
.trim_start_matches('/');
|
||||
if relative.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let output = safe_download_target(target, relative)?;
|
||||
if let Some(parent) = output.parent() {
|
||||
tokio::fs::create_dir_all(parent)
|
||||
.await
|
||||
.with_context(|| format!("Failed to create {}", parent.display()))?;
|
||||
}
|
||||
let bytes = get_s3_object_bytes(client, bucket, &key).await?;
|
||||
tokio::fs::write(&output, bytes)
|
||||
.await
|
||||
.with_context(|| format!("Failed to write {}", output.display()))?;
|
||||
println!("Downloaded s3://{bucket}/{key} -> {}", output.display());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn list_s3_keys(
|
||||
client: &S3Client,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
) -> Result<Vec<String>> {
|
||||
Ok(list_s3_objects(client, bucket, prefix)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|object| object.key)
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn list_s3_objects(
|
||||
client: &S3Client,
|
||||
bucket: &str,
|
||||
@@ -1158,78 +971,6 @@ async fn send_s3_list_objects_v2_page(
|
||||
unreachable!("S3 retry attempts are always greater than zero")
|
||||
}
|
||||
|
||||
pub(crate) async fn get_s3_object_bytes(
|
||||
client: &S3Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
) -> Result<bytes::Bytes> {
|
||||
let object = send_s3_get_object(client, bucket, key).await?;
|
||||
Ok(object
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.with_context(|| format!("Failed to collect s3://{bucket}/{key} body"))?
|
||||
.into_bytes())
|
||||
}
|
||||
|
||||
async fn send_s3_get_object(
|
||||
client: &S3Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
) -> Result<aws_sdk_s3::operation::get_object::GetObjectOutput> {
|
||||
let attempts = s3_retry_attempts();
|
||||
for attempt in 1..=attempts {
|
||||
match client.get_object().bucket(bucket).key(key).send().await {
|
||||
Ok(response) => return Ok(response),
|
||||
Err(error) => {
|
||||
let code = error
|
||||
.as_service_error()
|
||||
.and_then(|error| error.code())
|
||||
.map(ToOwned::to_owned);
|
||||
let status = error
|
||||
.raw_response()
|
||||
.map(|response| response.status().as_u16());
|
||||
if is_retryable_s3_error(code.as_deref(), status) && attempt < attempts {
|
||||
sleep_before_s3_retry(
|
||||
"read",
|
||||
&format!("s3://{bucket}/{key}"),
|
||||
attempt,
|
||||
attempts,
|
||||
code.as_deref(),
|
||||
status,
|
||||
)
|
||||
.await;
|
||||
continue;
|
||||
}
|
||||
let summary = s3_error_summary(code.as_deref(), status);
|
||||
return Err(error)
|
||||
.with_context(|| format!("Failed to read s3://{bucket}/{key}{summary}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
unreachable!("S3 retry attempts are always greater than zero")
|
||||
}
|
||||
|
||||
pub(crate) fn join_s3_key(prefix: &str, child: &str) -> String {
|
||||
let prefix = prefix.trim_matches('/');
|
||||
let child = child.trim_matches('/');
|
||||
match (prefix.is_empty(), child.is_empty()) {
|
||||
(true, true) => String::new(),
|
||||
(true, false) => child.to_string(),
|
||||
(false, true) => prefix.to_string(),
|
||||
(false, false) => format!("{prefix}/{child}"),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn s3_directory_prefix(prefix: &str) -> String {
|
||||
let prefix = prefix.trim_matches('/');
|
||||
if prefix.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("{prefix}/")
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn path_to_s3_key(path: &Path) -> String {
|
||||
path.components()
|
||||
.filter_map(|component| match component {
|
||||
@@ -1240,17 +981,6 @@ pub(crate) fn path_to_s3_key(path: &Path) -> String {
|
||||
.join("/")
|
||||
}
|
||||
|
||||
fn safe_download_target(target: &Path, relative: &str) -> Result<PathBuf> {
|
||||
let candidate = Path::new(relative);
|
||||
for component in candidate.components() {
|
||||
ensure!(
|
||||
matches!(component, std::path::Component::Normal(_)),
|
||||
"Refusing to write S3 object outside download target: {relative}"
|
||||
);
|
||||
}
|
||||
Ok(target.join(candidate))
|
||||
}
|
||||
|
||||
pub(crate) async fn download_file(url: &str, path: &Path) -> Result<()> {
|
||||
let bytes = Client::new()
|
||||
.get(url)
|
||||
@@ -1319,16 +1049,6 @@ pub(crate) fn count_files(root: &Path) -> Result<usize> {
|
||||
Ok(collect_files(root)?.len())
|
||||
}
|
||||
|
||||
pub(crate) fn count_files_min_depth(root: &Path, min_depth: usize) -> Result<usize> {
|
||||
let mut count = 0;
|
||||
for entry in WalkDir::new(root).min_depth(min_depth) {
|
||||
if entry?.file_type().is_file() {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
pub(crate) fn title_case(value: &str) -> String {
|
||||
let mut chars = value.chars();
|
||||
match chars.next() {
|
||||
@@ -1371,14 +1091,6 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn s3_key_helpers_are_platform_neutral() {
|
||||
assert_eq!(
|
||||
join_s3_key("/desktop/", "/canary/linux/"),
|
||||
"desktop/canary/linux"
|
||||
);
|
||||
assert_eq!(
|
||||
s3_directory_prefix("/_handoff/desktop/build/"),
|
||||
"_handoff/desktop/build/"
|
||||
);
|
||||
assert_eq!(
|
||||
path_to_s3_key(Path::new("assets").join("chunks").join("a.js").as_path()),
|
||||
"assets/chunks/a.js"
|
||||
@@ -1545,36 +1257,6 @@ mod tests {
|
||||
assert_eq!(s3_error_summary(None, Some(500)), " (HTTP status 500)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn directory_upload_plan_filters_and_prefixes_keys() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path();
|
||||
fs::create_dir_all(root.join("nested")).unwrap();
|
||||
fs::write(root.join("keep.txt"), "keep").unwrap();
|
||||
fs::write(root.join("nested").join("skip.map"), "skip").unwrap();
|
||||
fs::write(root.join("nested").join("keep.js"), "keep").unwrap();
|
||||
|
||||
let plan = directory_upload_plan("static", root, |relative| {
|
||||
relative.extension().and_then(OsStr::to_str) != Some("map")
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
plan.iter()
|
||||
.map(|item| item.key.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["static/keep.txt", "static/nested/keep.js"]
|
||||
);
|
||||
assert_eq!(
|
||||
plan[0].content_type.as_deref(),
|
||||
Some("text/plain; charset=utf-8")
|
||||
);
|
||||
assert_eq!(
|
||||
plan[1].content_type.as_deref(),
|
||||
Some("application/javascript; charset=utf-8")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s3_content_type_for_key_covers_browser_module_assets() {
|
||||
assert_eq!(
|
||||
|
||||
+130
-1180
File diff suppressed because it is too large
Load Diff
+303
-21
@@ -2,7 +2,7 @@
|
||||
|
||||
use crate::common::{CommandSpec, output_text, parse_version_instant, run_command};
|
||||
use crate::functions::sha256_reader;
|
||||
use anyhow::{Context, Result, bail, ensure};
|
||||
use anyhow::{Context, Result, anyhow, bail, ensure};
|
||||
use chrono::{DateTime, Utc};
|
||||
use clap::{Args, Subcommand};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -15,8 +15,125 @@ use std::time::Duration;
|
||||
pub(crate) const RELEASE_REPOSITORY: &str = "fluxerapp/fluxer";
|
||||
const RELEASE_COMPARE_URL: &str = "https://github.com/fluxerapp/fluxer/compare";
|
||||
pub(crate) const DESKTOP_RELEASE_DESCRIPTOR_SCHEMA_VERSION: u8 = 1;
|
||||
pub(crate) const DESKTOP_RELEASE_ROUTE_COUNT: usize = 28;
|
||||
pub(crate) const DESKTOP_RELEASE_ASSET_COUNT: usize = 24;
|
||||
const DESKTOP_RELEASE_ARCHES: [&str; 2] = ["x64", "arm64"];
|
||||
|
||||
struct DesktopReleasePlatform {
|
||||
platform: &'static str,
|
||||
shipped_formats: &'static [&'static str],
|
||||
updater_feeds: &'static [&'static str],
|
||||
update_payload_suffix: Option<&'static str>,
|
||||
one_build_serves_every_arch: bool,
|
||||
}
|
||||
|
||||
const DESKTOP_RELEASE_PLATFORMS: [DesktopReleasePlatform; 3] = [
|
||||
DesktopReleasePlatform {
|
||||
platform: "win32",
|
||||
shipped_formats: &["portable", "setup"],
|
||||
updater_feeds: &["RELEASES", "releases.win.json", "assets.win.json"],
|
||||
update_payload_suffix: Some("-full.nupkg"),
|
||||
one_build_serves_every_arch: false,
|
||||
},
|
||||
DesktopReleasePlatform {
|
||||
platform: "darwin",
|
||||
shipped_formats: &["dmg", "zip"],
|
||||
updater_feeds: &["RELEASES.json", "releases.json"],
|
||||
update_payload_suffix: None,
|
||||
one_build_serves_every_arch: true,
|
||||
},
|
||||
DesktopReleasePlatform {
|
||||
platform: "linux",
|
||||
shipped_formats: &["appimage", "deb", "rpm", "tar_gz"],
|
||||
updater_feeds: &[],
|
||||
update_payload_suffix: None,
|
||||
one_build_serves_every_arch: false,
|
||||
},
|
||||
];
|
||||
|
||||
fn desktop_release_platform(platform: &str) -> Result<&'static DesktopReleasePlatform> {
|
||||
DESKTOP_RELEASE_PLATFORMS
|
||||
.iter()
|
||||
.find(|entry| entry.platform == platform)
|
||||
.ok_or_else(|| anyhow!("Unsupported desktop release platform {platform:?}"))
|
||||
}
|
||||
|
||||
pub(crate) fn desktop_release_coordinates() -> Vec<(&'static str, &'static str)> {
|
||||
DESKTOP_RELEASE_PLATFORMS
|
||||
.iter()
|
||||
.flat_map(|entry| {
|
||||
DESKTOP_RELEASE_ARCHES
|
||||
.iter()
|
||||
.map(move |arch| (entry.platform, *arch))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn desktop_release_shipped_formats(platform: &str) -> Result<&'static [&'static str]> {
|
||||
Ok(desktop_release_platform(platform)?.shipped_formats)
|
||||
}
|
||||
|
||||
pub(crate) fn desktop_release_updater_feeds(platform: &str) -> Result<&'static [&'static str]> {
|
||||
Ok(desktop_release_platform(platform)?.updater_feeds)
|
||||
}
|
||||
|
||||
pub(crate) fn desktop_release_update_payload_suffix(
|
||||
platform: &str,
|
||||
) -> Result<Option<&'static str>> {
|
||||
Ok(desktop_release_platform(platform)?.update_payload_suffix)
|
||||
}
|
||||
|
||||
fn desktop_release_coordinate_routes(entry: &DesktopReleasePlatform) -> usize {
|
||||
entry.shipped_formats.len()
|
||||
+ entry.updater_feeds.len()
|
||||
+ usize::from(entry.update_payload_suffix.is_some())
|
||||
}
|
||||
|
||||
fn desktop_release_route_inventory() -> BTreeMap<String, usize> {
|
||||
DESKTOP_RELEASE_PLATFORMS
|
||||
.iter()
|
||||
.flat_map(|entry| {
|
||||
DESKTOP_RELEASE_ARCHES.iter().map(move |arch| {
|
||||
(
|
||||
format!("{}/{arch}", entry.platform),
|
||||
desktop_release_coordinate_routes(entry),
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn desktop_release_route_count() -> usize {
|
||||
desktop_release_route_inventory().values().sum()
|
||||
}
|
||||
|
||||
fn desktop_release_asset_count() -> usize {
|
||||
DESKTOP_RELEASE_PLATFORMS
|
||||
.iter()
|
||||
.map(|entry| {
|
||||
let builds = if entry.one_build_serves_every_arch {
|
||||
1
|
||||
} else {
|
||||
DESKTOP_RELEASE_ARCHES.len()
|
||||
};
|
||||
let feeds = entry
|
||||
.updater_feeds
|
||||
.iter()
|
||||
.map(|name| desktop_release_asset_basename(entry.platform, name))
|
||||
.collect::<BTreeSet<_>>()
|
||||
.len();
|
||||
entry.shipped_formats.len() * builds
|
||||
+ (feeds + usize::from(entry.update_payload_suffix.is_some()))
|
||||
* DESKTOP_RELEASE_ARCHES.len()
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
fn desktop_release_asset_basename<'a>(platform: &str, storage_filename: &'a str) -> &'a str {
|
||||
if platform == "darwin" && storage_filename.eq_ignore_ascii_case("releases.json") {
|
||||
"releases.json"
|
||||
} else {
|
||||
storage_filename
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct DesktopReleaseAsset {
|
||||
@@ -72,12 +189,7 @@ pub(crate) fn desktop_release_asset_name(
|
||||
if storage_filename.starts_with(&release_prefix) {
|
||||
return Ok(storage_filename.to_string());
|
||||
}
|
||||
let release_filename =
|
||||
if platform == "darwin" && storage_filename.eq_ignore_ascii_case("releases.json") {
|
||||
"releases.json"
|
||||
} else {
|
||||
storage_filename
|
||||
};
|
||||
let release_filename = desktop_release_asset_basename(platform, storage_filename);
|
||||
Ok(format!(
|
||||
"{release_prefix}{platform_token}-{arch}-{release_filename}"
|
||||
))
|
||||
@@ -123,9 +235,10 @@ pub(crate) fn validate_desktop_release_descriptor(
|
||||
);
|
||||
parse_version_instant(version)
|
||||
.with_context(|| format!("Invalid desktop release descriptor version {version:?}"))?;
|
||||
let route_count = desktop_release_route_count();
|
||||
ensure!(
|
||||
descriptor.assets.len() == DESKTOP_RELEASE_ROUTE_COUNT,
|
||||
"Desktop release descriptor must contain {DESKTOP_RELEASE_ROUTE_COUNT} routes, found {}",
|
||||
descriptor.assets.len() == route_count,
|
||||
"Desktop release descriptor must contain {route_count} routes, found {}",
|
||||
descriptor.assets.len()
|
||||
);
|
||||
let storage_prefix = format!("desktop/{channel}/");
|
||||
@@ -216,19 +329,13 @@ pub(crate) fn validate_desktop_release_descriptor(
|
||||
);
|
||||
}
|
||||
}
|
||||
let asset_count = desktop_release_asset_count();
|
||||
ensure!(
|
||||
release_assets.len() == DESKTOP_RELEASE_ASSET_COUNT,
|
||||
"Desktop release descriptor must contain {DESKTOP_RELEASE_ASSET_COUNT} unique release assets, found {}",
|
||||
release_assets.len() == asset_count,
|
||||
"Desktop release descriptor must contain {asset_count} unique release assets, found {}",
|
||||
release_assets.len()
|
||||
);
|
||||
let expected_route_counts = BTreeMap::from([
|
||||
("darwin/arm64".to_string(), 4usize),
|
||||
("darwin/x64".to_string(), 4usize),
|
||||
("linux/arm64".to_string(), 4usize),
|
||||
("linux/x64".to_string(), 4usize),
|
||||
("win32/arm64".to_string(), 6usize),
|
||||
("win32/x64".to_string(), 6usize),
|
||||
]);
|
||||
let expected_route_counts = desktop_release_route_inventory();
|
||||
ensure!(
|
||||
route_counts == expected_route_counts,
|
||||
"Desktop release descriptor route inventory mismatch: expected {expected_route_counts:?}, found {route_counts:?}"
|
||||
@@ -1027,6 +1134,181 @@ mod tests {
|
||||
use super::*;
|
||||
use anyhow::anyhow;
|
||||
|
||||
const SAMPLE_CHANNEL: &str = "canary";
|
||||
const SAMPLE_VERSION: &str = "2026.913.210037";
|
||||
const SAMPLE_SOURCE_SHA: &str = "0123456789abcdef0123456789abcdef01234567";
|
||||
|
||||
fn sample_storage_filenames(platform: &str, arch: &str, product: &str) -> Vec<String> {
|
||||
let prefix = format!("{product}-{SAMPLE_VERSION}");
|
||||
match platform {
|
||||
"win32" => vec![
|
||||
format!("{prefix}-portable-win-{arch}.zip"),
|
||||
format!("{prefix}-win-{arch}.exe"),
|
||||
"RELEASES".to_string(),
|
||||
"releases.win.json".to_string(),
|
||||
"assets.win.json".to_string(),
|
||||
format!("{prefix}-win-{arch}-full.nupkg"),
|
||||
],
|
||||
"darwin" => vec![
|
||||
format!("{prefix}-mac-universal.dmg"),
|
||||
format!("{prefix}-mac-universal.zip"),
|
||||
"RELEASES.json".to_string(),
|
||||
"releases.json".to_string(),
|
||||
],
|
||||
"linux" => vec![
|
||||
format!("{prefix}-linux-{arch}.AppImage"),
|
||||
format!("{prefix}-linux-{arch}.deb"),
|
||||
format!("{prefix}-linux-{arch}.rpm"),
|
||||
format!("{prefix}-linux-{arch}.tar.gz"),
|
||||
],
|
||||
other => panic!("unsupported desktop release platform {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_descriptor() -> DesktopReleaseDescriptor {
|
||||
let product = desktop_release_product(SAMPLE_CHANNEL).unwrap();
|
||||
let mut contents = BTreeMap::<String, (String, u64)>::new();
|
||||
let mut assets = Vec::new();
|
||||
for (platform, arch) in desktop_release_coordinates() {
|
||||
for filename in sample_storage_filenames(platform, arch, product) {
|
||||
let release_asset = desktop_release_asset_name(
|
||||
SAMPLE_CHANNEL,
|
||||
SAMPLE_VERSION,
|
||||
platform,
|
||||
arch,
|
||||
&filename,
|
||||
)
|
||||
.unwrap();
|
||||
let ordinal = contents.len() as u64 + 1;
|
||||
let (sha256, size) = contents
|
||||
.entry(release_asset.clone())
|
||||
.or_insert_with(|| (format!("{ordinal:064x}"), ordinal * 1024))
|
||||
.clone();
|
||||
assets.push(DesktopReleaseAsset {
|
||||
storage_key: format!("desktop/{SAMPLE_CHANNEL}/{platform}/{arch}/{filename}"),
|
||||
release_asset,
|
||||
sha256,
|
||||
size,
|
||||
});
|
||||
}
|
||||
}
|
||||
DesktopReleaseDescriptor {
|
||||
schema_version: DESKTOP_RELEASE_DESCRIPTOR_SCHEMA_VERSION,
|
||||
channel: SAMPLE_CHANNEL.to_string(),
|
||||
version: SAMPLE_VERSION.to_string(),
|
||||
release_tag: format!("fluxer-desktop-{SAMPLE_CHANNEL}@{SAMPLE_VERSION}"),
|
||||
source_sha: SAMPLE_SOURCE_SHA.to_string(),
|
||||
assets,
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_sample(descriptor: &DesktopReleaseDescriptor) -> Result<()> {
|
||||
validate_desktop_release_descriptor(
|
||||
descriptor,
|
||||
SAMPLE_CHANNEL,
|
||||
SAMPLE_VERSION,
|
||||
SAMPLE_SOURCE_SHA,
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_release_inventory_is_the_one_the_publisher_stages() {
|
||||
assert_eq!(
|
||||
desktop_release_route_inventory(),
|
||||
BTreeMap::from([
|
||||
("darwin/arm64".to_string(), 4usize),
|
||||
("darwin/x64".to_string(), 4usize),
|
||||
("linux/arm64".to_string(), 4usize),
|
||||
("linux/x64".to_string(), 4usize),
|
||||
("win32/arm64".to_string(), 6usize),
|
||||
("win32/x64".to_string(), 6usize),
|
||||
])
|
||||
);
|
||||
assert_eq!(desktop_release_route_count(), 28);
|
||||
assert_eq!(desktop_release_asset_count(), 24);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_complete_desktop_release_validates() {
|
||||
let descriptor = sample_descriptor();
|
||||
assert_eq!(descriptor.assets.len(), desktop_release_route_count());
|
||||
assert_eq!(
|
||||
descriptor
|
||||
.assets
|
||||
.iter()
|
||||
.map(|asset| asset.release_asset.as_str())
|
||||
.collect::<BTreeSet<_>>()
|
||||
.len(),
|
||||
desktop_release_asset_count()
|
||||
);
|
||||
validate_sample(&descriptor).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_two_macos_feed_names_and_the_universal_build_share_one_release_asset() {
|
||||
let descriptor = sample_descriptor();
|
||||
let asset_for = |storage_key_suffix: &str| {
|
||||
descriptor
|
||||
.assets
|
||||
.iter()
|
||||
.find(|asset| asset.storage_key.ends_with(storage_key_suffix))
|
||||
.map(|asset| asset.release_asset.clone())
|
||||
.unwrap()
|
||||
};
|
||||
assert_eq!(
|
||||
asset_for("darwin/x64/RELEASES.json"),
|
||||
asset_for("darwin/x64/releases.json")
|
||||
);
|
||||
assert_eq!(
|
||||
asset_for("darwin/x64/Fluxer-Canary-2026.913.210037-mac-universal.dmg"),
|
||||
asset_for("darwin/arm64/Fluxer-Canary-2026.913.210037-mac-universal.dmg")
|
||||
);
|
||||
assert_ne!(
|
||||
asset_for("darwin/x64/RELEASES.json"),
|
||||
asset_for("darwin/arm64/RELEASES.json")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_release_missing_a_route_is_refused() {
|
||||
let mut descriptor = sample_descriptor();
|
||||
descriptor.assets.pop().unwrap();
|
||||
assert_eq!(
|
||||
validate_sample(&descriptor).unwrap_err().to_string(),
|
||||
"Desktop release descriptor must contain 28 routes, found 27"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_release_carrying_an_extra_route_is_refused() {
|
||||
let mut descriptor = sample_descriptor();
|
||||
let extra = DesktopReleaseAsset {
|
||||
storage_key: format!("desktop/{SAMPLE_CHANNEL}/linux/x64/latest-linux.yml"),
|
||||
release_asset: format!("Fluxer-Canary-{SAMPLE_VERSION}-linux-x64-latest-linux.yml"),
|
||||
sha256: format!("{:064x}", 99u64),
|
||||
size: 4096,
|
||||
};
|
||||
descriptor.assets.push(extra);
|
||||
assert_eq!(
|
||||
validate_sample(&descriptor).unwrap_err().to_string(),
|
||||
"Desktop release descriptor must contain 28 routes, found 29"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_release_publishes_no_per_coordinate_manifest() {
|
||||
for (platform, _) in desktop_release_coordinates() {
|
||||
assert!(
|
||||
!desktop_release_updater_feeds(platform)
|
||||
.unwrap()
|
||||
.contains(&"manifest.json")
|
||||
);
|
||||
}
|
||||
for asset in sample_descriptor().assets {
|
||||
assert!(!asset.storage_key.ends_with("/manifest.json"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_publish_retries_until_a_publish_succeeds() {
|
||||
let mut calls = 0;
|
||||
|
||||
Reference in New Issue
Block a user