feat(ci): publish linux repositories from the desktop release (#2847)

tools/ci/src/common.rs

@@ -442,25 +442,6 @@ pub(crate) async fn s3_client(default_endpoint: Option<&str>) -> Result<S3Client
     Ok(S3Client::from_conf(s3_config.build()))
 }
 
-pub(crate) async fn upload_directory_to_s3<F>(
-    client: &S3Client,
-    bucket: &str,
-    prefix: &str,
-    root: &Path,
-    include: F,
-) -> Result<()>
-where
-    F: Fn(&Path) -> bool,
-{
-    let plan = directory_upload_plan(prefix, root, include)?;
-    let stats = upload_s3_plan_append_only(client, bucket, plan).await?;
-    println!(
-        "Append-only upload complete for s3://{bucket}/{prefix}: uploaded {}, skipped existing {}",
-        stats.uploaded, stats.skipped_existing
-    );
-    Ok(())
-}
-
 pub(crate) async fn upload_s3_plan_append_only(
     client: &S3Client,
     bucket: &str,
@@ -528,47 +509,6 @@ pub(crate) async fn upload_s3_plan_append_only(
     Ok(stats)
 }
 
-pub(crate) async fn upload_s3_plan_overwrite(
-    client: &S3Client,
-    bucket: &str,
-    plan: Vec<S3UploadPlanItem>,
-) -> Result<S3UploadStats> {
-    ensure_unique_s3_keys(&plan)?;
-    let concurrency = s3_write_concurrency();
-    let semaphore = Arc::new(Semaphore::new(concurrency));
-    let bucket = bucket.to_string();
-    let mut tasks = JoinSet::new();
-    for item in plan {
-        let permit = semaphore
-            .clone()
-            .acquire_owned()
-            .await
-            .context("S3 upload semaphore closed")?;
-        let client = client.clone();
-        let bucket = bucket.clone();
-        tasks.spawn(async move {
-            let _permit = permit;
-            put_file_to_s3_overwrite(&client, &bucket, &item)
-                .await
-                .with_context(|| {
-                    format!("Failed overwrite upload for s3://{}/{}", bucket, item.key)
-                })?;
-            Ok::<_, anyhow::Error>(S3UploadDisposition::Uploaded)
-        });
-    }
-
-    let mut stats = S3UploadStats::default();
-    while let Some(result) = tasks.join_next().await {
-        match result.context("S3 upload task failed")?? {
-            S3UploadDisposition::Uploaded => stats.uploaded += 1,
-            S3UploadDisposition::SkippedExisting => stats.skipped_existing += 1,
-            S3UploadDisposition::MetadataRepaired => stats.metadata_repaired += 1,
-        }
-    }
-
-    Ok(stats)
-}
-
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub(crate) struct S3ObjectMetadata {
     pub(crate) e_tag: Option<String>,
@@ -862,69 +802,6 @@ async fn repair_existing_s3_object_metadata(
     unreachable!("S3 retry attempts are always greater than zero")
 }
 
-async fn put_file_to_s3_overwrite(
-    client: &S3Client,
-    bucket: &str,
-    item: &S3UploadPlanItem,
-) -> Result<()> {
-    println!(
-        "Overwriting {} -> s3://{bucket}/{}",
-        item.path.display(),
-        item.key
-    );
-    let identity = s3_file_identity(&item.path)?;
-    let attempts = s3_retry_attempts();
-    for attempt in 1..=attempts {
-        let body = ByteStream::from_path(&item.path)
-            .await
-            .with_context(|| format!("Failed to read {}", item.path.display()))?;
-        let mut request = client
-            .put_object()
-            .bucket(bucket)
-            .key(&item.key)
-            .content_md5(identity.md5_base64.clone())
-            .body(body);
-        if let Some(content_type) = &item.content_type {
-            request = request.content_type(content_type);
-        }
-        if let Some(cache_control) = &item.cache_control {
-            request = request.cache_control(cache_control);
-        }
-        match request.send().await {
-            Ok(_) => return Ok(()),
-            Err(error) => {
-                let code = error
-                    .as_service_error()
-                    .and_then(|error| error.code())
-                    .map(ToOwned::to_owned);
-                let status = error
-                    .raw_response()
-                    .map(|response| response.status().as_u16());
-                if is_retryable_s3_error(code.as_deref(), status) && attempt < attempts {
-                    sleep_before_s3_retry(
-                        "overwrite upload",
-                        &format!("s3://{bucket}/{}", item.key),
-                        attempt,
-                        attempts,
-                        code.as_deref(),
-                        status,
-                    )
-                    .await;
-                    continue;
-                }
-                let summary = s3_error_summary(code.as_deref(), status);
-                return Err(error).with_context(|| {
-                    format!(
-                        "Failed to overwrite upload s3://{bucket}/{}{summary}",
-                        item.key
-                    )
-                });
-            }
-        }
-    }
-    unreachable!("S3 retry attempts are always greater than zero")
-}
-
 fn is_existing_object_error(code: Option<&str>, status: Option<u16>) -> bool {
     matches!(
         code,
@@ -991,27 +868,6 @@ fn s3_write_concurrency() -> usize {
         .unwrap_or(DEFAULT_S3_WRITE_CONCURRENCY)
 }
 
-pub(crate) fn directory_upload_plan<F>(
-    prefix: &str,
-    root: &Path,
-    include: F,
-) -> Result<Vec<S3UploadPlanItem>>
-where
-    F: Fn(&Path) -> bool,
-{
-    Ok(collect_files(root)?
-        .into_iter()
-        .filter_map(|file| {
-            let relative = file.strip_prefix(root).ok()?;
-            if !include(relative) {
-                return None;
-            }
-            let key = join_s3_key(prefix, &path_to_s3_key(relative));
-            Some(S3UploadPlanItem::new(file, key).with_detected_content_type())
-        })
-        .collect::<Vec<_>>())
-}
-
 pub(crate) fn s3_content_type_for_key(key: &str) -> Option<&'static str> {
     let ext = key.rsplit('.').next()?.to_ascii_lowercase();
     match ext.as_str() {
@@ -1045,49 +901,6 @@ pub(crate) fn s3_content_type_for_key(key: &str) -> Option<&'static str> {
     }
 }
 
-pub(crate) async fn download_s3_prefix(
-    client: &S3Client,
-    bucket: &str,
-    prefix: &str,
-    target: &Path,
-) -> Result<()> {
-    let list_prefix = s3_directory_prefix(prefix);
-    let keys = list_s3_keys(client, bucket, &list_prefix).await?;
-    for key in keys {
-        let relative = key
-            .strip_prefix(&list_prefix)
-            .unwrap_or(&key)
-            .trim_start_matches('/');
-        if relative.is_empty() {
-            continue;
-        }
-        let output = safe_download_target(target, relative)?;
-        if let Some(parent) = output.parent() {
-            tokio::fs::create_dir_all(parent)
-                .await
-                .with_context(|| format!("Failed to create {}", parent.display()))?;
-        }
-        let bytes = get_s3_object_bytes(client, bucket, &key).await?;
-        tokio::fs::write(&output, bytes)
-            .await
-            .with_context(|| format!("Failed to write {}", output.display()))?;
-        println!("Downloaded s3://{bucket}/{key} -> {}", output.display());
-    }
-    Ok(())
-}
-
-pub(crate) async fn list_s3_keys(
-    client: &S3Client,
-    bucket: &str,
-    prefix: &str,
-) -> Result<Vec<String>> {
-    Ok(list_s3_objects(client, bucket, prefix)
-        .await?
-        .into_iter()
-        .map(|object| object.key)
-        .collect())
-}
-
 async fn list_s3_objects(
     client: &S3Client,
     bucket: &str,
@@ -1158,78 +971,6 @@ async fn send_s3_list_objects_v2_page(
     unreachable!("S3 retry attempts are always greater than zero")
 }
 
-pub(crate) async fn get_s3_object_bytes(
-    client: &S3Client,
-    bucket: &str,
-    key: &str,
-) -> Result<bytes::Bytes> {
-    let object = send_s3_get_object(client, bucket, key).await?;
-    Ok(object
-        .body
-        .collect()
-        .await
-        .with_context(|| format!("Failed to collect s3://{bucket}/{key} body"))?
-        .into_bytes())
-}
-
-async fn send_s3_get_object(
-    client: &S3Client,
-    bucket: &str,
-    key: &str,
-) -> Result<aws_sdk_s3::operation::get_object::GetObjectOutput> {
-    let attempts = s3_retry_attempts();
-    for attempt in 1..=attempts {
-        match client.get_object().bucket(bucket).key(key).send().await {
-            Ok(response) => return Ok(response),
-            Err(error) => {
-                let code = error
-                    .as_service_error()
-                    .and_then(|error| error.code())
-                    .map(ToOwned::to_owned);
-                let status = error
-                    .raw_response()
-                    .map(|response| response.status().as_u16());
-                if is_retryable_s3_error(code.as_deref(), status) && attempt < attempts {
-                    sleep_before_s3_retry(
-                        "read",
-                        &format!("s3://{bucket}/{key}"),
-                        attempt,
-                        attempts,
-                        code.as_deref(),
-                        status,
-                    )
-                    .await;
-                    continue;
-                }
-                let summary = s3_error_summary(code.as_deref(), status);
-                return Err(error)
-                    .with_context(|| format!("Failed to read s3://{bucket}/{key}{summary}"));
-            }
-        }
-    }
-    unreachable!("S3 retry attempts are always greater than zero")
-}
-
-pub(crate) fn join_s3_key(prefix: &str, child: &str) -> String {
-    let prefix = prefix.trim_matches('/');
-    let child = child.trim_matches('/');
-    match (prefix.is_empty(), child.is_empty()) {
-        (true, true) => String::new(),
-        (true, false) => child.to_string(),
-        (false, true) => prefix.to_string(),
-        (false, false) => format!("{prefix}/{child}"),
-    }
-}
-
-pub(crate) fn s3_directory_prefix(prefix: &str) -> String {
-    let prefix = prefix.trim_matches('/');
-    if prefix.is_empty() {
-        String::new()
-    } else {
-        format!("{prefix}/")
-    }
-}
-
 pub(crate) fn path_to_s3_key(path: &Path) -> String {
     path.components()
         .filter_map(|component| match component {
@@ -1240,17 +981,6 @@ pub(crate) fn path_to_s3_key(path: &Path) -> String {
         .join("/")
 }
 
-fn safe_download_target(target: &Path, relative: &str) -> Result<PathBuf> {
-    let candidate = Path::new(relative);
-    for component in candidate.components() {
-        ensure!(
-            matches!(component, std::path::Component::Normal(_)),
-            "Refusing to write S3 object outside download target: {relative}"
-        );
-    }
-    Ok(target.join(candidate))
-}
-
 pub(crate) async fn download_file(url: &str, path: &Path) -> Result<()> {
     let bytes = Client::new()
         .get(url)
@@ -1319,16 +1049,6 @@ pub(crate) fn count_files(root: &Path) -> Result<usize> {
     Ok(collect_files(root)?.len())
 }
 
-pub(crate) fn count_files_min_depth(root: &Path, min_depth: usize) -> Result<usize> {
-    let mut count = 0;
-    for entry in WalkDir::new(root).min_depth(min_depth) {
-        if entry?.file_type().is_file() {
-            count += 1;
-        }
-    }
-    Ok(count)
-}
-
 pub(crate) fn title_case(value: &str) -> String {
     let mut chars = value.chars();
     match chars.next() {
@@ -1371,14 +1091,6 @@ mod tests {
 
     #[test]
     fn s3_key_helpers_are_platform_neutral() {
-        assert_eq!(
-            join_s3_key("/desktop/", "/canary/linux/"),
-            "desktop/canary/linux"
-        );
-        assert_eq!(
-            s3_directory_prefix("/_handoff/desktop/build/"),
-            "_handoff/desktop/build/"
-        );
         assert_eq!(
             path_to_s3_key(Path::new("assets").join("chunks").join("a.js").as_path()),
             "assets/chunks/a.js"
@@ -1545,36 +1257,6 @@ mod tests {
         assert_eq!(s3_error_summary(None, Some(500)), " (HTTP status 500)");
     }
 
-    #[test]
-    fn directory_upload_plan_filters_and_prefixes_keys() {
-        let temp = tempfile::tempdir().unwrap();
-        let root = temp.path();
-        fs::create_dir_all(root.join("nested")).unwrap();
-        fs::write(root.join("keep.txt"), "keep").unwrap();
-        fs::write(root.join("nested").join("skip.map"), "skip").unwrap();
-        fs::write(root.join("nested").join("keep.js"), "keep").unwrap();
-
-        let plan = directory_upload_plan("static", root, |relative| {
-            relative.extension().and_then(OsStr::to_str) != Some("map")
-        })
-        .unwrap();
-
-        assert_eq!(
-            plan.iter()
-                .map(|item| item.key.as_str())
-                .collect::<Vec<_>>(),
-            vec!["static/keep.txt", "static/nested/keep.js"]
-        );
-        assert_eq!(
-            plan[0].content_type.as_deref(),
-            Some("text/plain; charset=utf-8")
-        );
-        assert_eq!(
-            plan[1].content_type.as_deref(),
-            Some("application/javascript; charset=utf-8")
-        );
-    }
-
     #[test]
     fn s3_content_type_for_key_covers_browser_module_assets() {
         assert_eq!(