rust: produce install_only tar.gz when fetching release artifacts

Currently, we produce these .tar.gz archives part of CI. This adds
complexity to CI and there isn't much value in having the .tar.gz
CI artifacts before release since they are redundant with the .tar.zst
archive content.

This commit adds Rust code for normalizing the .tar.zst files to
the .tar.gz install_only archives. The Rust code should be comparable
to the Python code. However, it does result in the creation of more
install_only archives.

Author: indygreg

Cargo.lock

@@ -9,6 +9,7 @@ anyhow = "1"
 bytes = "1.1"
 clap = "2"
 duct = "0"
+flate2 = "1"
 futures = "0"
 goblin = "0"
 octocrab = { version = "0", features = ["rustls"] }

Cargo.toml

@@ -3,7 +3,7 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 use {
-    crate::release::RELEASE_TRIPLES,
+    crate::release::{produce_install_only, RELEASE_TRIPLES},
     anyhow::{anyhow, Result},
     clap::ArgMatches,
     futures::StreamExt,
@@ -102,22 +102,56 @@ pub async fn command_fetch_release_distributions(args: &ArgMatches<'_>) -> Resul
 
             let name = zf.name().to_string();
 
-            if let Some(release) = RELEASE_TRIPLES.iter().find_map(|(triple, release)| {
+            if let Some((triple, release)) = RELEASE_TRIPLES.iter().find_map(|(triple, release)| {
                 if name.contains(triple) {
-                    Some(release)
+                    Some((triple, release))
                 } else {
                     None
                 }
             }) {
-                if release.suffixes.iter().any(|suffix| name.contains(suffix)) {
-                    let dest_path = dest_dir.join(&name);
-                    let mut buf = vec![];
-                    zf.read_to_end(&mut buf)?;
-                    std::fs::write(&dest_path, &buf)?;
-
-                    println!("releasing {}", name);
+                let stripped_name = if let Some(s) = name.strip_suffix(".tar.zst") {
+                    s
                 } else {
+                    println!("{} not a .tar.zst artifact", name);
+                    continue;
+                };
+
+                let stripped_name = &stripped_name[0..stripped_name.len() - "-YYYYMMDDTHHMM".len()];
+
+                let triple_start = stripped_name
+                    .find(triple)
+                    .expect("validated triple presence above");
+
+                let build_suffix = &stripped_name[triple_start + triple.len() + 1..];
+
+                if !release
+                    .suffixes
+                    .iter()
+                    .any(|suffix| build_suffix == *suffix)
+                {
                     println!("{} not a release artifact for triple", name);
+                    continue;
+                }
+
+                let dest_path = dest_dir.join(&name);
+                let mut buf = vec![];
+                zf.read_to_end(&mut buf)?;
+                std::fs::write(&dest_path, &buf)?;
+
+                println!("releasing {}", name);
+
+                if build_suffix == release.install_only_suffix {
+                    println!("producing install_only archive from {}", name);
+
+                    let dest_path = produce_install_only(&dest_path)?;
+
+                    println!(
+                        "releasing {}",
+                        dest_path
+                            .file_name()
+                            .expect("should have file name")
+                            .to_string_lossy()
+                    );
                 }
             } else {
                 println!("{} does not match any registered release triples", name);

src/github.rs

@@ -2,7 +2,15 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use {once_cell::sync::Lazy, std::collections::BTreeMap};
+use {
+    anyhow::Result,
+    once_cell::sync::Lazy,
+    std::{
+        collections::BTreeMap,
+        io::{BufRead, Read, Write},
+        path::{Path, PathBuf},
+    },
+};
 
 /// Describes a release for a given target triple.
 pub struct TripleRelease {
@@ -128,3 +136,64 @@ pub static RELEASE_TRIPLES: Lazy<BTreeMap<&'static str, TripleRelease>> = Lazy::
 
     h
 });
+
+/// Convert a .tar.zst archive to an install only .tar.gz archive.
+pub fn convert_to_install_only<W: Write>(reader: impl BufRead, writer: W) -> Result<W> {
+    let dctx = zstd::stream::Decoder::new(reader)?;
+
+    let mut tar_in = tar::Archive::new(dctx);
+
+    let writer = flate2::write::GzEncoder::new(writer, flate2::Compression::default());
+
+    let mut builder = tar::Builder::new(writer);
+
+    for entry in tar_in.entries()? {
+        let mut entry = entry?;
+
+        if !entry.path_bytes().starts_with(b"python/install/") {
+            continue;
+        }
+
+        let mut data = vec![];
+        entry.read_to_end(&mut data)?;
+
+        let path = entry.path()?;
+        let new_path = PathBuf::from("python").join(path.strip_prefix("python/install/")?);
+
+        let mut header = entry.header().clone();
+        header.set_path(&new_path)?;
+        header.set_cksum();
+
+        builder.append(&header, std::io::Cursor::new(data))?;
+    }
+
+    Ok(builder.into_inner()?.finish()?)
+}
+
+pub fn produce_install_only(tar_zst_path: &Path) -> Result<PathBuf> {
+    let buf = std::fs::read(tar_zst_path)?;
+
+    let gz_data = convert_to_install_only(std::io::Cursor::new(buf), std::io::Cursor::new(vec![]))?
+        .into_inner();
+
+    let filename = tar_zst_path
+        .file_name()
+        .expect("should have filename")
+        .to_string_lossy();
+
+    let mut name_parts = filename
+        .split('-')
+        .map(|x| x.to_string())
+        .collect::<Vec<_>>();
+    let parts_len = name_parts.len();
+
+    name_parts[parts_len - 2] = "install_only".to_string();
+
+    let install_only_name = name_parts.join("-");
+    let install_only_name = install_only_name.replace(".tar.zst", ".tar.gz");
+
+    let dest_path = tar_zst_path.with_file_name(install_only_name);
+    std::fs::write(&dest_path, &gz_data)?;
+
+    Ok(dest_path)
+}