rust: exclude standard library test files from install_only archives

These files shouldn't be needed for most installs. So including them
in the `install_only` archives is wasteful.

On the 3.10 x86_64-unknown-linux-gnu distributions, this reduces the
archive size from 29,628,194 to 25,109,023. The gzip compressed
`install_only` archives are now smaller than the zstd compressed full
archives.

Part of #134.

Author: indygreg

src/release.rs

@@ -3,7 +3,8 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 use {
-    anyhow::Result,
+    crate::json::parse_python_json,
+    anyhow::{anyhow, Result},
     once_cell::sync::Lazy,
     semver::VersionReq,
     std::{
@@ -182,7 +183,25 @@ pub fn convert_to_install_only<W: Write>(reader: impl BufRead, writer: W) -> Res
 
     let mut builder = tar::Builder::new(writer);
 
-    for entry in tar_in.entries()? {
+    let mut entries = tar_in.entries()?;
+
+    // First entry in archive should be python/PYTHON.json.
+    let mut entry = entries.next().expect("tar must have content")?;
+    if entry.path_bytes().as_ref() != b"python/PYTHON.json" {
+        return Err(anyhow!("first archive entry not PYTHON.json"));
+    }
+
+    let mut json_data = vec![];
+    entry.read_to_end(&mut json_data)?;
+
+    let json_main = parse_python_json(&json_data)?;
+
+    let stdlib_path = json_main
+        .python_paths
+        .get("stdlib")
+        .expect("stdlib entry expected");
+
+    for entry in entries {
         let mut entry = entry?;
 
         let path_bytes = entry.path_bytes();
@@ -191,7 +210,7 @@ pub fn convert_to_install_only<W: Write>(reader: impl BufRead, writer: W) -> Res
             continue;
         }
 
-        // Also strip the libpython static library, as it significantly
+        // Strip the libpython static library, as it significantly
         // increases the size of the archive and isn't needed in most cases.
         if path_bytes
             .windows(b"/libpython".len())
@@ -202,6 +221,22 @@ pub fn convert_to_install_only<W: Write>(reader: impl BufRead, writer: W) -> Res
             continue;
         }
 
+        // Strip standard library test modules, as they aren't needed in regular
+        // installs. We do this based on the metadata in PYTHON.json for
+        // consistency.
+        if json_main
+            .python_stdlib_test_packages
+            .iter()
+            .any(|test_package| {
+                let package_path =
+                    format!("python/{}/{}/", stdlib_path, test_package.replace('.', "/"));
+
+                path_bytes.starts_with(package_path.as_bytes())
+            })
+        {
+            continue;
+        }
+
         let mut data = vec![];
         entry.read_to_end(&mut data)?;