pythonbuild: normalize all tar archives

The impetus for this was wanting to put PYTHON.json earlier
in the archive. But we do end up making archives deterministic where
they weren't before. So this seems like a good move all around.

Author: indygreg

cpython-windows/build.py

@@ -5,6 +5,7 @@
 
 import argparse
 import concurrent.futures
+import io
 import json
 import os
 import pathlib
@@ -24,6 +25,7 @@
     extract_tar_to_directory,
     extract_zip_to_directory,
     compress_python_archive,
+    normalize_tar_archive,
     release_tag_from_git,
     validate_python_json,
 )
@@ -2260,8 +2262,19 @@ def build_cpython(
             "cpython-%s-%s-%s.tar" % (entry["version"], target_triple, profile,)
         )
 
+        data = io.BytesIO()
+        create_tar_from_directory(data, td / "out")
+        data.seek(0)
+
+        data = normalize_tar_archive(data)
+
         with dest_path.open("wb") as fh:
-            create_tar_from_directory(fh, td / "out")
+            while True:
+                chunk = data.read(32768)
+                if not chunk:
+                    break
+
+                fh.write(chunk)
 
         return dest_path
 

pythonbuild/buildenv.py

@@ -14,7 +14,12 @@
 from .docker import container_exec, container_get_archive, copy_file_to_container
 from .downloads import DOWNLOADS
 from .logging import log
-from .utils import create_tar_from_directory, exec_and_log, extract_tar_to_directory
+from .utils import (
+    create_tar_from_directory,
+    exec_and_log,
+    extract_tar_to_directory,
+    normalize_tar_archive,
+)
 
 
 class ContainerContext(object):
@@ -108,6 +113,8 @@ def get_output_archive(self, path=None, as_tar=False):
         data = container_get_archive(self.container, p)
         data = io.BytesIO(data)
 
+        data = normalize_tar_archive(data)
+
         if as_tar:
             return tarfile.open(fileobj=data)
         else:
@@ -212,9 +219,10 @@ def get_output_archive(self, path, as_tar=False):
 
         data = io.BytesIO()
         create_tar_from_directory(data, p, path_prefix=p.parts[-1])
-
         data.seek(0)
 
+        data = normalize_tar_archive(data)
+
         if as_tar:
             return tarfile.open(fileobj=data)
         else:

pythonbuild/utils.py

@@ -4,9 +4,11 @@
 
 import gzip
 import hashlib
+import io
 import multiprocessing
 import os
 import pathlib
+import stat
 import subprocess
 import sys
 import tarfile
@@ -247,6 +249,67 @@ def extract_zip_to_directory(source: pathlib.Path, dest: pathlib.Path):
         zf.extractall(dest)
 
 
+# 2021-01-01T00:00:00
+DEFAULT_MTIME = 1609488000
+
+
+def normalize_tar_archive(data: io.BytesIO) -> io.BytesIO:
+    """Normalize the contents of a tar archive.
+
+    We want tar archives to be as deterministic as possible. This function will
+    take tar archive data in a buffer and return a new buffer containing a more
+    deterministic tar archive.
+    """
+    members = []
+
+    with tarfile.open(fileobj=data) as tf:
+        for ti in tf:
+            # We don't care about directory entries. Tools can handle this fine.
+            if ti.isdir():
+                continue
+
+            filedata = tf.extractfile(ti)
+            if filedata is not None:
+                filedata = io.BytesIO(filedata.read())
+
+            members.append((ti, filedata))
+
+    # Sort the archive members. We put PYTHON.json first so metadata can
+    # be read without reading the entire archive.
+    def sort_key(v):
+        if v[0].name == "python/PYTHON.json":
+            return 0, v[0].name
+        else:
+            return 1, v[0].name
+
+    members.sort(key=sort_key)
+
+    # Normalize attributes on archive members.
+    for entry in members:
+        ti = entry[0]
+        ti.mtime = DEFAULT_MTIME
+        ti.uid = 0
+        ti.uname = "root"
+        ti.gid = 0
+        ti.gname = "root"
+
+        # Give user/group read/write on all entries.
+        ti.mode |= stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP
+
+        # If user executable, give to group as well.
+        if ti.mode & stat.S_IXUSR:
+            ti.mode |= stat.S_IXGRP
+
+    dest = io.BytesIO()
+    with tarfile.open(fileobj=dest, mode="w") as tf:
+        for (ti, filedata) in members:
+            tf.addfile(ti, filedata)
+
+    dest.seek(0)
+
+    return dest
+
+
 def compress_python_archive(
     source_path: pathlib.Path, dist_path: pathlib.Path, basename: str
 ):

src/main.rs

@@ -700,7 +700,23 @@ fn validate_distribution(dist_path: &Path) -> Result<Vec<String>> {
     let dctx = zstd::stream::Decoder::new(reader)?;
     let mut tf = tar::Archive::new(dctx);
 
-    for entry in tf.entries()? {
+    // First entry in archive should be python/PYTHON.json.
+    let mut entries = tf.entries()?;
+
+    let mut entry = entries.next().unwrap()?;
+    if entry.path()?.display().to_string() == "python/PYTHON.json" {
+        let mut data = Vec::new();
+        entry.read_to_end(&mut data)?;
+        let json = parse_python_json(&data).context("parsing PYTHON.json")?;
+        errors.extend(validate_json(&json, triple)?);
+    } else {
+        errors.push(format!(
+            "1st archive entry should be for python/PYTHON.json; got {}",
+            entry.path()?.display()
+        ));
+    }
+
+    for entry in entries {
         let mut entry = entry.map_err(|e| anyhow!("failed to iterate over archive: {}", e))?;
         let path = entry.path()?.to_path_buf();
 
@@ -741,7 +757,6 @@ fn validate_distribution(dist_path: &Path) -> Result<Vec<String>> {
 
         if path == PathBuf::from("python/PYTHON.json") {
             let json = parse_python_json(&data).context("parsing PYTHON.json")?;
-
             errors.extend(validate_json(&json, triple)?);
         }
     }