[ArchiveUtils] Make tarballs compressed with pigz reproducible (#270)

* [ArchiveUtils] Make tarballs compressed with `pigz` reproducible

* Do not run reproducibility test with p7zip/xz

* [ArchiveUtils] Pass also `--no-name` to pigz

Author: giordano

Project.toml

@@ -1,7 +1,7 @@
 name = "BinaryBuilderBase"
 uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e"
 authors = ["Elliot Saba <[email protected]>"]
-version = "1.15.0"
+version = "1.16.0"
 
 [deps]
 CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"

src/ArchiveUtils.jl

@@ -169,7 +169,7 @@ function package(src_dir::AbstractString, tarball_path::AbstractString;
     # Note: For compressing gzip files we use pigz since it uses threading where as p7zip
     # does not.
     compress_cmd = if format == "gzip"
-        pipeline(`$(pigz()) -9`, stdout=tarball_path)
+        pipeline(`$(pigz()) --no-time --no-name -9`, stdout=tarball_path)
     else
         pipeline(`$(p7zip()) a -si -t$format -mx9 $tarball_path`, stdout=devnull)
     end

test/archive_utils.jl

@@ -1,5 +1,6 @@
 using BinaryBuilderBase: Prefix, archive_artifact, package, list_tarball_files
 using Pkg.Artifacts: create_artifact, remove_artifact, with_artifacts_directory
+using SHA
 using Test
 
 @testset "Archive Utils" begin
@@ -25,11 +26,22 @@ using Test
             end
 
             mktempdir() do output_dir
-                for (format, ext) in [("gzip", "gz"), ("xz", "xz")]
+                for (format, ext, hash) in (("gzip", "gz", "568f743e965b63d3187b6a2647700a71d1d7520b4596fbf2bfb39ffa67c4bb55"),
+                                            # Compressing with p7zip/xz doesn't seem to be fully reproducible, at least not
+                                            # across different systems. We'll try to investigate more, but for the time being
+                                            # skip the reproducibility test for it.
+                                            ("xz", "xz", ""))
                     tarball_path =  joinpath(output_dir, "foo.tar.$ext")
                     package(prefix, tarball_path; format=format)
                     @test isfile(tarball_path)
 
+                    if !isempty(hash)
+                        tarball_hash = open(tarball_path, "r") do io
+                            bytes2hex(sha256(io))
+                        end
+                        @test tarball_hash == hash
+                    end
+
                     # Test that we can inspect the contents of the tarball
                     contents = list_tarball_files(tarball_path)
                     @test "bin/bar.sh" in contents
@@ -52,7 +64,8 @@ using Test
 
                 # Create an artifact containing the full source directory
                 mktempdir() do output_dir
-                    tarball, _, tree_hash = package(Prefix(src_dir), joinpath(output_dir, lib), v"1.2.3")
+                    tarball, tarball_hash, tree_hash = package(Prefix(src_dir), joinpath(output_dir, lib), v"1.2.3")
+                    @test tarball_hash == "46abbba8bf97315ecf3d979fb1c95e2e1a639ab71d8d569736e68e6a2ab4a427"
                     @test tree_hash == Base.SHA1("b316cc5e582cbd503b2da34bd1b79aaf3941ad80")
                     contents = list_tarball_files(tarball)
                     @test libname ∈ contents
@@ -61,7 +74,8 @@ using Test
 
                 # Create an artifact containing *only* the log file logs/libfoo.gz
                 mktempdir() do output_dir
-                    tarball, _, tree_hash = package(Prefix(src_dir), joinpath(output_dir, lib), v"1.2.3"; filter=(_, f) -> f == "logs")
+                    tarball, tarball_hash, tree_hash = package(Prefix(src_dir), joinpath(output_dir, lib), v"1.2.3"; filter=(_, f) -> f == "logs")
+                    @test tarball_hash == "6b51bcbed720e98d5eaef43a0b10d5b08e11ff05d05fe3ca9525686871767c4c"
                     @test tree_hash == Base.SHA1("3a3ccf24312676bdd8c2ec769232dbd3bd1b9857")
                     contents = list_tarball_files(tarball)
                     @test libname ∉ contents
@@ -77,7 +91,7 @@ using Test
             with_artifacts_directory(art_dir) do
                 hash = create_artifact(p -> touch(joinpath(p, "foo")))
                 tarball_path = joinpath(art_dir, "foo.tar.gz")
-                archive_artifact(hash, tarball_path)
+                @test archive_artifact(hash, tarball_path) == "59f35d4bde88850c3e80d6890770902ca6e9e6c3a3b3e9343df686a7b28e89b0"
                 @test "foo" in list_tarball_files(tarball_path)
                 rm(tarball_path)