Save log files to logdir(prefix; subdir=src_name) (#1059)

This avoids clash between log files of different packages when mounting all of
them together for a build.

Author: giordano

.github/workflows/documentation.yml

@@ -7,6 +7,9 @@ on:
     tags: '*'
   pull_request:
 
+env:
+  JULIA_PKG_SERVER: ""
+
 jobs:
   build:
     runs-on: ubuntu-latest

Manifest.toml

@@ -28,9 +28,11 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
 [[BinaryBuilderBase]]
 deps = ["CodecZlib", "Downloads", "InteractiveUtils", "JSON", "LibGit2", "Libdl", "Logging", "OutputCollectors", "Pkg", "Random", "SHA", "Scratch", "SimpleBufferStream", "TOML", "Tar", "UUIDs", "p7zip_jll", "pigz_jll"]
-git-tree-sha1 = "e3f4db24195c438d88e8753bab588a5385de2c6b"
+git-tree-sha1 = "56393545b3506aba0211c24d0de873e00788d3b8"
+repo-rev = "master"
+repo-url = "https://github.com/JuliaPackaging/BinaryBuilderBase.jl.git"
 uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e"
-version = "0.6.3"
+version = "0.6.5"
 
 [[CodecZlib]]
 deps = ["TranscodingStreams", "Zlib_jll"]
@@ -40,9 +42,9 @@ version = "0.7.0"
 
 [[Compat]]
 deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
-git-tree-sha1 = "0a817fbe51c976de090aa8c997b7b719b786118d"
+git-tree-sha1 = "0900bc19193b8e672d9cd477e6cd92d9e7c02f99"
 uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
-version = "3.28.0"
+version = "3.29.0"
 
 [[DataAPI]]
 git-tree-sha1 = "dfb3b7e89e395be1e25c2ad6d7690dc29cc53b1d"
@@ -86,9 +88,9 @@ uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
 
 [[GitForge]]
 deps = ["Dates", "HTTP", "JSON2"]
-git-tree-sha1 = "ded1b4ce9518e7568bad8dc20171a34684e8c6d2"
+git-tree-sha1 = "0e6b8635059d6ac9e421825e75f8fca170f557d2"
 uuid = "8f6bce27-0656-5410-875b-07a5572985df"
-version = "0.1.6"
+version = "0.1.7"
 
 [[GitHub]]
 deps = ["Base64", "Dates", "HTTP", "JSON", "MbedTLS", "Sockets", "SodiumSeal"]
@@ -233,9 +235,9 @@ uuid = "d8793406-e978-5875-9003-1fc021f44a92"
 version = "0.3.6"
 
 [[OrderedCollections]]
-git-tree-sha1 = "4fa2ba51070ec13fcc7517db714445b4ab986bdf"
+git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c"
 uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
-version = "1.4.0"
+version = "1.4.1"
 
 [[OutputCollectors]]
 git-tree-sha1 = "d86c19b7fa8ad6a4dc8ec2c726642cc6291b2941"
@@ -266,19 +268,19 @@ version = "0.2.0"
 
 [[Preferences]]
 deps = ["TOML"]
-git-tree-sha1 = "ea79e4c9077208cd3bc5d29631a26bc0cff78902"
+git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a"
 uuid = "21216c6a-2e73-6563-6e65-726566657250"
-version = "1.2.1"
+version = "1.2.2"
 
 [[Printf]]
 deps = ["Unicode"]
 uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 
 [[ProgressMeter]]
 deps = ["Distributed", "Printf"]
-git-tree-sha1 = "d85d8f0339a9937afac93e152c76f4745b386202"
+git-tree-sha1 = "1be8800271c86f572d334fef6e3b8364eaece7d9"
 uuid = "92933f4c-e287-5a05-a399-4b506db050ca"
-version = "1.6.0"
+version = "1.6.2"
 
 [[REPL]]
 deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
@@ -296,9 +298,9 @@ version = "0.2.0"
 
 [[Registrator]]
 deps = ["AutoHashEquals", "Base64", "Dates", "Distributed", "FileWatching", "GitForge", "GitHub", "HTTP", "JSON", "LibGit2", "Logging", "MbedTLS", "Mocking", "Mustache", "Mux", "Pkg", "RegistryTools", "Serialization", "Sockets", "TimeToLive", "UUIDs", "ZMQ"]
-git-tree-sha1 = "fc49368213d6dcbacdc444120efa4babcf248dcc"
+git-tree-sha1 = "c7e033175c3b9b466fb2cc8beab47042878a66b0"
 uuid = "4418983a-e44d-11e8-3aec-9789530b3b3e"
-version = "1.2.3"
+version = "1.2.4"
 
 [[RegistryTools]]
 deps = ["AutoHashEquals", "LibGit2", "Pkg", "UUIDs"]

Project.toml

@@ -1,7 +1,7 @@
 name = "BinaryBuilder"
 uuid = "12aac903-9f7c-5d81-afc2-d9565ea332ae"
 authors = ["Elliot Saba <[email protected]>"]
-version = "0.3.4"
+version = "0.3.5"
 
 [deps]
 ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
@@ -33,7 +33,7 @@ ghr_jll = "07c12ed4-43bc-5495-8a2a-d5838ef8d533"
 
 [compat]
 ArgParse = "1.1"
-BinaryBuilderBase = "0.6.1"
+BinaryBuilderBase = "0.6.5"
 GitHub = "5.1"
 HTTP = "0.8, 0.9"
 JLD2 = "0.1.6, 0.2, 0.3, 0.4"

src/Auditor.jl

@@ -24,11 +24,15 @@ include("auditor/codesigning.jl")
 #   something can't be opened.  Possibly use that within BinaryProvider too?
 
 """
-    audit(prefix::Prefix; platform::AbstractPlatform = HostPlatform();
+    audit(prefix::Prefix, src_name::AbstractString = "";
+                          io=stderr,
+                          platform::AbstractPlatform = HostPlatform(),
                           verbose::Bool = false,
                           silent::Bool = false,
                           autofix::Bool = false,
-                          require_license::Bool = true)
+                          has_csl::Bool = true,
+                          require_license::Bool = true,
+          )
 
 Audits a prefix to attempt to find deployability issues with the binary objects
 that have been installed within.  This auditing will check for relocatability
@@ -47,7 +51,8 @@ function audit(prefix::Prefix, src_name::AbstractString = "";
                                silent::Bool = false,
                                autofix::Bool = false,
                                has_csl::Bool = true,
-                               require_license::Bool = true)
+                               require_license::Bool = true,
+               )
     # This would be really weird, but don't let someone set `silent` and `verbose` to true
     if silent
         verbose = false
@@ -107,13 +112,13 @@ function audit(prefix::Prefix, src_name::AbstractString = "";
                         # DO THIS ONE LAST as it can actually mutate the file, which causes the previous
                         # checks to freak out a little bit.
                         all_ok &= check_dynamic_linkage(oh, prefix, bin_files;
-                                                        platform, silent, verbose, autofix)
+                                                        platform, silent, verbose, autofix, src_name)
                     end
                 end
             end
 
             # Ensure this file is codesigned (currently only does something on Apple platforms)
-            all_ok &= ensure_codesigned(f, prefix, platform; verbose)
+            all_ok &= ensure_codesigned(f, prefix, platform; verbose, subdir=src_name)
         catch e
             if !isa(e, ObjectFile.MagicMismatch)
                 rethrow(e)
@@ -172,7 +177,7 @@ function audit(prefix::Prefix, src_name::AbstractString = "";
         # Ensure that all libraries have at least some kind of SONAME, if we're
         # on that kind of platform
         if !Sys.iswindows(platform)
-            all_ok &= ensure_soname(prefix, f, platform; verbose=verbose, autofix=autofix)
+            all_ok &= ensure_soname(prefix, f, platform; verbose, autofix, subdir=src_name)
         end
 
         # Ensure that this library is available at its own SONAME
@@ -306,7 +311,9 @@ function check_dynamic_linkage(oh, prefix, bin_files;
                                platform::AbstractPlatform = HostPlatform(),
                                verbose::Bool = false,
                                silent::Bool = false,
-                               autofix::Bool = true)
+                               autofix::Bool = true,
+                               src_name::AbstractString = "",
+                               )
     all_ok = true
     # If it's a dynamic binary, check its linkage
     if isdynamic(oh)
@@ -331,7 +338,7 @@ function check_dynamic_linkage(oh, prefix, bin_files;
                     if verbose
                         @info("Rpathify'ing default library $(libname)")
                     end
-                    relink_to_rpath(prefix, platform, path(oh), libs[libname]; verbose=verbose)
+                    relink_to_rpath(prefix, platform, path(oh), libs[libname]; verbose, subdir=src_name)
                 end
                 continue
             end
@@ -345,7 +352,7 @@ function check_dynamic_linkage(oh, prefix, bin_files;
                     kidx = findfirst(known_bins .== lowercase(basename(libname)))
                     if kidx !== nothing
                         # If it is, point to that file instead!
-                        new_link = update_linkage(prefix, platform, path(oh), libs[libname], bin_files[kidx]; verbose=verbose)
+                        new_link = update_linkage(prefix, platform, path(oh), libs[libname], bin_files[kidx]; verbose, subdir=src_name)
 
                         if verbose && new_link !== nothing
                             @info("Linked library $(libname) has been auto-mapped to $(new_link)")
@@ -379,7 +386,7 @@ function check_dynamic_linkage(oh, prefix, bin_files;
 
         # If there is an identity mismatch (which only happens on macOS) fix it
         if autofix
-            fix_identity_mismatch(prefix, platform, path(oh), oh; verbose=verbose)
+            fix_identity_mismatch(prefix, platform, path(oh), oh; verbose, subdir=src_name)
         end
     end
     return all_ok

src/AutoBuild.jl

@@ -747,7 +747,7 @@ function autobuild(dir::AbstractString,
         """
 
         dest_prefix = Prefix(BinaryBuilderBase.destdir(prefix.path, concrete_platform))
-        did_succeed = with_logfile(dest_prefix, "$(src_name).log") do io
+        did_succeed = with_logfile(dest_prefix, "$(src_name).log"; subdir=src_name) do io
             # Let's start the presentations with BinaryBuilder.jl
             write(io, "BinaryBuilder.jl version: $(get_bb_version())\n\n")
             # Get the list of compilers...
@@ -837,7 +837,7 @@ function autobuild(dir::AbstractString,
         end
 
         # Compress log files
-        compress_dir(joinpath(dest_prefix.path, "logs"), verbose=verbose)
+        compress_dir(logdir(dest_prefix; subdir=src_name); verbose)
 
         # Once we're built up, go ahead and package this dest_prefix out
         tarball_path, tarball_hash, git_hash = package(

src/auditor/codesigning.jl

@@ -8,15 +8,16 @@ function check_codesigned(path::AbstractString, platform::AbstractPlatform)
     return run(ur, `/usr/local/bin/ldid -d $(basename(path))`)
 end
 
-function ensure_codesigned(path::AbstractString, prefix::Prefix, platform::AbstractPlatform; verbose::Bool = false)
+function ensure_codesigned(path::AbstractString, prefix::Prefix, platform::AbstractPlatform;
+                           verbose::Bool = false, subdir::AbstractString="")
     # We only perform ad-hoc codesigning on Apple platforms
     if !Sys.isapple(platform)
         return true
     end
 
     rel_path = relpath(path, prefix.path)
     ur = preferred_runner()(prefix.path; cwd="/workspace/", platform=platform)
-    with_logfile(prefix, "ldid_$(basename(rel_path)).log") do io
+    with_logfile(prefix, "ldid_$(basename(rel_path)).log"; subdir) do io
         run(ur, `/usr/local/bin/ldid -S -d $(rel_path)`, io; verbose=verbose)
     end
 end

src/auditor/dynamic_linkage.jl

@@ -296,7 +296,7 @@ function patchelf_flags(p::AbstractPlatform)
 end
 
 function relink_to_rpath(prefix::Prefix, platform::AbstractPlatform, path::AbstractString,
-                         old_libpath::AbstractString; verbose::Bool = false)
+                         old_libpath::AbstractString; verbose::Bool=false, subdir::AbstractString="")
     ur = preferred_runner()(prefix.path; cwd="/workspace/", platform=platform)
     rel_path = relpath(path, prefix.path)
     libname = basename(old_libpath)
@@ -311,13 +311,13 @@ function relink_to_rpath(prefix::Prefix, platform::AbstractPlatform, path::Abstr
     end
 
     # Create a new linkage that looks like @rpath/$lib on OSX
-    with_logfile(prefix, "relink_to_rpath_$(basename(rel_path)).log") do io
+    with_logfile(prefix, "relink_to_rpath_$(basename(rel_path)).log"; subdir) do io
         run(ur, relink_cmd, io; verbose=verbose)
     end
 end
 
 function fix_identity_mismatch(prefix::Prefix, platform::AbstractPlatform, path::AbstractString,
-                               oh::ObjectHandle; verbose::Bool = false)
+                               oh::ObjectHandle; verbose::Bool=false, subdir::AbstractString="")
     # Only macOS needs to fix identity mismatches
     if !Sys.isapple(platform)
         return nothing
@@ -344,7 +344,7 @@ function fix_identity_mismatch(prefix::Prefix, platform::AbstractPlatform, path:
     id_cmd = `$install_name_tool -id $(new_id) $(rel_path)`
 
     # Create a new linkage that looks like @rpath/$lib on OSX, 
-    with_logfile(prefix, "fix_identity_mismatch_$(basename(rel_path)).log") do io
+    with_logfile(prefix, "fix_identity_mismatch_$(basename(rel_path)).log"; subdir) do io
         run(ur, id_cmd, io; verbose=verbose)
     end
 end
@@ -361,7 +361,7 @@ MacOS or `patchelf` on Linux.  Windows platforms are completely skipped, as
 they do not encode paths or RPaths within their executables.
 """
 function update_linkage(prefix::Prefix, platform::AbstractPlatform, path::AbstractString,
-                        old_libpath, new_libpath; verbose::Bool = false)
+                        old_libpath, new_libpath; verbose::Bool=false, subdir::AbstractString="")
     # Windows doesn't do updating of linkage
     if Sys.iswindows(platform)
         return
@@ -421,7 +421,7 @@ function update_linkage(prefix::Prefix, platform::AbstractPlatform, path::Abstra
     if !(new_libdir in canonical_rpaths(path))
         libname = basename(old_libpath)
         cmd = add_rpath(normalize_rpath(relpath(new_libdir, dirname(path))))
-        with_logfile(prefix, "update_rpath_$(basename(path))_$(libname).log") do io
+        with_logfile(prefix, "update_rpath_$(basename(path))_$(libname).log"; subdir) do io
             run(ur, cmd, io; verbose=verbose)
         end
     end
@@ -440,7 +440,7 @@ function update_linkage(prefix::Prefix, platform::AbstractPlatform, path::Abstra
         new_libpath = basename(new_libpath)
     end
     cmd = relink(old_libpath, new_libpath)
-    with_logfile(prefix, "update_linkage_$(basename(path))_$(basename(old_libpath)).log") do io
+    with_logfile(prefix, "update_linkage_$(basename(path))_$(basename(old_libpath)).log"; subdir) do io
         run(ur, cmd, io; verbose=verbose)
     end
 

src/auditor/soname_matching.jl

@@ -39,7 +39,7 @@ end
 
 
 function ensure_soname(prefix::Prefix, path::AbstractString, platform::AbstractPlatform;
-                       verbose::Bool = false, autofix::Bool = false)
+                       verbose::Bool = false, autofix::Bool = false, subdir::AbstractString="")
     # Skip any kind of Windows platforms
     if Sys.iswindows(platform)
         return true
@@ -75,7 +75,7 @@ function ensure_soname(prefix::Prefix, path::AbstractString, platform::AbstractP
     end
 
     # Create a new linkage that looks like @rpath/$lib on OSX, 
-    retval = with_logfile(prefix, "set_soname_$(basename(rel_path))_$(soname).log") do io
+    retval = with_logfile(prefix, "set_soname_$(basename(rel_path))_$(soname).log"; subdir) do io
         run(ur, set_soname_cmd, io; verbose=verbose)
     end