Add auditor pass to ad-hoc codesign all products on Darwin (#996)

* Upgrade `BinaryBuilder` to work on Julia v1.6+

This updates some syntax and fixes some tests that were broken on Julia v1.6

* Add ad-hoc codesigning on all Darwin platforms

All darwin executables and libraries will need to be codesigned, so we
use `ldid` to automatically perform this codesigning, and check that it
happens in an audit pass.

* Experiment with CI fixes

* Try turning PkgServer on again

* Update src/auditor/codesigning.jl

Co-authored-by: Mosè Giordano <[email protected]>

Author: staticfloat

.ci/install_agents.sh

@@ -29,7 +29,7 @@ if [[ ! -f "${STORAGE_DIR}/rootfs/usr/local/bin/julia" ]]; then
     # Install Julia
     echo "Installing Julia..."
     mkdir -p "${STORAGE_DIR}/rootfs/depot"
-    JULIA_URL="https://julialangnightlies-s3.julialang.org/bin/linux/x64/julia-latest-linux64.tar.gz"
+    JULIA_URL="https://julialangnightlies-s3.julialang.org/bin/linux/x64/1.6/julia-latest-linux64.tar.gz"
     curl -# -L "$JULIA_URL" | tar --strip-components=1 -zx -C "${STORAGE_DIR}/rootfs/usr/local"
 fi
 

Manifest.toml

@@ -27,8 +27,8 @@ version = "0.2.0"
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
 [[BinaryBuilderBase]]
-deps = ["CodecZlib", "Downloads", "InteractiveUtils", "JSON", "LibGit2", "Libdl", "Logging", "OutputCollectors", "Pkg", "Random", "SHA", "SimpleBufferStream", "TOML", "Tar", "UUIDs", "p7zip_jll"]
-git-tree-sha1 = "61cbfed1cfaa5d1486fca1784682b75e8f1986f8"
+deps = ["CodecZlib", "Downloads", "InteractiveUtils", "JSON", "LibGit2", "Libdl", "Logging", "OutputCollectors", "Pkg", "Random", "SHA", "Scratch", "SimpleBufferStream", "TOML", "Tar", "UUIDs", "p7zip_jll", "pigz_jll"]
+git-tree-sha1 = "ba037f06ff8be240fcbd33c4ec4a397ebb9a9329"
 repo-rev = "master"
 repo-url = "https://github.com/JuliaPackaging/BinaryBuilderBase.jl.git"
 uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e"
@@ -41,9 +41,9 @@ uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
 version = "0.7.0"
 
 [[DataAPI]]
-git-tree-sha1 = "25ccd31003243d2ce83e474cf11663dddf48035f"
+git-tree-sha1 = "6d64b28d291cb94a0d84e6e41081fb081e7f717f"
 uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
-version = "1.4.1"
+version = "1.5.0"
 
 [[DataStructures]]
 deps = ["InteractiveUtils", "OrderedCollections"]
@@ -65,7 +65,7 @@ deps = ["Random", "Serialization", "Sockets"]
 uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 
 [[Downloads]]
-deps = ["ArgTools", "LibCURL"]
+deps = ["ArgTools", "LibCURL", "NetworkOptions"]
 uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 
 [[FileIO]]
@@ -144,13 +144,17 @@ deps = ["LibCURL_jll", "MozillaCACerts_jll"]
 uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
 
 [[LibCURL_jll]]
-deps = ["Libdl"]
+deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
 uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
 
 [[LibGit2]]
-deps = ["Printf"]
+deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
 uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
 
+[[LibSSH2_jll]]
+deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
+uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
+
 [[Libdl]]
 uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 
@@ -163,9 +167,9 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
 
 [[LoggingExtras]]
 deps = ["Dates"]
-git-tree-sha1 = "03289aba73c0abc25ff0229bed60f2a4129cd15c"
+git-tree-sha1 = "70518b2cce1fea30c5e9dd0c44407a90f394de47"
 uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36"
-version = "0.4.2"
+version = "0.4.4"
 
 [[MacroTools]]
 deps = ["Markdown", "Random"]
@@ -184,10 +188,8 @@ uuid = "739be429-bea8-5141-9913-cc70e7f3736d"
 version = "1.0.3"
 
 [[MbedTLS_jll]]
-deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "47e19f64fc939b86dea2b9b5e38c29c787f0d581"
+deps = ["Artifacts", "Libdl"]
 uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
-version = "2.24.0+1"
 
 [[Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
@@ -207,16 +209,19 @@ git-tree-sha1 = "2578b3cd03e4f568f213c7d51b2118f9e81c2617"
 uuid = "a975b10e-0019-58db-a62f-e48ff68538c9"
 version = "0.7.5"
 
+[[NetworkOptions]]
+uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
+
 [[ObjectFile]]
 deps = ["Reexport", "StructIO", "Test"]
 git-tree-sha1 = "e009c49f99dac98cb79f93b26c259ebca66eff26"
 uuid = "d8793406-e978-5875-9003-1fc021f44a92"
 version = "0.3.6"
 
 [[OrderedCollections]]
-git-tree-sha1 = "cf59cfed2e2c12e8a2ff0a4f1e9b2cd8650da6db"
+git-tree-sha1 = "d45739abcfc03b51f6a42712894a593f74c80a23"
 uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
-version = "1.3.2"
+version = "1.3.3"
 
 [[OutputCollectors]]
 git-tree-sha1 = "d86c19b7fa8ad6a4dc8ec2c726642cc6291b2941"
@@ -284,6 +289,12 @@ version = "1.5.3"
 [[SHA]]
 uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 
+[[Scratch]]
+deps = ["Dates"]
+git-tree-sha1 = "ad4b278adb62d185bbcb6864dc24959ab0627bf6"
+uuid = "6c6a2e73-6563-6170-7368-637461726353"
+version = "1.0.3"
+
 [[Serialization]]
 uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 
@@ -374,10 +385,8 @@ uuid = "8f1865be-045e-5c20-9c9f-bfbfb0764568"
 version = "4.3.2+5"
 
 [[Zlib_jll]]
-deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "32085929ad61a5ed99abea288ec7242be392bb8b"
+deps = ["Libdl"]
 uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
-version = "1.2.12+0"
 
 [[ghr_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
@@ -391,8 +400,16 @@ git-tree-sha1 = "7127f5f40332ccfa43ee07dcd0c4d81a27d9bb23"
 uuid = "a9144af2-ca23-56d9-984f-0d03f7b5ccf8"
 version = "1.0.18+1"
 
+[[nghttp2_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
+
 [[p7zip_jll]]
-deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "4b909fd780b3711197e3faa806dd631bdc5af510"
+deps = ["Artifacts", "Libdl"]
 uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
-version = "16.2.1+0"
+
+[[pigz_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "8c379a72c82099ceb4be53f4f427690376279052"
+uuid = "1bc43ea1-30af-5bc8-a9d4-c018457e6e3e"
+version = "2.5.0+0"

azure-pipelines.yml

@@ -13,7 +13,6 @@ variables:
   JULIA: unbuffer julia --project=$(Build.SourcesDirectory) --color=yes
   BINARYBUILDER_AUTOMATIC_APPLE: true
   BINARYBUILDER_USE_CCACHE: true
-  JULIA_PKG_SERVER: ""
 
 jobs:
 - job: Info
@@ -23,7 +22,7 @@ jobs:
     clean: true
   - bash: |
       set -e
-      $(JULIA) -e 'using Pkg; Pkg.instantiate()'
+      $(JULIA) -e 'using Pkg; Pkg.Registry.update(); Pkg.instantiate()'
       $(JULIA) -e 'using BinaryBuilder; BinaryBuilder.versioninfo()'
       $(JULIA) -e 'using Pkg; Pkg.status(; mode=PKGMODE_MANIFEST)'
     name: SystemInfo

src/Auditor.jl

@@ -16,6 +16,7 @@ include("auditor/compiler_abi.jl")
 include("auditor/soname_matching.jl")
 include("auditor/filesystems.jl")
 include("auditor/extra_checks.jl")
+include("auditor/codesigning.jl")
 
 # AUDITOR TODO LIST:
 #
@@ -82,35 +83,37 @@ function audit(prefix::Prefix, src_name::AbstractString = "";
                     end
                 else
                     # Check that the ISA isn't too high
-                    all_ok &= check_isa(oh, platform, prefix; verbose=verbose, silent=silent)
+                    all_ok &= check_isa(oh, platform, prefix; verbose, silent)
                     # Check that the OS ABI is set correctly (often indicates the wrong linker was used)
-                    all_ok &= check_os_abi(oh, platform, verbose = verbose)
+                    all_ok &= check_os_abi(oh, platform; verbose)
                     # Make sure all binary files are executables, if libraries aren't
                     # executables Julia may not be able to dlopen them:
                     # https://github.com/JuliaLang/julia/issues/38993.  In principle this
                     # should be done when autofix=true, but we have to run this fix on MKL
                     # for Windows, for which however we have to set autofix=false:
                     # https://github.com/JuliaPackaging/Yggdrasil/pull/922.
-                    all_ok &= ensure_executability(oh; verbose=verbose, silent=silent)
+                    all_ok &= ensure_executability(oh; verbose, silent)
 
                     # If this is a dynamic object, do the dynamic checks
                     if isdynamic(oh)
                         # Check that the libgfortran version matches
-                        all_ok &= check_libgfortran_version(oh, platform; verbose=verbose, has_csl = has_csl)
+                        all_ok &= check_libgfortran_version(oh, platform; verbose, has_csl)
                         # Check whether the library depends on any of the most common
                         # libraries provided by `CompilerSupportLibraries_jll`.
-                        all_ok &= check_csl_libs(oh, platform; verbose=verbose, has_csl=has_csl)
+                        all_ok &= check_csl_libs(oh, platform; verbose, has_csl)
                         # Check that the libstdcxx string ABI matches
-                        all_ok &= check_cxxstring_abi(oh, platform; verbose=verbose)
+                        all_ok &= check_cxxstring_abi(oh, platform; verbose)
                         # Check that this binary file's dynamic linkage works properly.  Note to always
                         # DO THIS ONE LAST as it can actually mutate the file, which causes the previous
                         # checks to freak out a little bit.
                         all_ok &= check_dynamic_linkage(oh, prefix, bin_files;
-                                                        platform=platform, silent=silent,
-                                                        verbose=verbose, autofix=autofix)
+                                                        platform, silent, verbose, autofix)
                     end
                 end
             end
+
+            # Ensure this file is codesigned (currently only does something on Apple platforms)
+            all_ok &= ensure_codesigned(f, prefix, platform; verbose)
         catch e
             if !isa(e, ObjectFile.MagicMismatch)
                 rethrow(e)

src/auditor/codesigning.jl

@@ -0,0 +1,22 @@
+function check_codesigned(path::AbstractString, platform::AbstractPlatform)
+    # We only perform ad-hoc codesigning on Apple platforms
+    if !Sys.isapple(platform)
+        return true
+    end
+
+    ur = preferred_runner()(dirname(path); cwd="/workspace/", platform=platform)
+    return run(ur, `/usr/local/bin/ldid -d $(basename(path))`)
+end
+
+function ensure_codesigned(path::AbstractString, prefix::Prefix, platform::AbstractPlatform; verbose::Bool = false)
+    # We only perform ad-hoc codesigning on Apple platforms
+    if !Sys.isapple(platform)
+        return true
+    end
+
+    rel_path = relpath(path, prefix.path)
+    ur = preferred_runner()(prefix.path; cwd="/workspace/", platform=platform)
+    with_logfile(prefix, "ldid_$(basename(rel_path)).log") do io
+        run(ur, `/usr/local/bin/ldid -S -d $(rel_path)`, io; verbose=verbose)
+    end
+end

test/auditing.jl

@@ -27,7 +27,7 @@ end
     @test compatible_marchs(Platform("x86_64", "linux"; march="avx2")) == ["x86_64", "avx", "avx2"]
     @test compatible_marchs(Platform("x86_64", "linux"; march="avx512")) == ["x86_64", "avx", "avx2", "avx512"]
     @test compatible_marchs(Platform("armv7l", "linux")) == ["armv7l"]
-    @test compatible_marchs(Platform("i686", "linux"; march="prescott")) == ["i686", "prescott"]
+    @test compatible_marchs(Platform("i686", "linux"; march="prescott")) == ["pentium4", "prescott"]
     @test compatible_marchs(Platform("aarch64", "linux"; march="armv8_1")) == ["armv8_0", "armv8_1"]
 
     product = ExecutableProduct("main", :main)
@@ -368,6 +368,9 @@ end
         wrong_id_path = locate(wrong_id, prefix; platform=platform)
         @test any(startswith.(wrong_id_path, libdirs(prefix)))
         @test get_dylib_id(wrong_id_path) == "@rpath/totally_different.dylib"
+
+        # Ensure that this bianry is codesigned
+        @test BinaryBuilder.Auditor.check_codesigned(right_id_path, platform)
     end
 end