Setup and cleanup host dependencies during the build (#995)

Author: giordano

Manifest.toml

@@ -28,7 +28,7 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
 [[BinaryBuilderBase]]
 deps = ["CodecZlib", "Downloads", "InteractiveUtils", "JSON", "LibGit2", "Libdl", "Logging", "OutputCollectors", "Pkg", "Random", "SHA", "SimpleBufferStream", "TOML", "Tar", "UUIDs", "p7zip_jll"]
-git-tree-sha1 = "b89a4d163105a65e4e3ad0f211e4e68d7dead95f"
+git-tree-sha1 = "61cbfed1cfaa5d1486fca1784682b75e8f1986f8"
 repo-rev = "master"
 repo-url = "https://github.com/JuliaPackaging/BinaryBuilderBase.jl.git"
 uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e"
@@ -41,9 +41,9 @@ uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
 version = "0.7.0"
 
 [[DataAPI]]
-git-tree-sha1 = "ad84f52c0b8f05aa20839484dbaf01690b41ff84"
+git-tree-sha1 = "25ccd31003243d2ce83e474cf11663dddf48035f"
 uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
-version = "1.4.0"
+version = "1.4.1"
 
 [[DataStructures]]
 deps = ["InteractiveUtils", "OrderedCollections"]
@@ -185,9 +185,9 @@ version = "1.0.3"
 
 [[MbedTLS_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "bf50d4a1911612ee8f662d0038fc05dda1bea57d"
+git-tree-sha1 = "47e19f64fc939b86dea2b9b5e38c29c787f0d581"
 uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
-version = "2.25.0+0"
+version = "2.24.0+1"
 
 [[Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
@@ -319,9 +319,9 @@ version = "1.0.0"
 
 [[Tables]]
 deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "TableTraits", "Test"]
-git-tree-sha1 = "240d19b8762006ff04b967bdd833269ad642d550"
+git-tree-sha1 = "8dc2bb7d3548e315d890706547b24502ed79504f"
 uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
-version = "1.2.2"
+version = "1.3.1"
 
 [[Tar]]
 deps = ["ArgTools", "SHA"]
@@ -357,9 +357,9 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
 
 [[WebSockets]]
 deps = ["Base64", "Dates", "HTTP", "Logging", "Sockets"]
-git-tree-sha1 = "a07c280e068acb96a7aeb0a6d35c801ba526e364"
+git-tree-sha1 = "f91a602e25fe6b89afc93cf02a4ae18ee9384ce3"
 uuid = "104b5d7c-a370-577a-8038-80a2059c5097"
-version = "1.5.7"
+version = "1.5.9"
 
 [[ZMQ]]
 deps = ["FileWatching", "Sockets", "ZeroMQ_jll"]

src/AutoBuild.jl

@@ -283,7 +283,7 @@ function build_tarballs(ARGS, src_name, src_version, sources, script,
 
         # For deploy discard build-only dependencies
         # and make sure we get a `Vector{Dependency}`
-        dependencies = Dependency[dep for dep in dependencies if !isa(dep, BuildDependency)]
+        dependencies = Dependency[dep for dep in dependencies if is_runtime_dependency(dep)]
 
         # The location the binaries will be available from
         bin_path = "https://github.com/$(deploy_jll_repo)/releases/download/$(tag)"
@@ -659,10 +659,13 @@ function autobuild(dir::AbstractString,
 
         prefix = setup_workspace(
             build_path,
-            source_files;
+            source_files,
+            concrete_platform,
+            default_host_platform;
             verbose=verbose,
         )
-        artifact_paths = setup_dependencies(prefix, getpkg.(dependencies), concrete_platform; verbose=verbose)
+        host_artifact_paths = setup_dependencies(prefix, Pkg.Types.PackageSpec[getpkg(d) for d in dependencies if is_host_dependency(d)], default_host_platform; verbose=verbose)
+        target_artifact_paths = setup_dependencies(prefix, Pkg.Types.PackageSpec[getpkg(d) for d in dependencies if is_target_dependency(d)], concrete_platform; verbose=verbose)
 
         # Create a runner to work inside this workspace with the nonce built-in
         ur = preferred_runner()(
@@ -718,7 +721,7 @@ function autobuild(dir::AbstractString,
         $(script)
         """
 
-        dest_prefix = Prefix(joinpath(prefix.path, "destdir"))
+        dest_prefix = Prefix(BinaryBuilderBase.destdir(prefix.path, concrete_platform))
         did_succeed = with_logfile(dest_prefix, "$(src_name).log") do io
             # Let's start the presentations with BinaryBuilder.jl
             write(io, "BinaryBuilder.jl version: $(get_bb_version())\n\n")
@@ -792,7 +795,8 @@ function autobuild(dir::AbstractString,
         end
 
         # Unsymlink all the deps from the dest_prefix
-        cleanup_dependencies(prefix, artifact_paths)
+        cleanup_dependencies(prefix, host_artifact_paths)
+        cleanup_dependencies(prefix, target_artifact_paths)
 
         # Search for dead links in dest_prefix; raise warnings about them.
         Auditor.warn_deadlinks(dest_prefix.path)
@@ -1067,7 +1071,7 @@ function build_jll_package(src_name::String,
     mkpath(joinpath(code_dir, "src", "wrappers"))
 
     # Drop BuildDependency objects
-    dependencies = Dependency[d for d in dependencies if !isa(d, BuildDependency)]
+    dependencies = Dependency[d for d in dependencies if is_runtime_dependency(d)]
 
     platforms = keys(build_output_meta)
     products_info = Dict{Product,Any}

src/BinaryBuilder.jl

@@ -25,7 +25,7 @@ export AnyPlatform
 export Product, LibraryProduct, FileProduct, ExecutableProduct, FrameworkProduct, satisfied,
        locate, write_deps_file, variable_name
 # BinaryBuilderBase/src/Dependency.jl
-export Dependency, BuildDependency
+export Dependency, BuildDependency, HostBuildDependency
 # BinaryBuilderBase/src/Sources.jl
 export ArchiveSource, FileSource, GitSource, DirectorySource
 # Auditor.jl

src/wizard/interactive_build.jl

@@ -1,7 +1,15 @@
-using BinaryBuilderBase: get_concrete_platform
+using BinaryBuilderBase: get_concrete_platform, destdir
 
 include("hints.jl")
 
+# Add new method to `get_concrete_platform`: this is a common pattern throughout
+# this file.
+BinaryBuilderBase.get_concrete_platform(platform::AbstractPlatform, state::WizardState) =
+    get_concrete_platform(platform;
+                          preferred_gcc_version = state.preferred_gcc_version,
+                          preferred_llvm_version = state.preferred_llvm_version,
+                          compilers = state.compilers)
+
 # When rerunning the script generated during the wizard we want to fail on error
 # and automatically install license at the end.
 full_wizard_script(state::WizardState) =
@@ -17,10 +25,11 @@ function step4(state::WizardState, ur::Runner, platform::AbstractPlatform,
                build_path::AbstractString, prefix::Prefix)
     printstyled(state.outs, "\t\t\t# Step 4: Select build products\n\n", bold=true)
 
+    concrete_platform = get_concrete_platform(platform, state)
     # Collect all executable/library files, explicitly exclude everything that is
     # a symlink to the artifacts directory, as usual.
-    destdir = joinpath(prefix, "destdir")
-    files = filter(f -> startswith(f, destdir), collapse_symlinks(collect_files(destdir)))
+    destdir_path = destdir(prefix, concrete_platform)
+    files = filter(f -> startswith(f, destdir_path), collapse_symlinks(collect_files(destdir_path)))
     files = filter_object_files(files)
 
     # Check if we can load them as an object file
@@ -31,7 +40,7 @@ function step4(state::WizardState, ur::Runner, platform::AbstractPlatform,
     end
 
     # Strip out the prefix from filenames
-    state.files = map(file->replace(file, "$(destdir)/" => ""), files)
+    state.files = map(file->replace(file, "$(destdir_path)/" => ""), files)
     state.file_kinds = map(files) do f
         readmeta(f) do oh
             if isexecutable(oh)
@@ -63,10 +72,7 @@ function step4(state::WizardState, ur::Runner, platform::AbstractPlatform,
 
         if choice == 1
             # Link dependencies into the prefix again
-            concrete_platform = get_concrete_platform(platform;
-                                                      preferred_gcc_version = state.preferred_gcc_version,
-                                                      preferred_llvm_version = state.preferred_llvm_version,
-                                                      compilers = state.compilers)
+            concrete_platform = get_concrete_platform(platform, state)
             artifact_paths = setup_dependencies(prefix, getpkg.(state.dependencies), concrete_platform)
             return step3_interactive(state, prefix, platform, ur, build_path, artifact_paths)
         elseif choice == 2
@@ -214,10 +220,7 @@ function bb_add(client, state::WizardState, prefix::Prefix, platform::AbstractPl
     new_dep = Dependency(jll)
     try
         # This will redo some work, but that may be ok
-        concrete_platform = get_concrete_platform(platform;
-                                                  preferred_gcc_version = state.preferred_gcc_version,
-                                                  preferred_llvm_version = state.preferred_llvm_version,
-                                                  compilers = state.compilers)
+        concrete_platform = get_concrete_platform(platform, state)
         # Clear out the prefix artifacts directory in case this change caused
         # any previous dependencies to change
         let artifacts_dir = joinpath(prefix, "artifacts")
@@ -408,7 +411,8 @@ function step3_interactive(state::WizardState, prefix::Prefix,
         cleanup_dependencies(prefix, artifact_paths)
         state.step = :step3_retry
     else
-        step3_audit(state, platform, joinpath(prefix, "destdir"))
+        concrete_platform = get_concrete_platform(platform, state)
+        step3_audit(state, platform, destdir(prefix, concrete_platform))
         # Unsymlink all the deps from the dest_prefix before moving to the next step
         cleanup_dependencies(prefix, artifact_paths)
         return step4(state, ur, platform, build_path, prefix)
@@ -429,11 +433,8 @@ function step3_retry(state::WizardState)
 
     build_path = tempname()
     mkpath(build_path)
-    prefix = setup_workspace(build_path, vcat(state.source_files, state.patches); verbose=false)
-    concrete_platform = get_concrete_platform(platform;
-                                              preferred_gcc_version = state.preferred_gcc_version,
-                                              preferred_llvm_version = state.preferred_llvm_version,
-                                              compilers = state.compilers)
+    concrete_platform = get_concrete_platform(platform, state)
+    prefix = setup_workspace(build_path, vcat(state.source_files, state.patches), concrete_platform; verbose=false)
     artifact_paths = setup_dependencies(prefix, getpkg.(state.dependencies), concrete_platform)
 
     ur = preferred_runner()(
@@ -448,7 +449,7 @@ function step3_retry(state::WizardState)
     with_logfile(joinpath(build_path, "out.log")) do io
         run(ur, `/bin/bash -c $(full_wizard_script(state))`, io; verbose=true, tee_stream=state.outs)
     end
-    step3_audit(state, platform, joinpath(prefix, "destdir"))
+    step3_audit(state, platform, destdir(prefix, concrete_platform))
     # Unsymlink all the deps from the dest_prefix before moving to the next step
     cleanup_dependencies(prefix, artifact_paths)
 
@@ -514,15 +515,13 @@ function step34(state::WizardState)
     build_path = tempname()
     mkpath(build_path)
     state.history = ""
+    concrete_platform = get_concrete_platform(platform, state)
     prefix = setup_workspace(
         build_path,
-        vcat(state.source_files, state.patches);
+        vcat(state.source_files, state.patches),
+        concrete_platform;
         verbose=false
     )
-    concrete_platform = get_concrete_platform(platform;
-                                              preferred_gcc_version = state.preferred_gcc_version,
-                                              preferred_llvm_version = state.preferred_llvm_version,
-                                              compilers = state.compilers)
     artifact_paths = setup_dependencies(prefix, getpkg.(state.dependencies), concrete_platform; verbose=true)
 
     provide_hints(state, joinpath(prefix, "srcdir"))
@@ -564,13 +563,10 @@ function step5_internal(state::WizardState, platform::AbstractPlatform)
     prefix_artifacts = Dict{Prefix,Vector{String}}()
     while !ok
         cd(build_path) do
-            prefix = setup_workspace(build_path, vcat(state.source_files, state.patches); verbose=true)
+            concrete_platform = get_concrete_platform(platform, state)
+            prefix = setup_workspace(build_path, vcat(state.source_files, state.patches), concrete_platform; verbose=true)
             # Clean up artifacts in case there are some
             cleanup_dependencies(prefix, get(prefix_artifacts, prefix, String[]))
-            concrete_platform = get_concrete_platform(platform;
-                                                      preferred_gcc_version = state.preferred_gcc_version,
-                                                      preferred_llvm_version = state.preferred_llvm_version,
-                                                      compilers = state.compilers)
             artifact_paths = setup_dependencies(prefix, getpkg.(state.dependencies), concrete_platform; verbose=true)
             # Record newly added artifacts for this prefix
             prefix_artifacts[prefix] = artifact_paths
@@ -597,7 +593,7 @@ function step5_internal(state::WizardState, platform::AbstractPlatform)
                     msg = "The build script failed (see above).\n"
                     println(state.outs, msg)
                 else
-                    audit(Prefix(joinpath(prefix, "destdir")); io=state.outs,
+                    audit(Prefix(destdir(prefix, concrete_platform)); io=state.outs,
                         platform=platform, verbose=true, autofix=true, require_license=true)
 
                     ok = isempty(match_files(state, prefix, platform, state.files))
@@ -638,17 +634,15 @@ function step5_internal(state::WizardState, platform::AbstractPlatform)
                     elseif choice == 2
                         rm(build_path; force=true, recursive=true)
                         mkpath(build_path)
+                        concrete_platform = get_concrete_platform(platform, state)
                         prefix = setup_workspace(
                             build_path,
-                            vcat(state.source_files, state.patches);
+                            vcat(state.source_files, state.patches),
+                            concrete_platform;
                             verbose=true,
                         )
                         # Clean up artifacts in case there are some
                         cleanup_dependencies(prefix, get(prefix_artifacts, prefix, String[]))
-                        concrete_platform = get_concrete_platform(platform;
-                                                                  preferred_gcc_version = state.preferred_gcc_version,
-                                                                  preferred_llvm_version = state.preferred_llvm_version,
-                                                                  compilers = state.compilers)
                         artifact_paths = setup_dependencies(prefix, getpkg.(state.dependencies), platform; verbose=true)
                         # Record newly added artifacts for this prefix
                         prefix_artifacts[prefix] = artifact_paths
@@ -775,15 +769,13 @@ function step5c(state::WizardState)
         mkpath(build_path)
         local ok = true
 
+        concrete_platform = get_concrete_platform(platform, state)
         prefix = setup_workspace(
             build_path,
-            vcat(state.source_files, state.patches);
+            vcat(state.source_files, state.patches),
+            concrete_platform;
             verbose=false,
         )
-        concrete_platform = get_concrete_platform(platform;
-                                                  preferred_gcc_version = state.preferred_gcc_version,
-                                                  preferred_llvm_version = state.preferred_llvm_version,
-                                                  compilers = state.compilers)
         artifact_paths = setup_dependencies(prefix, getpkg.(state.dependencies), concrete_platform; verbose=false)
         ur = preferred_runner()(
             prefix.path;
@@ -799,7 +791,7 @@ function step5c(state::WizardState)
         end
 
         if ok
-            audit(Prefix(joinpath(prefix, "destdir"));
+            audit(Prefix(destdir(prefix, concrete_platform));
                 io=state.outs,
                 platform=platform,
                 verbose=false,

test/building.jl

@@ -229,6 +229,35 @@ end
     end
 end
 
+@testset "HostBuildDependency" begin
+    # A platform for which we certainly can't run executable
+    p = Platform("x86_64", "freebsd")
+    mktempdir() do build_path
+        build_output_meta = autobuild(
+            build_path,
+            "host_dep",
+            v"1.0.0",
+            # No sources
+            DirectorySource[],
+            # Script: run hello_world from the HostBuildDependency
+            raw"""
+            hello_world
+            """,
+            # Platform
+            [p],
+            Product[],
+            # Install `HelloWorldC_jll` for both the target and the host.
+            [
+                HostBuildDependency("HelloWorldC_jll"),
+                Dependency("HelloWorldC_jll"),
+            ];
+            # Don't do audit passes
+            skip_audit=true,
+        )
+        @test haskey(build_output_meta, p)
+    end
+end
+
 @testset "Invalid Arguments" begin
     mktempdir() do build_path
         # Test that invalid JLL names both @warn and error()

test/jll.jl

@@ -99,7 +99,7 @@ end
             version = merged["version"]
             # Filter out build-time dependencies that will not go into the dependencies of
             # the JLL packages.
-            dependencies = Dependency[dep for dep in merged["dependencies"] if !isa(dep, BuildDependency)]
+            dependencies = Dependency[dep for dep in merged["dependencies"] if is_runtime_dependency(dep)]
             lazy_artifacts = merged["lazy_artifacts"]
             build_version = BinaryBuilder.get_next_wrapper_version(name, version)
             repo = "JuliaBinaryWrappers/$(name)_jll.jl"
@@ -109,7 +109,7 @@ end
             # Skip init of the remote repository
             # Filter out build-time dependencies also here
             for json_obj in [merged, objs_unmerged...]
-                json_obj["dependencies"] = Dependency[dep for dep in json_obj["dependencies"] if !isa(dep, BuildDependency)]
+                json_obj["dependencies"] = Dependency[dep for dep in json_obj["dependencies"] if is_runtime_dependency(dep)]
             end
 
             tag = "$(name)-v$(build_version)"