Move part of autobuild() into new get_meta_json() helper (#872)

fingolfin · web-flow · commit 2d68c43ed636 · 2020-08-05T23:17:37.000+02:00
The two new functions have similar but not identical signature; this change
means that we don't have to pass things to autobuild() which are not needed
for building, but rather just for the meta JSON; and vice versa.
diff --git a/src/AutoBuild.jl b/src/AutoBuild.jl
@@ -1,4 +1,4 @@
-export build_tarballs, autobuild, print_artifacts_toml, build
+export build_tarballs, autobuild, print_artifacts_toml, build, get_meta_json
 import GitHub: gh_get_json, DEFAULT_API
 import SHA: sha256, sha1
 using Pkg.TOML, Dates, UUIDs
@@ -83,10 +83,14 @@ see what it can do, you can call it with `--help` in the `ARGS` or see the
 [Command Line](@ref) section in the manual.
 
 The `kwargs` are passed on to [`autobuild`](@ref), see there for a list of
-supported ones. In addition, the keyword argument `init_block` may be set to
-a string containing Julia code; if present, this code will be inserted into
-the initialization path of the generated JLL package. This can for example be
-used to invoke an initialization API of a shared library.
+supported ones. A few additional keyword arguments are accept:
+
+* `lazy_artifacts` sets whether the artifacts should be lazy.
+
+* `init_block` may be set to a string containing Julia code; if present, this
+  code will be inserted into the initialization path of the generated JLL
+  package. This can for example be used to invoke an initialization API of a
+  shared library.
 
 !!! note
 
@@ -202,11 +206,7 @@ function build_tarballs(ARGS, src_name, src_version, sources, script,
         end
     end
 
-    # Build the given platforms using the given sources
-    build_output_meta = autobuild(
-        # Controls output product placement, mount directory placement, etc...
-        pwd(),
-
+    args = (
         # Source information
         src_name,
         src_version,
@@ -222,17 +222,34 @@ function build_tarballs(ARGS, src_name, src_version, sources, script,
         products,
 
         # Dependencies that must be downloaded
-        dependencies;
-
-        # Flags
-        verbose=verbose,
-        debug=debug,
-        meta_json_stream=meta_json_stream,
-        kwargs...,
+        dependencies,
     )
 
-    if meta_json_stream !== nothing && meta_json_stream !== stdout
-        close(meta_json_stream)
+    if meta_json_stream !== nothing
+        # If they've asked for the JSON metadata, by all means, give it to them!
+        dict = get_meta_json(args...;
+                             lazy_artifacts = lazy_artifacts,
+                             init_block = init_block)
+        println(meta_json_stream, JSON.json(dict))
+
+        if meta_json_stream !== stdout
+            close(meta_json_stream)
+        end
+
+        build_output_meta = Dict()
+    else
+        # Build the given platforms using the given sources
+        build_output_meta = autobuild(
+            # Controls output product placement, mount directory placement, etc...
+            pwd(),
+
+            args...;
+
+            # Flags
+            verbose=verbose,
+            debug=debug,
+            kwargs...,
+        )
     end
 
     if deploy_jll
@@ -468,6 +485,34 @@ function register_jll(name, build_version, dependencies;
     end
 end
 
+function get_meta_json(
+                   src_name::AbstractString,
+                   src_version::VersionNumber,
+                   sources::Vector{<:AbstractSource},
+                   script::AbstractString,
+                   platforms::Vector,
+                   products::Vector{<:Product},
+                   dependencies::Vector{<:AbstractDependency};
+                   lazy_artifacts::Bool = false,
+                   init_block::String = "")
+
+    dict = Dict(
+        "name" => src_name,
+        "version" => "v$(src_version)",
+        "sources" => sources,
+        "script" => script,
+        "products" => products,
+        "dependencies" => dependencies,
+        "lazy_artifacts" => lazy_artifacts,
+        "init_block" => init_block,
+    )
+    # Do not write the list of platforms when building only for `AnyPlatform`
+    if platforms != [AnyPlatform()]
+        dict["platforms"] = triplet.(platforms)
+    end
+    return dict
+end
+
 """
     autobuild(dir::AbstractString, src_name::AbstractString,
               src_version::VersionNumber, sources::Vector,
@@ -518,11 +563,6 @@ here are the relevant actors, broken down in brief:
 
 * `require_license` enables a special audit pass that requires licenses to be
    installed by all packages.
-
-* `lazy_artifacts` sets whether the artifacts should be lazy.
-
-* `meta_json_stream`: If this is set to an IOStream, do not actually build, just
-   output a JSON representation of all the metadata about this build to the stream.
 """
 function autobuild(dir::AbstractString,
                    src_name::AbstractString,
@@ -539,30 +579,8 @@ function autobuild(dir::AbstractString,
                    autofix::Bool = true,
                    code_dir::Union{String,Nothing} = nothing,
                    require_license::Bool = true,
-                   lazy_artifacts::Bool = false,
-                   init_block::String = "",
-                   meta_json_stream = nothing,
                    kwargs...)
     @nospecialize
-    # If they've asked for the JSON metadata, by all means, give it to them!
-    if meta_json_stream !== nothing
-        dict = Dict(
-            "name" => src_name,
-            "version" => "v$(src_version)",
-            "sources" => sources,
-            "script" => script,
-            "products" => products,
-            "dependencies" => dependencies,
-            "lazy_artifacts" => lazy_artifacts,
-            "init_block" => init_block,
-        )
-        # Do not write the list of platforms when building only for `AnyPlatform`
-        if platforms != [AnyPlatform()]
-            dict["platforms"] = triplet.(platforms)
-        end
-        println(meta_json_stream, JSON.json(dict))
-        return Dict()
-    end
 
     # If we're on CI and we're not verbose, schedule a task to output a "." every few seconds
     if (haskey(ENV, "TRAVIS") || haskey(ENV, "CI")) && !verbose
diff --git a/test/declarative.jl b/test/declarative.jl
@@ -3,122 +3,114 @@ using JSON, BinaryBuilder, Test
 import BinaryBuilder.BinaryBuilderBase: sourcify, dependencify, major, minor, patch, version
 
 @testset "Meta JSON" begin
-    mktempdir() do tmpdir
+    meta_json_buff = IOBuffer()
+
+    # Run autobuild() a few times to generate a moderately complex `meta.json`:
+    dict = get_meta_json(
+        "libfoo",
+        v"1.0.0",
+        [FileSource("https://julialang.org", "123123"), DirectorySource("./bundled")],
+        "exit 1",
+        [Linux(:x86_64)],
+        Product[LibraryProduct("libfoo", :libfoo), FrameworkProduct("fooFramework", :libfooFramework)],
+        [Dependency("Zlib_jll")];
+    )
+    println(meta_json_buff, JSON.json(dict))
+
+    dict = get_meta_json(
+        "libfoo",
+        v"1.0.0",
+        [GitSource("https://github.com/JuliaLang/julia.git", "5d4eaca0c9fa3d555c79dbacdccb9169fdf64b65")],
+        "exit 0",
+        [Linux(:x86_64), Windows(:x86_64)],
+        Product[ExecutableProduct("julia", :julia), LibraryProduct("libfoo2", :libfoo2; dlopen_flags=[:RTLD_GLOBAL])],
+        Dependency[];
+    )
+    println(meta_json_buff, JSON.json(dict))
+
+    # Now, deserialize the info:
+    seek(meta_json_buff, 0)
+
+    # Strip out ending newlines as that makes our while loop below sad
+    meta_json_buff = IOBuffer(strip(String(take!(meta_json_buff))))
+    objs = []
+    while !eof(meta_json_buff)
+        push!(objs, JSON.parse(meta_json_buff))
+    end
+
+    # Ensure that we get two JSON objects
+    @test length(objs) == 2
+
+    # Merge them, then test that the merged object contains everything we expect
+    meta = BinaryBuilder.cleanup_merged_object!(BinaryBuilder.merge_json_objects(objs))
+
+    @test all(haskey.(Ref(meta), ("name", "version", "script", "platforms", "products", "dependencies")))
+    @test meta["name"] == "libfoo"
+    @test meta["version"] == v"1.0.0"
+    @test Set(meta["platforms"]) == Set([Linux(:x86_64, libc=:glibc), Windows(:x86_64)])
+    @test length(meta["sources"]) == 3
+    @test all(in.(
+          (
+              FileSource("https://julialang.org", "123123"),
+              GitSource("https://github.com/JuliaLang/julia.git", "5d4eaca0c9fa3d555c79dbacdccb9169fdf64b65"),
+              DirectorySource("./bundled"),
+          ), Ref(meta["sources"])))
+    @test sourcify(Dict("type" => "directory", "path" => "foo")) == DirectorySource("foo")
+    @test sourcify(Dict("type" => "git", "url" => "https://github.com/JuliaLang/julia.git", "hash" => "12345")) == GitSource("https://github.com/JuliaLang/julia.git", "12345")
+    @test sourcify(Dict("type" => "file", "url" => "https://julialang.org", "hash" => "98765")) == FileSource("https://julialang.org", "98765")
+    @test_throws ErrorException sourcify(Dict("type" => "qux"))
+    # `PackageSpec(; name = "Foo") != PackageSpec(; name = "Foo")`, so we
+    # need to manually compare the fields we care about
+    d = dependencify(Dict("type" => "dependency", "name" => "Foo_jll", "uuid" => nothing,
+                          "version-major" => 0, "version-minor" => 0, "version-patch" => 0))
+    ref_d = Dependency(PackageSpec(; name = "Foo_jll"))
+    @test d.pkg.name == ref_d.pkg.name
+    @test d.pkg.uuid == ref_d.pkg.uuid
+    v = version(d)
+    ref_v = version(ref_d)
+    @test (major(v), minor(v), patch(v)) == (major(ref_v), minor(ref_v), patch(ref_v))
+    d = dependencify(Dict("type" => "builddependency", "name" => "Qux_jll", "uuid" => nothing,
+                          "version-major" => 1, "version-minor" => 2, "version-patch" => 3))
+    ref_d = Dependency(PackageSpec(; name = "Qux_jll", version = v"1.2.3"))
+    @test d.pkg.name == ref_d.pkg.name
+    @test d.pkg.uuid == ref_d.pkg.uuid
+    v = version(d)
+    ref_v = version(ref_d)
+    @test (major(v), minor(v), patch(v)) == (major(ref_v), minor(ref_v), patch(ref_v))
+    @test_throws ErrorException dependencify(Dict("type" => "bar"))
+    @test length(meta["products"]) == 4
+    @test all(in.((LibraryProduct("libfoo", :libfoo), ExecutableProduct("julia", :julia), LibraryProduct("libfoo2", :libfoo2; dlopen_flags=[:RTLD_GLOBAL]), FrameworkProduct("fooFramework", :libfooFramework)), Ref(meta["products"])))
+    @test length(meta["script"]) == 2
+    @test all(in.(("exit 0", "exit 1"), Ref(meta["script"])))
+
+    @testset "AnyPlatform" begin
         meta_json_buff = IOBuffer()
-        # Run autobuild() a few times to generate a moderately complex `meta.json`:
-        build_output_meta = autobuild(
-            tmpdir,
-            "libfoo",
+        dict = get_meta_json(
+            "any_file",
             v"1.0.0",
-            [FileSource("https://julialang.org", "123123"), DirectorySource("./bundled")],
+            FileSource[],
             "exit 1",
-            [Linux(:x86_64)],
-            Product[LibraryProduct("libfoo", :libfoo), FrameworkProduct("fooFramework", :libfooFramework)],
-            [Dependency("Zlib_jll")];
-            meta_json_stream=meta_json_buff,
+            [AnyPlatform()],
+            Product[FileProduct("file", :file)],
+            BuildDependency[];
         )
-        @test build_output_meta == Dict()
-
-        build_output_meta = autobuild(
-            tmpdir,
-            "libfoo",
-            v"1.0.0",
-            [GitSource("https://github.com/JuliaLang/julia.git", "5d4eaca0c9fa3d555c79dbacdccb9169fdf64b65")],
-            "exit 0",
-            [Linux(:x86_64), Windows(:x86_64)],
-            Product[ExecutableProduct("julia", :julia), LibraryProduct("libfoo2", :libfoo2; dlopen_flags=[:RTLD_GLOBAL])],
-            Dependency[];
-            meta_json_stream=meta_json_buff,
-        )
-        @test build_output_meta == Dict()
-
-        # Now, deserialize the info:
-        seek(meta_json_buff, 0)
+        println(meta_json_buff, JSON.json(dict))
 
+        # Deserialize the info:
+        seekstart(meta_json_buff)
         # Strip out ending newlines as that makes our while loop below sad
         meta_json_buff = IOBuffer(strip(String(take!(meta_json_buff))))
         objs = []
         while !eof(meta_json_buff)
             push!(objs, JSON.parse(meta_json_buff))
         end
-
-        # Ensure that we get two JSON objects
-        @test length(objs) == 2
-
+        # Ensure that we get one JSON object
+        @test length(objs) == 1
+        # Platform-independent build: the JSON file doesn't have a "platforms" key
+        @test !haskey(objs[1], "platforms")
         # Merge them, then test that the merged object contains everything we expect
         meta = BinaryBuilder.cleanup_merged_object!(BinaryBuilder.merge_json_objects(objs))
-
-        @test all(haskey.(Ref(meta), ("name", "version", "script", "platforms", "products", "dependencies")))
-        @test meta["name"] == "libfoo"
-        @test meta["version"] == v"1.0.0"
-        @test Set(meta["platforms"]) == Set([Linux(:x86_64, libc=:glibc), Windows(:x86_64)])
-        @test length(meta["sources"]) == 3
-        @test all(in.(
-              (
-                  FileSource("https://julialang.org", "123123"),
-                  GitSource("https://github.com/JuliaLang/julia.git", "5d4eaca0c9fa3d555c79dbacdccb9169fdf64b65"),
-                  DirectorySource("./bundled"),
-              ), Ref(meta["sources"])))
-        @test sourcify(Dict("type" => "directory", "path" => "foo")) == DirectorySource("foo")
-        @test sourcify(Dict("type" => "git", "url" => "https://github.com/JuliaLang/julia.git", "hash" => "12345")) == GitSource("https://github.com/JuliaLang/julia.git", "12345")
-        @test sourcify(Dict("type" => "file", "url" => "https://julialang.org", "hash" => "98765")) == FileSource("https://julialang.org", "98765")
-        @test_throws ErrorException sourcify(Dict("type" => "qux"))
-        # `PackageSpec(; name = "Foo") != PackageSpec(; name = "Foo")`, so we
-        # need to manually compare the fields we care about
-        d = dependencify(Dict("type" => "dependency", "name" => "Foo_jll", "uuid" => nothing,
-                              "version-major" => 0, "version-minor" => 0, "version-patch" => 0))
-        ref_d = Dependency(PackageSpec(; name = "Foo_jll"))
-        @test d.pkg.name == ref_d.pkg.name
-        @test d.pkg.uuid == ref_d.pkg.uuid
-        v = version(d)
-        ref_v = version(ref_d)
-        @test (major(v), minor(v), patch(v)) == (major(ref_v), minor(ref_v), patch(ref_v))
-        d = dependencify(Dict("type" => "builddependency", "name" => "Qux_jll", "uuid" => nothing,
-                              "version-major" => 1, "version-minor" => 2, "version-patch" => 3))
-        ref_d = Dependency(PackageSpec(; name = "Qux_jll", version = v"1.2.3"))
-        @test d.pkg.name == ref_d.pkg.name
-        @test d.pkg.uuid == ref_d.pkg.uuid
-        v = version(d)
-        ref_v = version(ref_d)
-        @test (major(v), minor(v), patch(v)) == (major(ref_v), minor(ref_v), patch(ref_v))
-        @test_throws ErrorException dependencify(Dict("type" => "bar"))
-        @test length(meta["products"]) == 4
-        @test all(in.((LibraryProduct("libfoo", :libfoo), ExecutableProduct("julia", :julia), LibraryProduct("libfoo2", :libfoo2; dlopen_flags=[:RTLD_GLOBAL]), FrameworkProduct("fooFramework", :libfooFramework)), Ref(meta["products"])))
-        @test length(meta["script"]) == 2
-        @test all(in.(("exit 0", "exit 1"), Ref(meta["script"])))
-    end
-
-    @testset "AnyPlatform" begin
-        mktempdir() do tmpdir
-            meta_json_buff = IOBuffer()
-            build_output_meta = autobuild(
-                tmpdir,
-                "any_file",
-                v"1.0.0",
-                FileSource[],
-                "exit 1",
-                [AnyPlatform()],
-                Product[FileProduct("file", :file)],
-                BuildDependency[];
-                meta_json_stream=meta_json_buff,
-            )
-            @test build_output_meta == Dict()
-            # Deserialize the info:
-            seekstart(meta_json_buff)
-            # Strip out ending newlines as that makes our while loop below sad
-            meta_json_buff = IOBuffer(strip(String(take!(meta_json_buff))))
-            objs = []
-            while !eof(meta_json_buff)
-                push!(objs, JSON.parse(meta_json_buff))
-            end
-            # Ensure that we get one JSON object
-            @test length(objs) == 1
-            # Platform-independent build: the JSON file doesn't have a "platforms" key
-            @test !haskey(objs[1], "platforms")
-            # Merge them, then test that the merged object contains everything we expect
-            meta = BinaryBuilder.cleanup_merged_object!(BinaryBuilder.merge_json_objects(objs))
-            # The "platforms" key comes back in the cleaned up object
-            @test meta["platforms"] == [AnyPlatform()]
-        end
+        # The "platforms" key comes back in the cleaned up object
+        @test meta["platforms"] == [AnyPlatform()]
     end
 end
diff --git a/test/jll.jl b/test/jll.jl
