[ext/HSG]: Enable generating historical stdlibs on macOS (#2417)

Also, update the historical stdlib list

(cherry picked from commit 7b87092)

Author: staticfloat

ext/HistoricaStdlibGenerator/Manifest.toml

@@ -74,7 +74,7 @@ uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
 version = "1.0.15"
 
 [[Pkg]]
-deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs"]
+deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
 path = "../.."
 uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78e"
 version = "1.5.0"
@@ -134,3 +134,7 @@ uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
 [[nghttp2_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
+
+[[p7zip_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"

ext/HistoricaStdlibGenerator/generate_historical_stdlibs.jl

@@ -34,7 +34,7 @@ function generate_nightly_url(major, minor, host = HostPlatform())
     # Map arch
     arch_str = Dict("x86_64" => "x64", "i686" => "x86", "aarch64" => "aarch64", "armv7l" => "armv7l", "ppc64le" => "ppc64le")[arch(host)]
     # Map OS name
-    os_str = Dict("linux" => "linux", "windows" => "winnt", "macos" => "macos", "freebsd" => "freebsd")[os(host)]    
+    os_str = Dict("linux" => "linux", "windows" => "winnt", "macos" => "mac", "freebsd" => "freebsd")[os(host)]
     # Map wordsize tag
     wordsize_str = Dict("x86_64" => "64", "i686" => "32", "aarch64" => "aarch64", "armv7l" => "armv7l", "ppc64le" => "ppc64")[arch(host)]
 
@@ -48,7 +48,7 @@ function generate_nightly_url(major, minor, host = HostPlatform())
         string(major), ".", string(minor), "/",
         "julia-latest-",
         # linux64
-        os(host), wordsize_str,
+        os_str, wordsize_str,
         ".tar.gz",
     )
 end
@@ -68,20 +68,56 @@ function get_stdlibs(scratch_dir, julia_installer_name)
     installer_path = joinpath(scratch_dir, julia_installer_name)
     mktempdir() do dir
         @info("Extracting $(julia_installer_name)")
-        run(`tar -C $(dir) --strip-components=1 -zxf $(installer_path)`)
-
-        jlexe = joinpath(dir, "bin", @static Sys.iswindows() ? "julia.exe" : "julia")
-        jlflags = ["--startup-file=no", "-O0"]
-        jlvers = VersionNumber(readchomp(`$(jlexe) $(jlflags) -e 'print(VERSION)'`))
-        jlvers = VersionNumber(jlvers.major, jlvers.minor, jlvers.patch)
-        @info("Auto-detected Julia version $(jlvers)")
-
-        if jlvers < v"1.1"
-            stdlibs_str = readchomp(`$(jlexe) $(jlflags) -e 'import Pkg; println(repr(Pkg.Types.gather_stdlib_uuids()))'`)
-        else
-            stdlibs_str = readchomp(`$(jlexe) $(jlflags) -e 'import Pkg; println(repr(Pkg.Types.load_stdlib()))'`)
+        mount_dir = joinpath(dir, "mount_dir")
+        try
+            if endswith(installer_path, ".dmg")
+                mkdir(mount_dir)
+                # Try to mount many times, as this seems to fail randomly
+                mount_cmd = `hdiutil mount $(installer_path) -mountpoint $(mount_dir)`
+                tries = 0
+                while !success(mount_cmd)
+                    if tries > 10
+                        error("Unable to mount via hdiutil!")
+                    end
+                    sleep(0.1)
+                    tries += 1
+                end
+                #@show readdir(mount_dir; join=true)
+                app_dir = first(filter(d -> startswith(basename(d), "Julia-"), readdir(mount_dir; join=true)))
+                symlink(joinpath(app_dir, "Contents", "Resources", "julia", "bin"), joinpath(dir, "bin"))
+            elseif endswith(installer_path, ".exe")
+                error("This script doesn't work with `.exe` downloads")
+            else
+                run(`tar -C $(dir) --strip-components=1 -zxf $(installer_path)`)
+            end
+
+            jlexe = joinpath(dir, "bin", @static Sys.iswindows() ? "julia.exe" : "julia")
+            jlflags = ["--startup-file=no", "-O0"]
+            jlvers = VersionNumber(readchomp(`$(jlexe) $(jlflags) -e 'print(VERSION)'`))
+            jlvers = VersionNumber(jlvers.major, jlvers.minor, jlvers.patch)
+            @info("Auto-detected Julia version $(jlvers)")
+
+            if jlvers < v"1.1"
+                stdlibs_str = readchomp(`$(jlexe) $(jlflags) -e 'import Pkg; println(repr(Pkg.Types.gather_stdlib_uuids()))'`)
+            else
+                stdlibs_str = readchomp(`$(jlexe) $(jlflags) -e 'import Pkg; println(repr(Pkg.Types.load_stdlib()))'`)
+            end
+
+            return (jlvers, stdlibs_str)
+        finally
+            # Clean up mounted directories
+            if isdir(mount_dir)
+                unmount_cmd = `hdiutil detach $(mount_dir)`
+                tries = 0
+                while !success(unmount_cmd)
+                    if tries > 10
+                        error("Unable to unmount $(mount_dir)")
+                    end
+                    sleep(0.1)
+                    tries += 1
+                end
+            end
         end
-        return (jlvers, stdlibs_str)
     end
 end
 
@@ -101,21 +137,35 @@ versions_dict = Dict()
     for _ in 1:num_concurrent_downloads
         @async begin
             for (url, hash) in jobs
-                fname = joinpath(scratch_dir, basename(url))
-                if !isfile(fname)
-                    @info("Downloading $(url)")
-                    Downloads.download(url, fname)
-                end
+                try
+                    fname = joinpath(scratch_dir, basename(url))
+                    if !isfile(fname)
+                        @info("Downloading $(url)")
+                        Downloads.download(url, fname)
+                    end
 
-                if !isempty(hash)
-                    calc_hash = bytes2hex(open(io -> sha256(io), fname, "r"))
-                    if calc_hash != hash
-                        @error("Hash mismatch on $(fname)")
+                    if !isempty(hash)
+                        calc_hash = bytes2hex(open(io -> sha256(io), fname, "r"))
+                        if calc_hash != hash
+                            @error("Hash mismatch on $(fname); deleting and re-downloading")
+                            rm(fname; force=true)
+                            Downloads.download(url, fname)
+                            calc_hash = bytes2hex(open(io -> sha256(io), fname, "r"))
+                            if calc_hash != hash
+                                @error("Hash mismatch on $(fname); re-download failed!")
+                                continue
+                            end
+                        end
                     end
-                end
 
-                version, stdlibs = get_stdlibs(scratch_dir, basename(url))
-                versions_dict[version] = eval(Meta.parse(stdlibs))
+                    version, stdlibs = get_stdlibs(scratch_dir, basename(url))
+                    versions_dict[version] = eval(Meta.parse(stdlibs))
+                catch e
+                    if isa(e, InterruptException)
+                        rethrow()
+                    end
+                    @error(e, exception=(e, catch_backtrace()))
+                end
             end
         end
     end
@@ -145,12 +195,12 @@ unregistered_stdlibs = filter(all_stdlibs) do (uuid, name)
 end
 
 # Helper function for getting these printed out in a nicely-sorted order
-function print_sorted(io::IO, d::Dict)
+function print_sorted(io::IO, d::Dict; indent::Int=0)
     println(io, "Dict(")
     for pair in sort(collect(d), by = kv-> kv[2])
-        println(io, "        ", pair, ",")
+        println(io, " "^indent, pair, ",")
     end
-    println(io, "    ),")
+    print(io, " "^(max(indent - 4, 0)), ")")
 end
 
 output_fname = joinpath(dirname(dirname(@__DIR__)), "src", "HistoricalStdlibs.jl")
@@ -168,18 +218,16 @@ open(output_fname, "w") do io
     """)
     for v in sorted_versions
         print(io, "    $(repr(v)) => ")
-        print_sorted(io, versions_dict[v])
+        print_sorted(io, versions_dict[v]; indent=8)
+        println(io, ",")
     end
     println(io, "]")
 
     print(io, """
     # Next, we also embed a list of stdlibs that must _always_ be treated as stdlibs,
     # because they cannot be resolved in the registry; they have only ever existed within
     # the Julia stdlib source tree, and because of that, trying to resolve them will fail.
-    const UNREGISTERED_STDLIBS = Dict(
-    """)
-    for (uuid, name) in unregistered_stdlibs
-        println(io, "    $(repr(uuid)) => $(repr(name)),")
-    end
-    println(io, ")")
+    const UNREGISTERED_STDLIBS = """)
+    print_sorted(io, unregistered_stdlibs; indent=4)
+    println(io)
 end

src/HistoricalStdlibs.jl

@@ -100,32 +100,31 @@ const STDLIBS_BY_VERSION = [
 # because they cannot be resolved in the registry; they have only ever existed within
 # the Julia stdlib source tree, and because of that, trying to resolve them will fail.
 const UNREGISTERED_STDLIBS = Dict(
-    UUID("9a3f8284-a2c9-5f02-9a11-845980a1fd5c") => "Random",
-    UUID("10745b16-79ce-11e8-11f9-7d13ad32a3b2") => "Statistics",
-    UUID("8bb1440f-4735-579b-a4ab-409b98df4dab") => "DelimitedFiles",
-    UUID("a63ad114-7e13-5084-954f-fe012c677804") => "Mmap",
-    UUID("8f399da3-3557-5675-b5ff-fb832c97cbdb") => "Libdl",
+    UUID("2a0f44e3-6c83-55bd-87e4-b1978d98bd5f") => "Base64",
     UUID("8bf52ea8-c179-5cab-976a-9e18b702a9bc") => "CRC32c",
+    UUID("ade2ca70-3891-5945-98fb-dc099432e06a") => "Dates",
+    UUID("8bb1440f-4735-579b-a4ab-409b98df4dab") => "DelimitedFiles",
+    UUID("8ba89e20-285c-5b6f-9357-94700520ee1b") => "Distributed",
+    UUID("7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee") => "FileWatching",
+    UUID("9fa8497b-333b-5362-9e8d-4d0656e87820") => "Future",
     UUID("b77e0a4c-d291-57a0-90e8-8db25a27a240") => "InteractiveUtils",
-    UUID("de0858da-6303-5e67-8744-51eddeeeb8d7") => "Printf",
+    UUID("76f85450-5226-5b5a-8eaa-529ad045b433") => "LibGit2",
+    UUID("8f399da3-3557-5675-b5ff-fb832c97cbdb") => "Libdl",
+    UUID("37e2e46d-f89d-539d-b4ee-838fcccc9c8e") => "LinearAlgebra",
+    UUID("56ddb016-857b-54e1-b83d-db4d58db5568") => "Logging",
     UUID("d6f4376e-aef5-505a-96c1-9c027394607a") => "Markdown",
-    UUID("4af54fe1-eca0-43a8-85a7-787d91b784e3") => "LazyArtifacts",
-    UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb") => "REPL",
-    UUID("6462fe0b-24de-5631-8697-dd941f90decc") => "Sockets",
+    UUID("a63ad114-7e13-5084-954f-fe012c677804") => "Mmap",
     UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f") => "Pkg",
-    UUID("37e2e46d-f89d-539d-b4ee-838fcccc9c8e") => "LinearAlgebra",
+    UUID("de0858da-6303-5e67-8744-51eddeeeb8d7") => "Printf",
     UUID("9abbd945-dff8-562f-b5e8-e1ebf5ef1b79") => "Profile",
-    UUID("2a0f44e3-6c83-55bd-87e4-b1978d98bd5f") => "Base64",
-    UUID("7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee") => "FileWatching",
+    UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb") => "REPL",
+    UUID("9a3f8284-a2c9-5f02-9a11-845980a1fd5c") => "Random",
     UUID("9e88b42a-f829-5b0c-bbe9-9e923198166b") => "Serialization",
-    UUID("56ddb016-857b-54e1-b83d-db4d58db5568") => "Logging",
+    UUID("1a1011a3-84de-559e-8e89-a11a2f7dc383") => "SharedArrays",
+    UUID("6462fe0b-24de-5631-8697-dd941f90decc") => "Sockets",
+    UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf") => "SparseArrays",
+    UUID("10745b16-79ce-11e8-11f9-7d13ad32a3b2") => "Statistics",
+    UUID("8dfed614-e22c-5e08-85e1-65c5234f0b40") => "Test",
     UUID("cf7118a7-6976-5b1a-9a39-7adc72f591a4") => "UUIDs",
-    UUID("ade2ca70-3891-5945-98fb-dc099432e06a") => "Dates",
     UUID("4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5") => "Unicode",
-    UUID("8dfed614-e22c-5e08-85e1-65c5234f0b40") => "Test",
-    UUID("9fa8497b-333b-5362-9e8d-4d0656e87820") => "Future",
-    UUID("76f85450-5226-5b5a-8eaa-529ad045b433") => "LibGit2",
-    UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf") => "SparseArrays",
-    UUID("1a1011a3-84de-559e-8e89-a11a2f7dc383") => "SharedArrays",
-    UUID("8ba89e20-285c-5b6f-9357-94700520ee1b") => "Distributed",
 )