[ext/HSG]: Store next release _and_ latest nightly (#2418)

In the event that we are in an RC phase where we have three "latest"
branch tips (e.g. latest stable: 1.5.3, latest RC: 1.6.0-rc1 and latest
`master`: 1.7.0-DEV) each with their own set of standard libraries, we
must store this information in Pkg so that we pass tests when running on
latest `master`. It is harmless for this information to be baked into an
older Julia, as we make no guarantees that Julia 1.6.0 final should have
perfect information about what standard libraries 1.7.0 will have.

(cherry picked from commit 7a9d965)

Author: staticfloat

ext/HistoricaStdlibGenerator/generate_historical_stdlibs.jl

@@ -30,31 +30,38 @@ function select_url_hash(data, host = HostPlatform())
 end
 version_urls = sort(select_url_hash.(values(versions)), by = pair -> pair[1])
 
-function generate_nightly_url(major, minor, host = HostPlatform())
+function generate_nightly_url(jlver, host = HostPlatform())
     # Map arch
     arch_str = Dict("x86_64" => "x64", "i686" => "x86", "aarch64" => "aarch64", "armv7l" => "armv7l", "ppc64le" => "ppc64le")[arch(host)]
     # Map OS name
     os_str = Dict("linux" => "linux", "windows" => "winnt", "macos" => "mac", "freebsd" => "freebsd")[os(host)]
     # Map wordsize tag
     wordsize_str = Dict("x86_64" => "64", "i686" => "32", "aarch64" => "aarch64", "armv7l" => "armv7l", "ppc64le" => "ppc64")[arch(host)]
 
+    # If `jlver` is nothing, we don't namespace by version and just get the absolute latest version
+    ver_str = ""
+    if jlver !== nothing
+        ver_str = string(jlver.major, ".", jlver.minor, "/")
+    end
+
     return string(
         "https://julialangnightlies-s3.julialang.org/bin/",
         # linux/
         os_str, "/",
         # x64/
         arch_str, "/",
-        # 1.6/
-        string(major), ".", string(minor), "/",
+        # 1.6/ (or nothing, if `jlver === nothing`)
+        ver_str,
         "julia-latest-",
         # linux64
         os_str, wordsize_str,
         ".tar.gz",
     )
 end
 highest_release = maximum(VersionNumber.(string.(keys(versions))))
-next_release_url = generate_nightly_url(highest_release.major, highest_release.minor + 1)
-push!(version_urls, (next_release_url, ""))
+next_release = VersionNumber(highest_release.major, highest_release.minor + 1, 0)
+push!(version_urls, (generate_nightly_url(next_release), ""))
+push!(version_urls, (generate_nightly_url(nothing),      ""))
 @info("Identified $(length(version_urls)) versions to try...")
 
 # Next, we're going to download each of these to a scratch space
@@ -138,7 +145,9 @@ versions_dict = Dict()
         @async begin
             for (url, hash) in jobs
                 try
-                    fname = joinpath(scratch_dir, basename(url))
+                    # We might try to download two files that have the same basename
+                    url_tag = bytes2hex(sha256(url))
+                    fname = joinpath(scratch_dir, string(url_tag, "-", basename(url)))
                     if !isfile(fname)
                         @info("Downloading $(url)")
                         Downloads.download(url, fname)
@@ -158,7 +167,7 @@ versions_dict = Dict()
                         end
                     end
 
-                    version, stdlibs = get_stdlibs(scratch_dir, basename(url))
+                    version, stdlibs = get_stdlibs(scratch_dir, basename(fname))
                     versions_dict[version] = eval(Meta.parse(stdlibs))
                 catch e
                     if isa(e, InterruptException)

src/HistoricalStdlibs.jl

@@ -95,6 +95,66 @@ const STDLIBS_BY_VERSION = [
         UUID("8e850ede-7688-5339-a07c-302acd2aaf8d") => "nghttp2_jll",
         UUID("3f19e933-33d8-53b3-aaab-bd5110c3b7a0") => "p7zip_jll",
     ),
+    v"1.7.0" => Dict(
+        UUID("0dad84c5-d112-42e6-8d28-ef12dabb789f") => "ArgTools",
+        UUID("56f22d72-fd6d-98f1-02f0-08ddc0907c33") => "Artifacts",
+        UUID("2a0f44e3-6c83-55bd-87e4-b1978d98bd5f") => "Base64",
+        UUID("8bf52ea8-c179-5cab-976a-9e18b702a9bc") => "CRC32c",
+        UUID("e66e0078-7015-5450-92f7-15fbd957f2ae") => "CompilerSupportLibraries_jll",
+        UUID("ade2ca70-3891-5945-98fb-dc099432e06a") => "Dates",
+        UUID("8bb1440f-4735-579b-a4ab-409b98df4dab") => "DelimitedFiles",
+        UUID("8ba89e20-285c-5b6f-9357-94700520ee1b") => "Distributed",
+        UUID("f43a241f-c20a-4ad4-852c-f6b1247861c6") => "Downloads",
+        UUID("7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee") => "FileWatching",
+        UUID("9fa8497b-333b-5362-9e8d-4d0656e87820") => "Future",
+        UUID("781609d7-10c4-51f6-84f2-b8444358ff6d") => "GMP_jll",
+        UUID("b77e0a4c-d291-57a0-90e8-8db25a27a240") => "InteractiveUtils",
+        UUID("47c5dbc3-30ba-59ef-96a6-123e260183d9") => "LLVMLibUnwind_jll",
+        UUID("4af54fe1-eca0-43a8-85a7-787d91b784e3") => "LazyArtifacts",
+        UUID("b27032c2-a3e7-50c8-80cd-2d36dbcbfd21") => "LibCURL",
+        UUID("deac9b47-8bc7-5906-a0fe-35ac56dc84c0") => "LibCURL_jll",
+        UUID("76f85450-5226-5b5a-8eaa-529ad045b433") => "LibGit2",
+        UUID("e37daf67-58a4-590a-8e99-b0245dd2ffc5") => "LibGit2_jll",
+        UUID("29816b5a-b9ab-546f-933c-edad1886dfa8") => "LibSSH2_jll",
+        UUID("183b4373-6708-53ba-ad28-60e28bb38547") => "LibUV_jll",
+        UUID("745a5e78-f969-53e9-954f-d19f2f74f4e3") => "LibUnwind_jll",
+        UUID("8f399da3-3557-5675-b5ff-fb832c97cbdb") => "Libdl",
+        UUID("37e2e46d-f89d-539d-b4ee-838fcccc9c8e") => "LinearAlgebra",
+        UUID("56ddb016-857b-54e1-b83d-db4d58db5568") => "Logging",
+        UUID("3a97d323-0669-5f0c-9066-3539efd106a3") => "MPFR_jll",
+        UUID("d6f4376e-aef5-505a-96c1-9c027394607a") => "Markdown",
+        UUID("c8ffd9c3-330d-5841-b78e-0817d7145fa1") => "MbedTLS_jll",
+        UUID("a63ad114-7e13-5084-954f-fe012c677804") => "Mmap",
+        UUID("14a3606d-f60d-562e-9121-12d972cd8159") => "MozillaCACerts_jll",
+        UUID("ca575930-c2e3-43a9-ace4-1e988b2c1908") => "NetworkOptions",
+        UUID("4536629a-c528-5b80-bd46-f80d51c5b363") => "OpenBLAS_jll",
+        UUID("05823500-19ac-5b8b-9628-191a04bc5112") => "OpenLibm_jll",
+        UUID("efcefdf7-47ab-520b-bdef-62a2eaa19f15") => "PCRE2_jll",
+        UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f") => "Pkg",
+        UUID("de0858da-6303-5e67-8744-51eddeeeb8d7") => "Printf",
+        UUID("9abbd945-dff8-562f-b5e8-e1ebf5ef1b79") => "Profile",
+        UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb") => "REPL",
+        UUID("9a3f8284-a2c9-5f02-9a11-845980a1fd5c") => "Random",
+        UUID("ea8e919c-243c-51af-8825-aaa63cd721ce") => "SHA",
+        UUID("9e88b42a-f829-5b0c-bbe9-9e923198166b") => "Serialization",
+        UUID("1a1011a3-84de-559e-8e89-a11a2f7dc383") => "SharedArrays",
+        UUID("6462fe0b-24de-5631-8697-dd941f90decc") => "Sockets",
+        UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf") => "SparseArrays",
+        UUID("10745b16-79ce-11e8-11f9-7d13ad32a3b2") => "Statistics",
+        UUID("4607b0f0-06f3-5cda-b6b1-a6196a1729e9") => "SuiteSparse",
+        UUID("bea87d4a-7f5b-5778-9afe-8cc45184846c") => "SuiteSparse_jll",
+        UUID("fa267f1f-6049-4f14-aa54-33bafae1ed76") => "TOML",
+        UUID("a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e") => "Tar",
+        UUID("8dfed614-e22c-5e08-85e1-65c5234f0b40") => "Test",
+        UUID("cf7118a7-6976-5b1a-9a39-7adc72f591a4") => "UUIDs",
+        UUID("4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5") => "Unicode",
+        UUID("83775a58-1f1d-513f-b197-d71354ab007a") => "Zlib_jll",
+        UUID("05ff407c-b0c1-5878-9df8-858cc2e60c36") => "dSFMT_jll",
+        UUID("8f36deef-c2a5-5394-99ed-8e07531fb29a") => "libLLVM_jll",
+        UUID("8e850b90-86db-534c-a0d3-1478176c7d93") => "libblastrampoline_jll",
+        UUID("8e850ede-7688-5339-a07c-302acd2aaf8d") => "nghttp2_jll",
+        UUID("3f19e933-33d8-53b3-aaab-bd5110c3b7a0") => "p7zip_jll",
+    ),
 ]
 # Next, we also embed a list of stdlibs that must _always_ be treated as stdlibs,
 # because they cannot be resolved in the registry; they have only ever existed within