Merge pull request #2354 from ianshmean/ib/16_more_backport

[release-1.6] more backport

Author: KristofferC

ext/HistoricaStdlibGenerator/generate_historical_stdlibs.jl

@@ -164,7 +164,7 @@ open(output_fname, "w") do io
 
     # Julia standard libraries with duplicate entries removed so as to store only the
     # first release in a set of releases that all contain the same set of stdlibs.
-    STDLIBS_BY_VERSION = [
+    const STDLIBS_BY_VERSION = [
     """)
     for v in sorted_versions
         print(io, "    $(repr(v)) => ")
@@ -176,7 +176,7 @@ open(output_fname, "w") do io
     # Next, we also embed a list of stdlibs that must _always_ be treated as stdlibs,
     # because they cannot be resolved in the registry; they have only ever existed within
     # the Julia stdlib source tree, and because of that, trying to resolve them will fail.
-    UNREGISTERED_STDLIBS = Dict(
+    const UNREGISTERED_STDLIBS = Dict(
     """)
     for (uuid, name) in unregistered_stdlibs
         println(io, "    $(repr(uuid)) => $(repr(name)),")

src/API.jl

@@ -910,11 +910,16 @@ function make_pkgspec(man, uuid)
 end
 
 precompile(; kwargs...) = precompile(Context(); kwargs...)
-function precompile(ctx::Context; internal_call::Bool=false, kwargs...)
+function precompile(ctx::Context; internal_call::Bool=false, strict::Bool=false, kwargs...)
     Context!(ctx; kwargs...)
     instantiate(ctx; allow_autoprecomp=false, kwargs...)
     time_start = time_ns()
-    num_tasks = parse(Int, get(ENV, "JULIA_NUM_PRECOMPILE_TASKS", string(Sys.CPU_THREADS::Int + 1)))
+
+    # Windows sometimes hits a ReadOnlyMemoryError, so we halve the default number of tasks. Issue #2323
+    # TODO: Investigate why this happens in windows and restore the full task limit
+    default_num_tasks = Sys.iswindows() ? div(Sys.CPU_THREADS::Int, 2) + 1 : Sys.CPU_THREADS::Int + 1
+
+    num_tasks = parse(Int, get(ENV, "JULIA_NUM_PRECOMPILE_TASKS", string(default_num_tasks)))
     parallel_limiter = Base.Semaphore(num_tasks)
     io = ctx.io
     fancyprint = can_fancyprint(io)
@@ -1139,7 +1144,7 @@ function precompile(ctx::Context; internal_call::Bool=false, kwargs...)
                         end
                     catch err
                         if err isa ErrorException
-                            failed_deps[pkg] = is_direct_dep ? String(take!(iob)) : ""
+                            failed_deps[pkg] = (strict || is_direct_dep) ? String(take!(iob)) : ""
                             !fancyprint && lock(print_lock) do
                                 println(io, string(color_string("  ✗ ", Base.error_color()), name))
                             end
@@ -1208,15 +1213,16 @@ function precompile(ctx::Context; internal_call::Bool=false, kwargs...)
             err_str = ""
             n_direct_errs = 0
             for (dep, err) in failed_deps
-                if dep in direct_deps
+                if strict || (dep in direct_deps)
                     err_str *= "\n" * "$dep" * "\n\n" * err * (n_direct_errs > 0 ? "\n" : "")
                     n_direct_errs += 1
                 end
             end
             if err_str != ""
                 println(io, "")
                 plural = n_direct_errs == 1 ? "y" : "ies"
-                pkgerror("The following $( n_direct_errs) direct dependenc$(plural) failed to precompile:\n$(err_str[1:end-1])")
+                direct = strict ? "" : "direct "
+                pkgerror("The following $n_direct_errs $(direct)dependenc$(plural) failed to precompile:\n$(err_str[1:end-1])")
             end
         end
     end

src/HistoricalStdlibs.jl

@@ -4,7 +4,7 @@ using Base: UUID
 
 # Julia standard libraries with duplicate entries removed so as to store only the
 # first release in a set of releases that all contain the same set of stdlibs.
-STDLIBS_BY_VERSION = [
+const STDLIBS_BY_VERSION = [
     v"1.0.0" => Dict(
         UUID("2a0f44e3-6c83-55bd-87e4-b1978d98bd5f") => "Base64",
         UUID("8bf52ea8-c179-5cab-976a-9e18b702a9bc") => "CRC32c",
@@ -99,7 +99,7 @@ STDLIBS_BY_VERSION = [
 # Next, we also embed a list of stdlibs that must _always_ be treated as stdlibs,
 # because they cannot be resolved in the registry; they have only ever existed within
 # the Julia stdlib source tree, and because of that, trying to resolve them will fail.
-UNREGISTERED_STDLIBS = Dict(
+const UNREGISTERED_STDLIBS = Dict(
     UUID("9a3f8284-a2c9-5f02-9a11-845980a1fd5c") => "Random",
     UUID("10745b16-79ce-11e8-11f9-7d13ad32a3b2") => "Statistics",
     UUID("8bb1440f-4735-579b-a4ab-409b98df4dab") => "DelimitedFiles",

src/Operations.jl

@@ -116,9 +116,15 @@ function is_instantiated(ctx::Context)::Bool
     pkgs = load_all_deps(ctx)
     # If the top-level project is a package, ensure it is instantiated as well
     if ctx.env.pkg !== nothing
-        push!(pkgs, Types.PackageSpec(
-            name=ctx.env.pkg.name, uuid=ctx.env.pkg.uuid, version=ctx.env.pkg.version, path=dirname(ctx.env.project_file)
-        ))
+        # Top-level project may already be in the manifest (cyclic deps)
+        # so only add it if it isn't there
+        idx = findfirst(x -> x.uuid == ctx.env.pkg.uuid, pkgs)
+        if idx === nothing
+            push!(pkgs, Types.PackageSpec(
+                name=ctx.env.pkg.name, uuid=ctx.env.pkg.uuid, version=ctx.env.pkg.version,
+                path=dirname(ctx.env.project_file)
+            ))
+        end
     end
     # Make sure all paths/artifacts exist
     return all(pkg -> is_package_downloaded(ctx, pkg), pkgs)

src/Pkg.jl

@@ -127,12 +127,13 @@ See also [`PackageSpec`](@ref).
 const add = API.add
 
 """
-    Pkg.precompile()
+    Pkg.precompile(; strict::Bool=false)
 
 Precompile all the dependencies of the project in parallel.
 !!! note
     Errors will only throw when precompiling the top-level dependencies, given that
     not all manifest dependencies may be loaded by the top-level dependencies on the given system.
+    This can be overridden to make errors in all dependencies throw by setting the kwarg `strict` to `true`
 
 !!! note
     This method is called automatically after any Pkg action that changes the manifest.
@@ -586,7 +587,7 @@ end
 ##################
 
 function _auto_precompile(ctx::Types.Context)
-    if tryparse(Int, get(ENV, "JULIA_PKG_PRECOMPILE_AUTO", "1")) == 1
+    if Base.JLOptions().use_compiled_modules == 1 && tryparse(Int, get(ENV, "JULIA_PKG_PRECOMPILE_AUTO", "1")) == 1
         Pkg.precompile(ctx; internal_call=true)
     end
 end

src/PlatformEngines.jl

@@ -27,7 +27,7 @@ function find7z()
     name = "7z"
     Sys.iswindows() && (name = "$name.exe")
     for dir in (joinpath("..", "libexec"), ".")
-        path = normpath(Sys.BINDIR, dir, name)
+        path = normpath(Sys.BINDIR::String, dir, name)
         isfile(path) && return path
     end
     path = Sys.which(name)

src/Types.jl

@@ -986,6 +986,9 @@ end
 
 pkg_server_url_hash(url::String) = split(url, '/')[end]
 
+registry_use_pkg_server() =
+    !Sys.iswindows() || haskey(ENV, "JULIA_PKG_SERVER")
+
 # entry point for `registry add`
 function clone_or_cp_registries(ctx::Context, regs::Vector{RegistrySpec}, depot::String=depots1())
     populate_known_registries_with_urls!(regs)
@@ -998,7 +1001,7 @@ function clone_or_cp_registries(ctx::Context, regs::Vector{RegistrySpec}, depot:
         mktempdir() do tmp
             url, registry_urls = pkg_server_registry_url(reg.uuid, registry_urls)
             # on Windows we prefer git cloning because untarring is so slow
-            if !Sys.iswindows() && url !== nothing
+            if url !== nothing && registry_use_pkg_server()
                 # download from Pkg server
                 try
                     download_verify_unpack(url, nothing, tmp, ignore_existence = true)
@@ -1031,7 +1034,7 @@ function clone_or_cp_registries(ctx::Context, regs::Vector{RegistrySpec}, depot:
             # slug = Base.package_slug(UUID(registry["uuid"]))
             regpath = joinpath(depot, "registries", registry["name"]::String#=, slug=#)
             ispath(dirname(regpath)) || mkpath(dirname(regpath))
-            if Pkg.isdir_nothrow(regpath)
+            if isfile(joinpath(regpath, "Registry.toml"))
                 existing_registry = read_registry(joinpath(regpath, "Registry.toml"))
                 if registry["uuid"] == existing_registry["uuid"]
                     println(ctx.io,
@@ -1043,7 +1046,7 @@ function clone_or_cp_registries(ctx::Context, regs::Vector{RegistrySpec}, depot:
                         "`$(Base.contractuser(joinpath(depot, "registries", registry["name"]*"-2")))`."))
                 end
             else
-                mv(tmp, regpath)
+                mv(tmp, regpath, force=true)
                 printpkgstyle(ctx, :Added, "registry `$(registry["name"])` to `$(Base.contractuser(regpath))`")
             end
         end

test/misc.jl

@@ -0,0 +1,9 @@
+using Test
+using Pkg
+
+@testset "inference" begin
+    f() = Pkg.Types.STDLIBS_BY_VERSION
+    @inferred f()
+    f() = Pkg.Types.UNREGISTERED_STDLIBS
+    @inferred f()
+end

test/new.jl

@@ -2413,6 +2413,21 @@ end
             Pkg.add(path="A")
         end
     end
+    # test #2302
+    isolate(loaded_depot=true) do
+        cd_tempdir() do dir
+            Pkg.generate("A")
+            Pkg.generate("B")
+            git_init_and_commit("B")
+            Pkg.develop(path="B")
+            Pkg.activate("A")
+            Pkg.add(path="B")
+            git_init_and_commit("A")
+            Pkg.activate("B")
+            # This shouldn't error even though A has a dependency on B
+            Pkg.add(path="A")
+        end
+    end
 end
 
 @testset "Offline mode" begin

test/runtests.jl

@@ -26,6 +26,7 @@ include("binaryplatforms.jl")
 include("platformengines.jl")
 include("sandbox.jl")
 include("resolve.jl")
+include("misc.jl")
 
 # clean up locally cached registry
 rm(joinpath(@__DIR__, "registries"); force = true, recursive = true)