Merge pull request #2592 from JuliaLang/fe/backports-release-1.6

Backports for Pkg 1.6.2

Author: fredrikekre

src/API.jl

@@ -16,6 +16,7 @@ import ..can_fancyprint, ..DEFAULT_IO
 using ..Types, ..TOML
 using ..Types: VersionTypes
 using Base.BinaryPlatforms
+import ..stderr_f, ..stdout_f
 using ..Artifacts: artifact_paths
 using ..MiniProgressBars
 
@@ -73,7 +74,7 @@ for f in (:develop, :add, :rm, :up, :pin, :free, :test, :build, :status)
     @eval begin
         $f(pkg::Union{AbstractString, PackageSpec}; kwargs...) = $f([pkg]; kwargs...)
         $f(pkgs::Vector{<:AbstractString}; kwargs...)          = $f([PackageSpec(pkg) for pkg in pkgs]; kwargs...)
-        function $f(pkgs::Vector{PackageSpec}; io::IO=DEFAULT_IO[], kwargs...)
+        function $f(pkgs::Vector{PackageSpec}; io::IO=$(f === :status ? :stdout_f : :stderr_f)(), kwargs...)
             ctx = Context()
             kwargs = merge((;kwargs...), (:io => io,))
             ret = $f(ctx, pkgs; kwargs...)
@@ -261,7 +262,7 @@ function up(ctx::Context, pkgs::Vector{PackageSpec};
     return
 end
 
-resolve(; io::IO=DEFAULT_IO[], kwargs...) = resolve(Context(;io); kwargs...)
+resolve(; io::IO=stderr_f(), kwargs...) = resolve(Context(;io); kwargs...)
 function resolve(ctx::Context; kwargs...)
     up(ctx; level=UPLEVEL_FIXED, mode=PKGMODE_MANIFEST, update_registry=false, kwargs...)
     return nothing
@@ -924,6 +925,7 @@ function precompile(ctx::Context; internal_call::Bool=false, strict::Bool=false,
     # Windows sometimes hits a ReadOnlyMemoryError, so we halve the default number of tasks. Issue #2323
     # TODO: Investigate why this happens in windows and restore the full task limit
     default_num_tasks = Sys.iswindows() ? div(Sys.CPU_THREADS::Int, 2) + 1 : Sys.CPU_THREADS::Int + 1
+    default_num_tasks = min(default_num_tasks, 16) # limit for better stability on shared resource systems
 
     num_tasks = parse(Int, get(ENV, "JULIA_NUM_PRECOMPILE_TASKS", string(default_num_tasks)))
     parallel_limiter = Base.Semaphore(num_tasks)
@@ -1008,9 +1010,11 @@ function precompile(ctx::Context; internal_call::Bool=false, strict::Bool=false,
     ansi_disablecursor = "\e[?25l"
     n_done::Int = 0
     n_already_precomp::Int = 0
+    n_loaded::Int = 0
 
     function handle_interrupt(err)
         notify(interrupted_or_done)
+        sleep(0.2) # yield for a period to let the print loop cease first
         if err isa InterruptException
             lock(print_lock) do
                 println(io, " Interrupted: Exiting precompilation...")
@@ -1054,15 +1058,16 @@ function precompile(ctx::Context; internal_call::Bool=false, strict::Bool=false,
                         bar.max = n_total - n_already_precomp
                         final_loop || print(iostr, sprint(io -> show_progress(io, bar); context=io), "\n")
                         for dep in pkg_queue_show
+                            loaded = haskey(Base.loaded_modules, dep)
                             name = dep in direct_deps ? dep.name : string(color_string(dep.name, :light_black))
                             if dep in precomperr_deps
                                 print(iostr, color_string("  ? ", Base.warn_color()), name, "\n")
                             elseif haskey(failed_deps, dep)
                                 print(iostr, color_string("  ✗ ", Base.error_color()), name, "\n")
                             elseif was_recompiled[dep]
-                                interrupted_or_done.set && continue
-                                print(iostr, color_string("  ✓ ", :green), name, "\n")
-                                @async begin # keep successful deps visible for short period
+                                !loaded && interrupted_or_done.set && continue
+                                print(iostr, color_string("  ✓ ", loaded ? Base.warn_color() : :green), name, "\n")
+                                loaded || @async begin # keep successful deps visible for short period
                                     sleep(1);
                                     filter!(!isequal(dep), pkg_queue)
                                 end
@@ -1109,6 +1114,7 @@ function precompile(ctx::Context; internal_call::Bool=false, strict::Bool=false,
 
         task = @async begin
             try
+                loaded = haskey(Base.loaded_modules, pkg)
                 for dep in deps # wait for deps to finish
                     wait(was_processed[dep])
                 end
@@ -1141,7 +1147,12 @@ function precompile(ctx::Context; internal_call::Bool=false, strict::Bool=false,
                     end
                     try
                         ret = Logging.with_logger(Logging.NullLogger()) do
-                            Base.compilecache(pkg, sourcepath, iob, devnull) # capture stderr, send stdout to devnull
+                            @static if hasmethod(Base.compilecache, Tuple{Base.PkgId, String, IO, IO, Bool})
+                                # capture stderr, send stdout to devnull, don't skip loaded modules
+                                Base.compilecache(pkg, sourcepath, iob, devnull, false)
+                            else
+                                Base.compilecache(pkg, sourcepath, iob, devnull)
+                            end
                         end
                         if ret isa Base.PrecompilableError
                             push!(precomperr_deps, pkg)
@@ -1152,13 +1163,14 @@ function precompile(ctx::Context; internal_call::Bool=false, strict::Bool=false,
                         else
                             queued && (haskey(man, pkg.uuid) && precomp_dequeue!(make_pkgspec(man, pkg.uuid)))
                             !fancyprint && lock(print_lock) do
-                                println(io, string(color_string("  ✓ ", :green), name))
+                                println(io, string(color_string("  ✓ ", loaded ? Base.warn_color() : :green), name))
                             end
                             was_recompiled[pkg] = true
                         end
+                        loaded && (n_loaded += 1)
                     catch err
-                        if err isa ErrorException
-                            failed_deps[pkg] = (strict || is_direct_dep) ? String(take!(iob)) : ""
+                        if err isa ErrorException || (err isa ArgumentError && startswith(err.msg, "Invalid header in cache file"))
+                            failed_deps[pkg] = (strict || is_direct_dep) ? string(sprint(showerror, err), "\n", String(take!(iob))) : ""
                             !fancyprint && lock(print_lock) do
                                 println(io, string(color_string("  ✗ ", Base.error_color()), name))
                             end
@@ -1211,6 +1223,15 @@ function precompile(ctx::Context; internal_call::Bool=false, strict::Bool=false,
                 !isempty(skipped_deps) && (print(iostr, ", $(length(skipped_deps)) skipped during auto due to previous errors"))
                 print(iostr, ")")
             end
+            if n_loaded > 0
+                plural1 = n_loaded == 1 ? "y" : "ies"
+                plural2 = n_loaded == 1 ? "a different version is" : "different versions are"
+                plural3 = n_loaded == 1 ? "" : "s"
+                print(iostr, "\n  ",
+                    color_string(string(n_loaded), Base.warn_color()),
+                    " dependenc$(plural1) precompiled but $(plural2) currently loaded. Restart julia to access the new version$(plural3)"
+                )
+            end
             if !isempty(precomperr_deps)
                 plural = length(precomperr_deps) == 1 ? "y" : "ies"
                 print(iostr, "\n  ",
@@ -1406,9 +1427,9 @@ function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool=false, mode=
 end
 
 
-function activate(;temp=false, shared=false, io::IO=DEFAULT_IO[])
+function activate(;temp=false, shared=false, io::IO=stderr_f())
     shared && pkgerror("Must give a name for a shared environment")
-    temp && return activate(mktempdir())
+    temp && return activate(mktempdir(); io)
     Base.ACTIVE_PROJECT[] = nothing
     p = Base.active_project()
     p === nothing || printpkgstyle(io, :Activating, "environment at $(pathrepr(p))")
@@ -1432,7 +1453,7 @@ function _activate_dep(dep_name::AbstractString)
         end
     end
 end
-function activate(path::AbstractString; shared::Bool=false, temp::Bool=false, io::IO=DEFAULT_IO[])
+function activate(path::AbstractString; shared::Bool=false, temp::Bool=false, io::IO=stderr_f())
     temp && pkgerror("Can not give `path` argument when creating a temporary environment")
     if !shared
         # `pkg> activate path`/`Pkg.activate(path)` does the following

src/Artifacts.jl

@@ -11,7 +11,7 @@ import ..GitTools
 import ..TOML
 import ..Types: parse_toml, write_env_usage, printpkgstyle
 import ..Pkg
-import ..Pkg: pkg_server, can_fancyprint, DEFAULT_IO
+import ..Pkg: pkg_server, can_fancyprint, stderr_f
 using ..Pkg.MiniProgressBars
 using ..PlatformEngines
 using SHA
@@ -286,7 +286,7 @@ end
 
 """
     download_artifact(tree_hash::SHA1, tarball_url::String, tarball_hash::String;
-                      verbose::Bool = false, io::IO=DEFAULT_IO[])
+                      verbose::Bool = false, io::IO=stderr)
 
 Download/install an artifact into the artifact store.  Returns `true` on success.
 
@@ -299,7 +299,7 @@ function download_artifact(
     tarball_hash::Union{String, Nothing} = nothing;
     verbose::Bool = false,
     quiet_download::Bool = false,
-    io::IO=DEFAULT_IO[],
+    io::IO=stderr_f(),
 )
     if artifact_exists(tree_hash)
         return true
@@ -321,12 +321,13 @@ function download_artifact(
         try
             download_verify_unpack(tarball_url, tarball_hash, dest_dir, ignore_existence=true,
                                    verbose=verbose, quiet_download=quiet_download, io=io)
-        catch e
+        catch err
+            @debug "download_artifact error" tree_hash tarball_url tarball_hash err
             # Clean that destination directory out if something went wrong
             rm(dest_dir; force=true, recursive=true)
 
-            if isa(e, InterruptException)
-                rethrow(e)
+            if isa(err, InterruptException)
+                rethrow(err)
             end
             return false
         end
@@ -343,9 +344,10 @@ function download_artifact(
                 download_verify_unpack(tarball_url, tarball_hash, dir, ignore_existence=true, verbose=verbose,
                     quiet_download=quiet_download, io=io)
             end
-        catch e
-            if isa(e, InterruptException)
-                rethrow(e)
+        catch err
+            @debug "download_artifact error" tree_hash tarball_url tarball_hash err
+            if isa(err, InterruptException)
+                rethrow(err)
             end
             # If something went wrong during download, return false
             return false
@@ -384,7 +386,7 @@ end
                               pkg_uuid::Union{Base.UUID,Nothing}=nothing,
                               verbose::Bool = false,
                               quiet_download::Bool = false,
-                              io::IO=DEFAULT_IO[])
+                              io::IO=stderr)
 
 Ensures an artifact is installed, downloading it via the download information stored in
 `artifacts_toml` if necessary.  Throws an error if unable to install.
@@ -397,7 +399,7 @@ function ensure_artifact_installed(name::String, artifacts_toml::String;
                                    pkg_uuid::Union{Base.UUID,Nothing}=nothing,
                                    verbose::Bool = false,
                                    quiet_download::Bool = false,
-                                   io::IO=DEFAULT_IO[])
+                                   io::IO=stderr_f())
     meta = artifact_meta(name, artifacts_toml; pkg_uuid=pkg_uuid, platform=platform)
     if meta === nothing
         error("Cannot locate artifact '$(name)' in '$(artifacts_toml)'")
@@ -411,7 +413,7 @@ function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::Str
                                    platform::AbstractPlatform = HostPlatform(),
                                    verbose::Bool = false,
                                    quiet_download::Bool = false,
-                                   io::IO=DEFAULT_IO[])
+                                   io::IO=stderr_f())
     hash = SHA1(meta["git-tree-sha1"])
 
     if !artifact_exists(hash)
@@ -470,7 +472,7 @@ end
                                    include_lazy = false,
                                    verbose = false,
                                    quiet_download = false,
-                                   io::IO=DEFAULT_IO[])
+                                   io::IO=stderr)
 
 Installs all non-lazy artifacts from a given `(Julia)Artifacts.toml` file. `package_uuid` must
 be provided to properly support overrides from `Overrides.toml` entries in depots.
@@ -486,7 +488,7 @@ function ensure_all_artifacts_installed(artifacts_toml::String;
                                         include_lazy::Bool = false,
                                         verbose::Bool = false,
                                         quiet_download::Bool = false,
-                                        io::IO=DEFAULT_IO[])
+                                        io::IO=stderr_f())
     if !isfile(artifacts_toml)
         return
     end

src/MiniProgressBars.jl

@@ -7,7 +7,7 @@ using Printf
 Base.@kwdef mutable struct MiniProgressBar
     max::Int = 1.0
     header::String = ""
-    color::Symbol = :white
+    color::Symbol = :nothing
     width::Int = 40
     current::Int = 0.0
     prev::Int = 0.0

src/Operations.jl

@@ -14,7 +14,7 @@ import ..Artifacts: ensure_all_artifacts_installed, artifact_names, extract_all_
 using Base.BinaryPlatforms
 import ...Pkg
 import ...Pkg: pkg_server
-import ...Pkg: can_fancyprint, DEFAULT_IO
+import ...Pkg: can_fancyprint, stderr_f
 import ..Types: printpkgstyle
 
 #########
@@ -567,7 +567,7 @@ function install_archive(
     urls::Vector{Pair{String,Bool}},
     hash::SHA1,
     version_path::String;
-    io::IO=DEFAULT_IO[]
+    io::IO=stderr_f()
 )::Bool
     tmp_objects = String[]
     url_success = false
@@ -686,7 +686,7 @@ end
 function download_artifacts(ctx::Context, pkg_roots::Vector{String};
                             platform::AbstractPlatform=HostPlatform(),
                             verbose::Bool=false,
-                            io::IO=DEFAULT_IO[])
+                            io::IO=stderr_f())
     # List of Artifacts.toml files that we're going to download from
     artifacts_tomls = String[]
 
@@ -850,7 +850,7 @@ function prune_manifest(ctx::Context)
     ctx.env.manifest = prune_manifest(ctx.env.manifest, keep)
 end
 
-function prune_manifest(manifest::Dict, keep::Vector{UUID})
+function prune_manifest(manifest::Manifest, keep::Vector{UUID})
     while !isempty(keep)
         clean = true
         for (uuid, entry) in manifest
@@ -863,7 +863,8 @@ function prune_manifest(manifest::Dict, keep::Vector{UUID})
         end
         clean && break
     end
-    return Dict(uuid => entry for (uuid, entry) in manifest if uuid in keep)
+    manifest.deps = Dict(uuid => entry for (uuid, entry) in manifest if uuid in keep)
+    return manifest
 end
 
 function any_package_not_installed(ctx)
@@ -1463,9 +1464,9 @@ function sandbox_preserve(ctx::Context, target::PackageSpec, test_project::Strin
     return prune_manifest(env.manifest, keep)
 end
 
-abspath!(ctx::Context, manifest::Dict{UUID,PackageEntry}) =
+abspath!(ctx::Context, manifest::Manifest) =
     abspath!(dirname(ctx.env.project_file), manifest)
-function abspath!(project::String, manifest::Dict{UUID,PackageEntry})
+function abspath!(project::String, manifest::Manifest)
     for (uuid, entry) in manifest
         if entry.path !== nothing
             entry.path = project_rel_path(project, entry.path)
@@ -1524,7 +1525,7 @@ function sandbox(fn::Function, ctx::Context, target::PackageSpec, target_path::S
             catch err# TODO
                 @debug err
                 @warn "Could not use exact versions of packages in manifest, re-resolving"
-                temp_ctx.env.manifest = Dict(uuid => entry for (uuid, entry) in temp_ctx.env.manifest if isfixed(entry))
+                temp_ctx.env.manifest.deps = Dict(uuid => entry for (uuid, entry) in temp_ctx.env.manifest.deps if isfixed(entry))
                 Pkg.resolve(temp_ctx; io=devnull)
                 @debug "Using _clean_ dep graph"
             end
@@ -1657,7 +1658,7 @@ function test(ctx::Context, pkgs::Vector{PackageSpec};
         printpkgstyle(ctx, :Testing, pkg.name)
         sandbox(ctx, pkg, source_path, testdir(source_path), test_project_override) do
             test_fn !== nothing && test_fn()
-            sandbox_ctx = Context()
+            sandbox_ctx = Context(;io=ctx.io)
             status(sandbox_ctx; mode=PKGMODE_COMBINED)
             Pkg._auto_precompile(sandbox_ctx)
             printpkgstyle(ctx, :Testing, "Running tests...")
@@ -1740,7 +1741,7 @@ function stat_rep(x::PackageSpec; name=true)
     return join(filter(!isempty, [name,version,repo,path,pinned]), " ")
 end
 
-print_single(ctx::Context, pkg::PackageSpec) = printstyled(ctx.io, stat_rep(pkg); color=:white)
+print_single(ctx::Context, pkg::PackageSpec) = print(ctx.io, stat_rep(pkg))
 
 is_instantiated(::Nothing) = false
 is_instantiated(x::PackageSpec) = x.version != VersionSpec() || is_stdlib(x.uuid)