Merge pull request #2629 from JuliaLang/kc/1.7

Backports for 1.7, take 2.

Author: KristofferC

src/API.jl

@@ -1408,13 +1408,14 @@ end
 instantiate(; kwargs...) = instantiate(Context(); kwargs...)
 function instantiate(ctx::Context; manifest::Union{Bool, Nothing}=nothing,
                      update_registry::Bool=true, verbose::Bool=false,
-                     platform::AbstractPlatform=HostPlatform(), allow_autoprecomp::Bool=true, kwargs...)
+                     platform::AbstractPlatform=HostPlatform(), allow_build::Bool=true, allow_autoprecomp::Bool=true, kwargs...)
     Context!(ctx; kwargs...)
     if Registry.download_default_registries(ctx.io)
         copy!(ctx.registries, Registry.reachable_registries())
     end
     if !isfile(ctx.env.project_file) && isfile(ctx.env.manifest_file)
         _manifest = Pkg.Types.read_manifest(ctx.env.manifest_file)
+        Types.check_warn_manifest_julia_version_compat(_manifest, ctx.env.manifest_file)
         deps = Dict{String,String}()
         for (uuid, pkg) in _manifest
             if pkg.name in keys(deps)
@@ -1433,6 +1434,7 @@ function instantiate(ctx::Context; manifest::Union{Bool, Nothing}=nothing,
     if !isfile(ctx.env.manifest_file) && manifest == true
         pkgerror("expected manifest file at `$(ctx.env.manifest_file)` but it does not exist")
     end
+    Types.check_warn_manifest_julia_version_compat(ctx.env.manifest, ctx.env.manifest_file)
     Operations.prune_manifest(ctx.env)
     for (name, uuid) in ctx.env.project.deps
         get(ctx.env.manifest, uuid, nothing) === nothing || continue
@@ -1496,7 +1498,7 @@ function instantiate(ctx::Context; manifest::Union{Bool, Nothing}=nothing,
     # Install all artifacts
     Operations.download_artifacts(ctx.env; platform=platform, verbose=verbose)
     # Run build scripts
-    Operations.build_versions(ctx, union(new_apply, new_git); verbose=verbose)
+    allow_build && Operations.build_versions(ctx, union(new_apply, new_git); verbose=verbose)
 
     allow_autoprecomp && Pkg._auto_precompile(ctx)
 end

src/Operations.jl

@@ -131,8 +131,8 @@ function update_manifest!(env::EnvCache, pkgs::Vector{PackageSpec}, deps_map, ju
         entry = PackageEntry(;name = pkg.name, version = pkg.version, pinned = pkg.pinned,
                              tree_hash = pkg.tree_hash, path = pkg.path, repo = pkg.repo, uuid=pkg.uuid)
         if is_stdlib(pkg.uuid, julia_version)
-            # do not set version for stdlibs
-            entry.version = nothing
+            # Only set stdlib versions for versioned (external) stdlibs
+            entry.version = stdlib_version(pkg.uuid, julia_version)
         end
         if Types.is_project(env, pkg)
             entry.deps = env.project.deps
@@ -288,10 +288,13 @@ end
 function resolve_versions!(env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, julia_version)
     # compatibility
     if julia_version !== nothing
+        env.manifest.julia_version = julia_version
         v = intersect(julia_version, get_compat(env.project, "julia"))
         if isempty(v)
             @warn "julia version requirement for project not satisfied" _module=nothing _file=nothing
         end
+    else
+        env.manifest.julia_version = VERSION
     end
     names = Dict{UUID, String}(uuid => stdlib for (uuid, stdlib) in stdlibs())
     # recursive search for packages which are tracking a path
@@ -381,6 +384,7 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance}
         union!(uuids, fixed_uuids)
     end
 
+    stdlibs_for_julia_version = Types.get_last_stdlibs(julia_version)
     seen = Set{UUID}()
 
     # pkg -> version -> (dependency => compat):
@@ -396,13 +400,21 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance}
         for uuid in unseen
             push!(seen, uuid)
             uuid in keys(fixed) && continue
-            all_compat_u   = get_or_make!(all_compat,   uuid)
+            all_compat_u = get_or_make!(all_compat,   uuid)
+
+            uuid_is_stdlib = false
+            stdlib_name = ""
+            stdlib_version = nothing
+            if haskey(stdlibs_for_julia_version, uuid)
+                uuid_is_stdlib = true
+                stdlib_name, stdlib_version = stdlibs_for_julia_version[uuid]
+            end
 
-            # If we're requesting resolution of a package that is an stdlib and is not registered,
-            # we must special-case it here.  This is further complicated by the fact that we can
-            # ask this question relative to a particular Julia version.
-            if is_unregistered_stdlib(uuid) || is_stdlib(uuid, julia_version)
-                path = Types.stdlib_path(stdlibs()[uuid])
+            # If we're requesting resolution of a package that is an unregistered stdlib (or one
+            # that tracks no version information) we must special-case it here.  This is further
+            # complicated by the fact that we can ask this question relative to a Julia version.
+            if is_unregistered_stdlib(uuid) || (uuid_is_stdlib && stdlib_version === nothing)
+                path = Types.stdlib_path(stdlibs_for_julia_version[uuid][1])
                 proj_file = projectfile_path(path; strict=true)
                 @assert proj_file !== nothing
                 proj = read_package(proj_file)
@@ -813,7 +825,7 @@ end
 
 function build(ctx::Context, uuids::Set{UUID}, verbose::Bool)
     if any_package_not_installed(ctx.env.manifest) || !isfile(ctx.env.manifest_file)
-        Pkg.instantiate(ctx)
+        Pkg.instantiate(ctx, allow_build = false, allow_autoprecomp = false)
     end
     all_uuids = get_deps(ctx.env, uuids)
     build_versions(ctx, all_uuids; verbose)
@@ -1398,11 +1410,11 @@ function sandbox_preserve(env::EnvCache, target::PackageSpec, test_project::Stri
     if env.pkg !== nothing
         env.manifest[env.pkg.uuid] = PackageEntry(;name=env.pkg.name, path=dirname(env.project_file),
                                                   deps=env.project.deps)
-        # if the source manifest is an old format, upgrade the manifest_format so
-        # that warnings aren't thrown for the temp sandbox manifest
-        if env.manifest.manifest_format < v"2.0"
-            env.manifest.manifest_format = v"2.0"
-        end
+    end
+    # if the source manifest is an old format, upgrade the manifest_format so
+    # that warnings aren't thrown for the temp sandbox manifest
+    if env.manifest.manifest_format < v"2.0"
+        env.manifest.manifest_format = v"2.0"
     end
     # preserve important nodes
     keep = [target.uuid]

src/Types.jl

@@ -22,7 +22,7 @@ using SHA
 
 export UUID, SHA1, VersionRange, VersionSpec,
     PackageSpec, PackageEntry, EnvCache, Context, GitRepo, Context!, Manifest, Project, err_rep,
-    PkgError, pkgerror, has_name, has_uuid, is_stdlib, is_unregistered_stdlib, stdlibs, write_env, write_env_usage, parse_toml, find_registered!,
+    PkgError, pkgerror, has_name, has_uuid, is_stdlib, stdlib_version, is_unregistered_stdlib, stdlibs, write_env, write_env_usage, parse_toml, find_registered!,
     project_resolve!, project_deps_resolve!, manifest_resolve!, registry_resolve!, stdlib_resolve!, handle_repos_develop!, handle_repos_add!, ensure_resolved,
     registered_name,
     manifest_info,
@@ -261,7 +261,7 @@ Base.:(==)(t1::PackageEntry, t2::PackageEntry) = t1.name == t2.name &&
 Base.hash(x::PackageEntry, h::UInt) = foldr(hash, [x.name, x.version, x.path, x.pinned, x.repo, x.tree_hash, x.deps, x.uuid], init=h)  # omits `other`
 
 Base.@kwdef mutable struct Manifest
-    julia_version::Union{Nothing,VersionNumber} = Base.VERSION
+    julia_version::Union{Nothing,VersionNumber} = nothing # only set to VERSION when resolving
     manifest_format::VersionNumber = v"2.0.0"
     deps::Dict{UUID,PackageEntry} = Dict{UUID,PackageEntry}()
     other::Dict{String,Any} = Dict{String,Any}()
@@ -411,8 +411,9 @@ is_stdlib(uuid::UUID) = uuid in keys(stdlibs())
 
 # Find the entry in `STDLIBS_BY_VERSION`
 # that corresponds to the requested version, and use that.
+# If we can't find one, defaults to `UNREGISTERED_STDLIBS`
 function get_last_stdlibs(julia_version::VersionNumber)
-    last_stdlibs = Dict{UUID,String}()
+    last_stdlibs = UNREGISTERED_STDLIBS
     for (version, stdlibs) in STDLIBS_BY_VERSION
         if VersionNumber(julia_version.major, julia_version.minor, julia_version.patch) < version
             break
@@ -421,6 +422,10 @@ function get_last_stdlibs(julia_version::VersionNumber)
     end
     return last_stdlibs
 end
+# If `julia_version` is set to `nothing`, that means (essentially) treat all registered
+# stdlibs as normal packages so that we get the latest versions of everything, ignoring
+# julia compat.  So we set the list of stdlibs to that of only the unregistered stdlibs.
+get_last_stdlibs(::Nothing) = UNREGISTERED_STDLIBS
 
 # Allow asking if something is an stdlib for a particular version of Julia
 function is_stdlib(uuid::UUID, julia_version::Union{VersionNumber, Nothing})
@@ -429,23 +434,24 @@ function is_stdlib(uuid::UUID, julia_version::Union{VersionNumber, Nothing})
         return is_stdlib(uuid)
     end
 
-    # If this UUID is known to be unregistered, always return `true`
-    if haskey(UNREGISTERED_STDLIBS, uuid)
-        return true
-    end
-
-    # Otherwise, if the `julia_version` is `nothing`, all registered stdlibs
-    # will be treated like normal packages.
-    if julia_version === nothing
-        return false
-    end
-
     last_stdlibs = get_last_stdlibs(julia_version)
     # Note that if the user asks for something like `julia_version = 0.7.0`, we'll
     # fall through with an empty `last_stdlibs`, which will always return `false`.
     return uuid in keys(last_stdlibs)
 end
 
+# Return the version of a stdlib with respect to a particular Julia version, or
+# `nothing` if that stdlib is not versioned.  We only store version numbers for
+# stdlibs that are external and thus could be installed from their repositories,
+# e.g. things like `GMP_jll`, `Tar`, etc...
+function stdlib_version(uuid::UUID, julia_version::Union{VersionNumber,Nothing})
+    last_stdlibs = get_last_stdlibs(julia_version)
+    if !(uuid in keys(last_stdlibs))
+        return nothing
+    end
+    return last_stdlibs[uuid][2]
+end
+
 is_unregistered_stdlib(uuid::UUID) = haskey(UNREGISTERED_STDLIBS, uuid)
 
 Context!(kw_context::Vector{Pair{Symbol,Any}})::Context =

src/manifest.jl

@@ -135,9 +135,9 @@ function Manifest(raw::Dict, f_or_io::Union{String, IO})::Manifest
     manifest_format = VersionNumber(raw["manifest_format"])
     if !in(manifest_format.major, 1:2)
         if f_or_io isa IO
-            @warn "Unknown Manifest.toml format version detected in streamed manifest. Unexpected behavior may occur" manifest_format maxlog = 1
+            @warn "Unknown Manifest.toml format version detected in streamed manifest. Unexpected behavior may occur" manifest_format
         else
-            @warn "Unknown Manifest.toml format version detected in file `$(f_or_io)`. Unexpected behavior may occur" manifest_format maxlog = 1
+            @warn "Unknown Manifest.toml format version detected in file `$(f_or_io)`. Unexpected behavior may occur" manifest_format maxlog = 1 _id = Symbol(f_or_io)
         end
     end
     stage1 = Dict{String,Vector{Stage1}}()
@@ -160,7 +160,7 @@ function Manifest(raw::Dict, f_or_io::Union{String, IO})::Manifest
                 deps = read_deps(get(info::Dict, "deps", nothing))
             catch
                 # TODO: Should probably not unconditionally log something
-                @error "Could not parse entry for `$name`"
+                @error "Could not parse entry for `$name`" f_or_io
                 rethrow()
             end
             entry.other = info::Union{Dict,Nothing}
@@ -285,7 +285,7 @@ end
 function write_manifest(manifest::Manifest, manifest_file::AbstractString)
     if manifest.manifest_format.major == 1
         @warn """The active manifest file at `$(manifest_file)` has an old format that is being maintained.
-            To update to the new format run `Pkg.upgrade_manifest()` which will upgrade the format without re-resolving.""" maxlog = 1
+            To update to the new format run `Pkg.upgrade_manifest()` which will upgrade the format without re-resolving.""" maxlog = 1 _id = Symbol(manifest_file)
     end
     return write_manifest(destructure(manifest), manifest_file)
 end
@@ -304,3 +304,27 @@ function write_manifest(raw_manifest::Dict, manifest_file::AbstractString)
     str = sprint(write_manifest, raw_manifest)
     write(manifest_file, str)
 end
+
+############
+# METADATA #
+############
+
+function check_warn_manifest_julia_version_compat(manifest::Manifest, manifest_file::String)
+    isempty(manifest.deps) && return
+    if manifest.manifest_format < v"2"
+        @warn """The active manifest file is an older format with no julia version entry. Dependencies may have \
+        been resolved with a different julia version.""" maxlog = 1 _file = manifest_file _line = 0 _module = nothing
+        return
+    end
+    v = manifest.julia_version
+    if v === nothing
+        @warn """The active manifest file is missing a julia version entry. Dependencies may have \
+        been resolved with a different julia version.""" maxlog = 1 _file = manifest_file _line = 0 _module = nothing
+        return
+    end
+    if v.major != VERSION.major && v.minor != VERSION.minor
+        ver_str = something(manifest.julia_version, "pre-1.7")
+        @warn """The active manifest file has dependencies that were resolved with a different julia \
+        version ($(manifest.julia_version)). Unexpected behavior may occur.""" maxlog = 1 _file = manifest_file _line = 0 _module = nothing
+    end
+end