respect versions in sysimage (#3002)

Author: KristofferC

CHANGELOG.md

@@ -1,3 +1,10 @@
+Pkg v1.9 Release Notes
+=======================
+
+- Pkg will now respect the version of packages put into the sysimage using e.g. PackageCompiler. For example,
+  if version 1.3.2 of package A is in the sysimage, Pkg will always install that version when adding the package,
+  or when the packge is installed as a dependency to some other package. This can be disabled by calling `Pkg.respect_sysimage_versions(false)`.
+
 Pkg v1.8 Release Notes
 ======================
 

docs/src/api.md

@@ -43,6 +43,7 @@ Pkg.status
 Pkg.compat
 Pkg.precompile
 Pkg.offline
+Pkg.respect_sysimage_versions
 Pkg.setprotocol!
 Pkg.dependencies
 Pkg.project

src/Operations.jl

@@ -14,7 +14,7 @@ import ..Artifacts: ensure_artifact_installed, artifact_names, extract_all_hashe
                     artifact_exists, select_downloadable_artifacts
 using Base.BinaryPlatforms
 import ...Pkg
-import ...Pkg: pkg_server, Registry, pathrepr, can_fancyprint, printpkgstyle, stderr_f, OFFLINE_MODE, UPDATED_REGISTRY_THIS_SESSION
+import ...Pkg: pkg_server, Registry, pathrepr, can_fancyprint, printpkgstyle, stderr_f, OFFLINE_MODE, UPDATED_REGISTRY_THIS_SESSION, RESPECT_SYSIMAGE_VERSIONS
 
 #########
 # Utils #
@@ -389,6 +389,7 @@ end
 get_or_make!(d::Dict{K,V}, k::K) where {K,V} = get!(d, k) do; V() end
 
 const JULIA_UUID = UUID("1222c4b2-2114-5bfd-aeef-88e4692bbb3e")
+const PKGORIGIN_HAVE_VERSION = :version in fieldnames(Base.PkgOrigin)
 function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance}, uuid_to_name::Dict{UUID,String},
                     reqs::Resolve.Requires, fixed::Dict{UUID,Resolve.Fixed}, julia_version)
     uuids = Set{UUID}()
@@ -456,6 +457,19 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance}
                             is_package_downloaded(env.project_file, pkg_spec) || continue
                         end
 
+                        # Skip package version that are not the same as external packages in sysimage
+                        if PKGORIGIN_HAVE_VERSION && RESPECT_SYSIMAGE_VERSIONS[] && julia_version == VERSION
+                            pkgid = Base.PkgId(uuid, pkg.name)
+                            if Base.in_sysimage(pkgid)
+                                pkgorigin = get(Base.pkgorigins, pkgid, nothing)
+                                if pkgorigin !== nothing && pkgorigin.version !== nothing
+                                    if v != pkgorigin.version
+                                        continue
+                                    end
+                                end
+                            end
+                        end
+
                         all_compat_u[v] = compat_info
                         union!(uuids, keys(compat_info))
                     end
@@ -1857,6 +1871,14 @@ function status_compat_info(pkg::PackageSpec, env::EnvCache, regs::Vector{Regist
     max_version == v"0" && return nothing
     pkg.version >= max_version && return nothing
 
+    pkgid = Base.PkgId(pkg.uuid, pkg.name)
+    if PKGORIGIN_HAVE_VERSION && RESPECT_SYSIMAGE_VERSIONS[] && Base.in_sysimage(pkgid)
+        pkgorigin = get(Base.pkgorigins, pkgid, nothing)
+        if pkgorigin !== nothing && pkg.version !== nothing && pkg.version == pkgorigin.version
+            return ["sysimage"], max_version, max_version_in_compat
+        end
+    end
+
     # Check compat of project
     if pkg.version == max_version_in_compat && max_version_in_compat != max_version
         return ["compat"], max_version, max_version_in_compat
@@ -2055,6 +2077,8 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie
             printstyled(io, " (<v", max_version, ")"; color=Base.warn_color())
             if packages_holding_back == ["compat"]
                 printstyled(io, " [compat]"; color=:light_magenta)
+            elseif packages_holding_back == ["sysimage"]
+                printstyled(io, " [sysimage]"; color=:light_magenta)
             else
                 pkg_str = isempty(packages_holding_back) ? "" : string(": ", join(packages_holding_back, ", "))
                 printstyled(io, pkg_str; color=Base.warn_color())

src/Pkg.jl

@@ -33,6 +33,7 @@ devdir(depot = depots1()) = get(ENV, "JULIA_PKG_DEVDIR", joinpath(depot, "dev"))
 envdir(depot = depots1()) = joinpath(depot, "environments")
 const UPDATED_REGISTRY_THIS_SESSION = Ref(false)
 const OFFLINE_MODE = Ref(false)
+const RESPECT_SYSIMAGE_VERSIONS = Ref(true)
 # For globally overriding in e.g. tests
 const DEFAULT_IO = Ref{Union{IO,Nothing}}(nothing)
 stderr_f() = something(DEFAULT_IO[], stderr)
@@ -513,6 +514,19 @@ set the environment variable `JULIA_PKG_OFFLINE` to `"true"`.
 """
 offline(b::Bool=true) = (OFFLINE_MODE[] = b; nothing)
 
+"""
+    Pkg.respect_sysimage_versions(b::Bool=true)
+
+Enable (`b=true`) or disable (`b=false`) respecting versions that are in the
+sysimage (enabled by default).
+
+If this option is enabled, Pkg will only install packages that have been put into the sysimage
+(e.g. via PackageCompiler) at the version of the package in the sysimage.
+Also, trying to add a package at a URL or `develop` a package that is in the sysimage
+will error.
+"""
+respect_sysimage_versions(b::Bool=true) = (RESPECT_SYSIMAGE_VERSIONS[] = b; nothing)
+
 """
     PackageSpec(name::String, [uuid::UUID, version::VersionNumber])
     PackageSpec(; name, url, path, subdir, rev, version, mode, level)

src/Resolve/graphtype.jl

@@ -624,7 +624,13 @@ function init_log!(data::GraphData)
         else
             vspec = range_compressed_versionspec(versions)
             vers = logstr(id, vspec)
+            uuid = data.pkgs[p0]
+            name = data.uuid_to_name[uuid]
+            pkgid = Base.PkgId(uuid, name)
             msg = "possible versions are: $vers or uninstalled"
+            if Base.in_sysimage(pkgid)
+                msg *= " (package in sysimage!)"
+            end
         end
         first_entry = get!(rlog.pool, p) do
             ResolveLogEntry(rlog.journal, p, "$(logstr(id)) log:")

src/Types.jl

@@ -12,7 +12,7 @@ using REPL.TerminalMenus
 
 using TOML
 import ..Pkg, ..Registry
-import ..Pkg: GitTools, depots, depots1, logdir, set_readonly, safe_realpath, pkg_server, stdlib_dir, stdlib_path, isurl, stderr_f
+import ..Pkg: GitTools, depots, depots1, logdir, set_readonly, safe_realpath, pkg_server, stdlib_dir, stdlib_path, isurl, stderr_f, RESPECT_SYSIMAGE_VERSIONS
 import Base.BinaryPlatforms: Platform
 using ..Pkg.Versions
 
@@ -526,6 +526,18 @@ function devpath(env::EnvCache, name::AbstractString, shared::Bool)
     return joinpath(dev_dir, name)
 end
 
+function error_if_in_sysimage(pkg::PackageSpec)
+    RESPECT_SYSIMAGE_VERSIONS[] || return false
+    if pkg.uuid === nothing
+        @error "Expected package to have a set UUID, please file a bug report"
+        return false
+    end
+    pkgid = Base.PkgId(pkg.uuid, pkg.name)
+    if Base.in_sysimage(pkgid)
+        pkgerror("tried to develop or add a package by URL that is already in the sysimage")
+    end
+end
+
 function handle_repo_develop!(ctx::Context, pkg::PackageSpec, shared::Bool)
     # First, check if we can compute the path easily (which requires a given local path or name)
     is_local_path = pkg.repo.source !== nothing && !isurl(pkg.repo.source)
@@ -544,6 +556,7 @@ function handle_repo_develop!(ctx::Context, pkg::PackageSpec, shared::Bool)
         end
         if isdir(dev_path)
             resolve_projectfile!(ctx.env, pkg, dev_path)
+            error_if_in_sysimage(pkg)
             if is_local_path
                 pkg.path = isabspath(dev_path) ? dev_path : relative_project_path(ctx.env.project_file, dev_path)
             else
@@ -605,6 +618,7 @@ function handle_repo_develop!(ctx::Context, pkg::PackageSpec, shared::Bool)
     if !has_uuid(pkg)
         resolve_projectfile!(ctx.env, pkg, dev_path)
     end
+    error_if_in_sysimage(pkg)
     pkg.path = shared ? dev_path : relative_project_path(ctx.env.project_file, dev_path)
     if pkg.repo.subdir !== nothing
         pkg.path = joinpath(pkg.path, pkg.repo.subdir)
@@ -735,13 +749,15 @@ function handle_repo_add!(ctx::Context, pkg::PackageSpec)
 
             # If we already resolved a uuid, we can bail early if this package is already installed at the current tree_hash
             if has_uuid(pkg)
+                error_if_in_sysimage(pkg)
                 version_path = Pkg.Operations.source_path(ctx.env.project_file, pkg, ctx.julia_version)
                 isdir(version_path) && return false
             end
 
             temp_path = mktempdir()
             GitTools.checkout_tree_to_path(repo, tree_hash_object, temp_path)
-            package = resolve_projectfile!(ctx.env, pkg, temp_path)
+            resolve_projectfile!(ctx.env, pkg, temp_path)
+            error_if_in_sysimage(pkg)
 
             # Now that we are fully resolved (name, UUID, tree_hash, repo.source, repo.rev), we can finally
             # check to see if the package exists at its canonical path.

test/new.jl

@@ -561,7 +561,7 @@ end
         Pkg.add("Markdown")
         Pkg.add("Markdown")
         Pkg.dependencies(markdown_uuid) do pkg
-            pkg.name == "Markdown"
+            @test pkg.name == "Markdown"
         end
         @test haskey(Pkg.project().dependencies, "Markdown")
     end
@@ -2893,4 +2893,41 @@ end
     end
 end
 
+if :version in fieldnames(Base.PkgOrigin)
+@testset "sysimage functionality" begin
+    old_sysimage_modules = copy(Base._sysimage_modules)
+    old_pkgorigins = copy(Base.pkgorigins)
+    try
+        # Fake having a packages in the sysimage.
+        json_pkgid = Base.PkgId(json_uuid, "JSON")
+        push!(Base._sysimage_modules, json_pkgid)
+        Base.pkgorigins[json_pkgid] = Base.PkgOrigin(nothing, nothing, v"0.20.1")
+        isolate(loaded_depot=true) do
+            Pkg.add("JSON"; io=devnull)
+            Pkg.dependencies(json_uuid) do pkg
+                pkg.version == v"0.20.1"
+            end
+            io = IOBuffer()
+            Pkg.status(; outdated=true, io=io)
+            str = String(take!(io))
+            @test occursin("⌅ [682c06a0] JSON v0.20.1", str)
+            @test occursin("[sysimage]", str)
+
+            @test_throws PkgError Pkg.add(name="JSON", rev="master"; io=devnull)
+            @test_throws PkgError Pkg.develop("JSON"; io=devnull)
+
+            Pkg.respect_sysimage_versions(false)
+            Pkg.add("JSON"; io=devnull)
+            Pkg.dependencies(json_uuid) do pkg
+                pkg.version != v"0.20.1"
+            end
+        end
+    finally
+        copy!(Base._sysimage_modules, old_sysimage_modules)
+        copy!(Base.pkgorigins, old_pkgorigins)
+        Pkg.respect_sysimage_versions(true)
+    end
+end
+end
+
 end #module