prefer adding already loaded dep versions

Author: IanButterworth

CHANGELOG.md

@@ -3,6 +3,9 @@ Pkg v1.13 Release Notes
 
 - `Pkg.test` now respects the `--check-bounds` setting from the parent Julia session instead of forcing `--check-bounds=yes`.
 
+- `Pkg.add` now prefers versions of packages that are already loaded in the current Julia session when resolving
+  dependencies. This helps maintain compatibility with code already running in your session. The behavior can be
+  disabled using `Pkg.add(pkg; prefer_loaded_versions=false)`. ([#4507])
 - Project.toml environments now support a `readonly` field to mark environments as read-only, preventing modifications.
   ([#4284])
 - `Pkg.build` now supports an `allow_reresolve` keyword argument to control whether the build process can re-resolve

src/API.jl

@@ -320,7 +320,8 @@ end
 
 function add(
         ctx::Context, pkgs::Vector{PackageSpec}; preserve::PreserveLevel = Operations.default_preserve(),
-        platform::AbstractPlatform = HostPlatform(), target::Symbol = :deps, allow_autoprecomp::Bool = true, kwargs...
+        platform::AbstractPlatform = HostPlatform(), target::Symbol = :deps, allow_autoprecomp::Bool = true,
+        prefer_loaded_versions::Bool = true, kwargs...
     )
     require_not_empty(pkgs, :add)
     Context!(ctx; kwargs...)
@@ -376,7 +377,7 @@ function add(
         update_source_if_set(ctx.env, pkg)
     end
 
-    Operations.add(ctx, pkgs, new_git; allow_autoprecomp, preserve, platform, target)
+    Operations.add(ctx, pkgs, new_git; allow_autoprecomp, preserve, platform, target, prefer_loaded_versions)
     return
 end
 

src/Operations.jl

@@ -708,7 +708,7 @@ end
 # all versioned packages should have a `tree_hash`
 function resolve_versions!(
         env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, julia_version,
-        installed_only::Bool
+        installed_only::Bool, preferred_versions::Dict{UUID, VersionNumber} = Dict{UUID, VersionNumber}()
     )
     installed_only = installed_only || OFFLINE_MODE[]
 
@@ -777,7 +777,10 @@ function resolve_versions!(
     unbind_stdlibs = julia_version === VERSION
     reqs = Resolve.Requires(pkg.uuid => is_stdlib(pkg.uuid) && unbind_stdlibs ? VersionSpec("*") : VersionSpec(pkg.version) for pkg in pkgs)
     deps_map_compressed, compat_map_compressed, weak_deps_map_compressed, weak_compat_map_compressed, pkg_versions_map, pkg_versions_per_registry, uuid_to_name, reqs, fixed = deps_graph(env, registries, names, reqs, fixed, julia_version, installed_only)
-    graph = Resolve.Graph(deps_map_compressed, compat_map_compressed, weak_deps_map_compressed, weak_compat_map_compressed, pkg_versions_map, pkg_versions_per_registry, uuid_to_name, reqs, fixed, false, julia_version)
+    graph = Resolve.Graph(
+        deps_map_compressed, compat_map_compressed, weak_deps_map_compressed, weak_compat_map_compressed,
+        pkg_versions_map, pkg_versions_per_registry, uuid_to_name, reqs, fixed, false, julia_version, preferred_versions
+    )
     Resolve.simplify_graph!(graph)
     vers = Resolve.resolve(graph)
 
@@ -844,6 +847,84 @@ function resolve_versions!(
     return final_deps_map
 end
 
+function collect_preferred_loaded_versions(env::EnvCache)
+    preferred = Dict{UUID, VersionNumber}()
+    for (pkgid, mod) in Base.loaded_modules
+        pkgid isa Base.PkgId || continue
+        pkg_uuid = pkgid.uuid
+        pkg_uuid isa UUID || continue
+        Types.is_stdlib(pkg_uuid) && continue
+        haskey(env.manifest, pkg_uuid) && continue
+        env.pkg !== nothing && pkg_uuid == env.pkg.uuid && continue
+        version = Base.pkgversion(mod)
+        version isa VersionNumber || continue
+        preferred[pkg_uuid] = version
+    end
+    return preferred
+end
+
+function preferred_loaded_packages_usage(
+        pkgs::Vector{PackageSpec}, preferred_versions::Dict{UUID, VersionNumber}, manifest_uuids::Set{UUID},
+        direct_requested_uuids::Set{UUID}
+    )
+    (isempty(preferred_versions) || isempty(pkgs)) && return String[], 0
+    direct_names = String[]
+    indirect_count = 0
+    for pkg in pkgs
+        uuid = pkg.uuid
+        uuid isa UUID || continue
+        uuid in manifest_uuids && continue
+        preferred_version = get(preferred_versions, uuid, nothing)
+        preferred_version === nothing && continue
+        pkg_version = pkg.version
+        pkg_version isa VersionNumber || continue
+        pkg_version == preferred_version || continue
+        pkg.name === nothing && continue
+        if uuid in direct_requested_uuids
+            push!(direct_names, pkg.name)
+        else
+            indirect_count += 1
+        end
+    end
+    sort!(direct_names)
+    unique!(direct_names)
+    return direct_names, indirect_count
+end
+
+function maybe_print_preferred_loaded_note(io::IO, direct_names::Vector{String}, indirect_count::Int)
+    isempty(direct_names) && indirect_count == 0 && return
+    parts = String[]
+    if !isempty(direct_names)
+        push!(parts, join(direct_names, ", "))
+    end
+    if indirect_count > 0
+        dep_word = indirect_count == 1 ? "dependency" : "dependencies"
+        push!(parts, "$(indirect_count) $(dep_word)")
+    end
+    joined = length(parts) == 2 ? string(parts[1], " and ", parts[2]) : parts[1]
+    msg = if length(direct_names) + indirect_count > 1
+        "Preferring versions of $(joined) that are already loaded"
+    else
+        "Preferring the version of $(joined) that is already loaded"
+    end
+    printpkgstyle(io, :Resolve, msg; color = Base.info_color())
+    return
+end
+
+function apply_preferred_versions_to_direct!(pkgs::Vector{PackageSpec}, preferred_versions::Dict{UUID, VersionNumber})
+    isempty(preferred_versions) && return
+    empty_spec = VersionSpec()
+    for pkg in pkgs
+        pkg.version == empty_spec || continue
+        uuid = pkg.uuid
+        uuid isa UUID || continue
+        pref_version = get(preferred_versions, uuid, nothing)
+        pref_version === nothing && continue
+        pkg.version = VersionSpec(pref_version)
+    end
+    return
+end
+
 get_or_make!(d::Dict{K, V}, k::K) where {K, V} = get!(d, k) do;
     V()
 end
@@ -2072,63 +2153,67 @@ end
 
 function tiered_resolve(
         env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, julia_version,
-        try_all_installed::Bool
+        try_all_installed::Bool; preferred_versions::Dict{UUID, VersionNumber} = Dict{UUID, VersionNumber}()
     )
     if try_all_installed
         try # do not modify existing subgraph and only add installed versions of the new packages
             @debug "tiered_resolve: trying PRESERVE_ALL_INSTALLED"
-            return targeted_resolve(env, registries, pkgs, PRESERVE_ALL_INSTALLED, julia_version)
+            return targeted_resolve(env, registries, pkgs, PRESERVE_ALL_INSTALLED, julia_version; preferred_versions)
         catch err
             err isa Resolve.ResolverError || rethrow()
         end
     end
     try # do not modify existing subgraph
         @debug "tiered_resolve: trying PRESERVE_ALL"
-        return targeted_resolve(env, registries, pkgs, PRESERVE_ALL, julia_version)
+        return targeted_resolve(env, registries, pkgs, PRESERVE_ALL, julia_version; preferred_versions)
     catch err
         err isa Resolve.ResolverError || rethrow()
     end
     try # do not modify existing direct deps
         @debug "tiered_resolve: trying PRESERVE_DIRECT"
-        return targeted_resolve(env, registries, pkgs, PRESERVE_DIRECT, julia_version)
+        return targeted_resolve(env, registries, pkgs, PRESERVE_DIRECT, julia_version; preferred_versions)
     catch err
         err isa Resolve.ResolverError || rethrow()
     end
     try
         @debug "tiered_resolve: trying PRESERVE_SEMVER"
-        return targeted_resolve(env, registries, pkgs, PRESERVE_SEMVER, julia_version)
+        return targeted_resolve(env, registries, pkgs, PRESERVE_SEMVER, julia_version; preferred_versions)
     catch err
         err isa Resolve.ResolverError || rethrow()
     end
     @debug "tiered_resolve: trying PRESERVE_NONE"
-    return targeted_resolve(env, registries, pkgs, PRESERVE_NONE, julia_version)
+    return targeted_resolve(env, registries, pkgs, PRESERVE_NONE, julia_version; preferred_versions)
 end
 
-function targeted_resolve(env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, preserve::PreserveLevel, julia_version)
+function targeted_resolve(
+        env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, preserve::PreserveLevel,
+        julia_version; preferred_versions::Dict{UUID, VersionNumber} = Dict{UUID, VersionNumber}()
+    )
     if preserve == PRESERVE_ALL || preserve == PRESERVE_ALL_INSTALLED
         pkgs = load_all_deps(env, pkgs; preserve)
     else
         pkgs = load_direct_deps(env, pkgs; preserve)
     end
     check_registered(registries, pkgs)
 
-    deps_map = resolve_versions!(env, registries, pkgs, julia_version, preserve == PRESERVE_ALL_INSTALLED)
+    deps_map = resolve_versions!(env, registries, pkgs, julia_version, preserve == PRESERVE_ALL_INSTALLED, preferred_versions)
     return pkgs, deps_map
 end
 
 function _resolve(
         io::IO, env::EnvCache, registries::Vector{Registry.RegistryInstance},
-        pkgs::Vector{PackageSpec}, preserve::PreserveLevel, julia_version
+        pkgs::Vector{PackageSpec}, preserve::PreserveLevel, julia_version;
+        preferred_versions::Dict{UUID, VersionNumber} = Dict{UUID, VersionNumber}()
     )
     usingstrategy = preserve != PRESERVE_TIERED ? " using $preserve" : ""
     printpkgstyle(io, :Resolving, "package versions$(usingstrategy)...")
     return try
         if preserve == PRESERVE_TIERED_INSTALLED
-            tiered_resolve(env, registries, pkgs, julia_version, true)
+            tiered_resolve(env, registries, pkgs, julia_version, true; preferred_versions)
         elseif preserve == PRESERVE_TIERED
-            tiered_resolve(env, registries, pkgs, julia_version, false)
+            tiered_resolve(env, registries, pkgs, julia_version, false; preferred_versions)
         else
-            targeted_resolve(env, registries, pkgs, preserve, julia_version)
+            targeted_resolve(env, registries, pkgs, preserve, julia_version; preferred_versions)
         end
     catch err
 
@@ -2191,7 +2276,7 @@ end
 function add(
         ctx::Context, pkgs::Vector{PackageSpec}, new_git = Set{UUID}();
         allow_autoprecomp::Bool = true, preserve::PreserveLevel = default_preserve(), platform::AbstractPlatform = HostPlatform(),
-        target::Symbol = :deps
+        target::Symbol = :deps, prefer_loaded_versions::Bool = true
     )
     assert_can_add(ctx, pkgs)
     # load manifest data
@@ -2235,11 +2320,34 @@ function add(
         return
     end
 
+    preferred_loaded_versions = Dict{UUID, VersionNumber}()
+    existing_manifest_uuids = Set{UUID}()
+    preferred_direct_note_names = String[]
+    preferred_indirect_note_count = 0
+    direct_requested_uuids = Set{UUID}()
     foreach(pkg -> target_field[pkg.name] = pkg.uuid, pkgs) # update set of deps/weakdeps/extras
+    for pkg in pkgs
+        uuid = pkg.uuid
+        uuid isa UUID || continue
+        push!(direct_requested_uuids, uuid)
+    end
+
+    if target == :deps && prefer_loaded_versions
+        preferred_loaded_versions = collect_preferred_loaded_versions(ctx.env)
+        existing_manifest_uuids = Set(keys(ctx.env.manifest))
+    end
 
     if target == :deps # nothing to resolve/install if it's weak or extras
         # resolve
-        man_pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, preserve, ctx.julia_version)
+        apply_preferred_versions_to_direct!(pkgs, preferred_loaded_versions)
+        man_pkgs, deps_map = _resolve(
+            ctx.io, ctx.env, ctx.registries, pkgs, preserve, ctx.julia_version;
+            preferred_versions = preferred_loaded_versions
+        )
+        preferred_direct_note_names, preferred_indirect_note_count = preferred_loaded_packages_usage(
+            man_pkgs, preferred_loaded_versions, existing_manifest_uuids, direct_requested_uuids
+        )
+        maybe_print_preferred_loaded_note(ctx.io, preferred_direct_note_names, preferred_indirect_note_count)
         update_manifest!(ctx.env, man_pkgs, deps_map, ctx.julia_version, ctx.registries)
         new_apply = download_source(ctx)
         fixups_from_projectfile!(ctx)

src/Pkg.jl

@@ -162,8 +162,8 @@ const PreserveLevel = Types.PreserveLevel
 
 # Define new variables so tab comleting Pkg. works.
 """
-    Pkg.add(pkg::Union{String, Vector{String}}; preserve=PRESERVE_TIERED, target::Symbol=:deps)
-    Pkg.add(pkg::Union{PackageSpec, Vector{PackageSpec}}; preserve=PRESERVE_TIERED, target::Symbol=:deps)
+    Pkg.add(pkg::Union{String, Vector{String}}; preserve=PRESERVE_TIERED, target::Symbol=:deps, prefer_loaded_versions::Bool=true)
+    Pkg.add(pkg::Union{PackageSpec, Vector{PackageSpec}}; preserve=PRESERVE_TIERED, target::Symbol=:deps, prefer_loaded_versions::Bool=true)
 
 Add a package to the current project. This package will be available by using the
 `import` and `using` keywords in the Julia REPL, and if the current project is
@@ -175,6 +175,16 @@ added automatically with a lower bound of the added version.
 To add as a weak dependency (in the `[weakdeps]` field) set the kwarg `target=:weakdeps`.
 To add as an extra dep (in the `[extras]` field) set `target=:extras`.
 
+## Loaded Version Preference
+
+By default, when adding packages, Pkg will prefer versions of packages (and their dependencies) that are
+already loaded in the current Julia session. This helps maintain compatibility with code already running
+in your session. To disable this behavior and resolve versions independently of what's currently loaded,
+set `prefer_loaded_versions=false`.
+
+!!! compat "Julia 1.13"
+    The `prefer_loaded_versions` kwarg requires at least Julia 1.13.
+
 ## Resolution Tiers
 `Pkg` resolves the set of packages in your environment using a tiered algorithm.
 The `preserve` keyword argument allows you to key into a specific tier in the resolve algorithm.
@@ -213,6 +223,7 @@ precompiled before, or the precompile cache has been deleted by the LRU cache st
 Pkg.add("Example") # Add a package from registry
 Pkg.add("Example", target=:weakdeps) # Add a package as a weak dependency
 Pkg.add("Example", target=:extras) # Add a package to the `[extras]` list
+Pkg.add("Example"; prefer_loaded_versions=false) # Add a package, ignoring versions already loaded in this session
 Pkg.add("Example"; preserve=Pkg.PRESERVE_ALL) # Add the `Example` package and strictly preserve existing dependencies
 Pkg.add(name="Example", version="0.3") # Specify version; latest release in the 0.3 series
 Pkg.add(name="Example", version="0.3.1") # Specify version; exact release

src/Resolve/graphtype.jl

@@ -234,6 +234,7 @@ mutable struct Graph
     newmsg::Vector{FieldValue}
     diff::Vector{FieldValue}
     cavfld::Vector{FieldValue}
+    preferred_versions::Dict{UUID, VersionNumber}
 
     function Graph(
             deps_compressed::Dict{UUID, Vector{Dict{VersionRange, Set{UUID}}}},
@@ -246,7 +247,8 @@ mutable struct Graph
             reqs::Requires,
             fixed::Dict{UUID, Fixed},
             verbose::Bool = false,
-            julia_version::Union{VersionNumber, Nothing} = VERSION
+            julia_version::Union{VersionNumber, Nothing} = VERSION,
+            preferred_versions::Dict{UUID, VersionNumber} = Dict{UUID, VersionNumber}()
         )
 
         # Tell the resolver about julia itself
@@ -391,7 +393,7 @@ mutable struct Graph
 
         graph = new(
             data, gadj, gmsk, gconstr, adjdict, req_inds, fix_inds, ignored, solve_stack, spp, np,
-            FieldValue[], FieldValue[], FieldValue[]
+            FieldValue[], FieldValue[], FieldValue[], Dict{UUID, VersionNumber}(preferred_versions)
         )
 
         _add_fixed!(graph, fixed)
@@ -416,7 +418,8 @@ mutable struct Graph
         ignored = copy(graph.ignored)
         solve_stack = [([copy(gc0) for gc0 in sav_gconstr], copy(sav_ignored)) for (sav_gconstr, sav_ignored) in graph.solve_stack]
 
-        return new(data, gadj, gmsk, gconstr, adjdict, req_inds, fix_inds, ignored, solve_stack, spp, np)
+        preferred_versions = copy(graph.preferred_versions)
+        return new(data, gadj, gmsk, gconstr, adjdict, req_inds, fix_inds, ignored, solve_stack, spp, np, FieldValue[], FieldValue[], FieldValue[], preferred_versions)
     end
 end
 

src/Resolve/maxsum.jl

@@ -1,6 +1,7 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
 const DEFAULT_MAX_TIME = "300"
+const PREFERRED_VERSION_WEIGHT_BONUS = VersionWeight(typemax(Int32))
 
 # Some parameters to drive the decimation process
 mutable struct MaxSumParams
@@ -55,6 +56,19 @@ mutable struct Messages
 
         ## generate wveights (v0 == spp[p0] is the "uninstalled" state)
         vweight = [[VersionWeight(v0 < spp[p0] ? pvers[p0][v0] : v"0") for v0 in 1:spp[p0]] for p0 in 1:np]
+        preferred_versions = graph.preferred_versions
+        if !isempty(preferred_versions)
+            pkgs = graph.data.pkgs
+            vdict = graph.data.vdict
+            for p0 in 1:np
+                uuid = pkgs[p0]
+                pref_version = get(preferred_versions, uuid, nothing)
+                pref_version === nothing && continue
+                idx = get(vdict[p0], pref_version, 0)
+                (idx > 0 && idx < spp[p0]) || continue
+                vweight[p0][idx] += PREFERRED_VERSION_WEIGHT_BONUS
+            end
+        end
 
         # external fields: favor newest versions over older, and no-version over all;
         #                  explicit requirements use level l1 instead of l2

test/new.jl

@@ -3922,4 +3922,28 @@ end
     end
 end
 
+@testset "Pkg.add prefers loaded dependency versions" begin
+    isolate(loaded_depot = true) do
+        script = """
+        using Pkg, Test
+        Pkg.activate(; temp = true)
+        io = IOBuffer()
+        Pkg.add(name = "Example", version = v"0.5.4", io = io)
+        add_output = String(take!(io))
+        @test occursin("[7876af07] + Example v0.5.4", add_output)
+        using Example
+        Pkg.activate(; temp = true)
+        Pkg.add("Example", io = io) # v0.5.5 exists, but v0.5.4 is loaded
+        add_output = String(take!(io))
+        @test occursin("Preferring the version of Example that is already loaded", add_output)
+        @test occursin("[7876af07] + Example v0.5.4", add_output)
+        """
+        cmd = addenv(
+            `$(Base.julia_cmd()) --startup-file=no --project=$(dirname(@__DIR__)) -e $script`,
+            "JULIA_DEPOT_PATH" => join(DEPOT_PATH, Sys.iswindows() ? ";" : ":")
+        )
+        @test Utils.show_output_if_command_errors(cmd)
+    end
+end
+
 end #module