diff --git a/CHANGELOG.md b/CHANGELOG.md index 986836d5cc..b79765fb3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ Pkg v1.13 Release Notes - `Pkg.test` now respects the `--check-bounds` setting from the parent Julia session instead of forcing `--check-bounds=yes`. +- `Pkg.add` now prefers versions of packages that are already loaded in the current Julia session when resolving + dependencies. This helps maintain compatibility with code already running in your session. The behavior can be + disabled using `Pkg.add(pkg; prefer_loaded_versions=false)`. ([#4507]) - Project.toml environments now support a `readonly` field to mark environments as read-only, preventing modifications. ([#4284]) - `Pkg.build` now supports an `allow_reresolve` keyword argument to control whether the build process can re-resolve diff --git a/src/API.jl b/src/API.jl index af10414dc1..6b016f993e 100644 --- a/src/API.jl +++ b/src/API.jl @@ -320,7 +320,8 @@ end function add( ctx::Context, pkgs::Vector{PackageSpec}; preserve::PreserveLevel = Operations.default_preserve(), - platform::AbstractPlatform = HostPlatform(), target::Symbol = :deps, allow_autoprecomp::Bool = true, kwargs... + platform::AbstractPlatform = HostPlatform(), target::Symbol = :deps, allow_autoprecomp::Bool = true, + prefer_loaded_versions::Bool = true, kwargs... ) require_not_empty(pkgs, :add) Context!(ctx; kwargs...) @@ -376,7 +377,7 @@ function add( update_source_if_set(ctx.env, pkg) end - Operations.add(ctx, pkgs, new_git; allow_autoprecomp, preserve, platform, target) + Operations.add(ctx, pkgs, new_git; allow_autoprecomp, preserve, platform, target, prefer_loaded_versions) return end diff --git a/src/Operations.jl b/src/Operations.jl index 2c452e222f..0cadb09db3 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -708,7 +708,7 @@ end # all versioned packages should have a `tree_hash` function resolve_versions!( env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, julia_version, - installed_only::Bool + installed_only::Bool, preferred_versions::Dict{UUID, VersionNumber} = Dict{UUID, VersionNumber}() ) installed_only = installed_only || OFFLINE_MODE[] @@ -777,7 +777,10 @@ function resolve_versions!( unbind_stdlibs = julia_version === VERSION reqs = Resolve.Requires(pkg.uuid => is_stdlib(pkg.uuid) && unbind_stdlibs ? VersionSpec("*") : VersionSpec(pkg.version) for pkg in pkgs) deps_map_compressed, compat_map_compressed, weak_deps_map_compressed, weak_compat_map_compressed, pkg_versions_map, pkg_versions_per_registry, uuid_to_name, reqs, fixed = deps_graph(env, registries, names, reqs, fixed, julia_version, installed_only) - graph = Resolve.Graph(deps_map_compressed, compat_map_compressed, weak_deps_map_compressed, weak_compat_map_compressed, pkg_versions_map, pkg_versions_per_registry, uuid_to_name, reqs, fixed, false, julia_version) + graph = Resolve.Graph( + deps_map_compressed, compat_map_compressed, weak_deps_map_compressed, weak_compat_map_compressed, + pkg_versions_map, pkg_versions_per_registry, uuid_to_name, reqs, fixed, false, julia_version, preferred_versions + ) Resolve.simplify_graph!(graph) vers = Resolve.resolve(graph) @@ -844,6 +847,84 @@ function resolve_versions!( return final_deps_map end +function collect_preferred_loaded_versions(env::EnvCache) + preferred = Dict{UUID, VersionNumber}() + for (pkgid, mod) in Base.loaded_modules + pkgid isa Base.PkgId || continue + pkg_uuid = pkgid.uuid + pkg_uuid isa UUID || continue + Types.is_stdlib(pkg_uuid) && continue + haskey(env.manifest, pkg_uuid) && continue + env.pkg !== nothing && pkg_uuid == env.pkg.uuid && continue + version = Base.pkgversion(mod) + version isa VersionNumber || continue + preferred[pkg_uuid] = version + end + return preferred +end + +function preferred_loaded_packages_usage( + pkgs::Vector{PackageSpec}, preferred_versions::Dict{UUID, VersionNumber}, manifest_uuids::Set{UUID}, + direct_requested_uuids::Set{UUID} + ) + (isempty(preferred_versions) || isempty(pkgs)) && return String[], 0 + direct_names = String[] + indirect_count = 0 + for pkg in pkgs + uuid = pkg.uuid + uuid isa UUID || continue + uuid in manifest_uuids && continue + preferred_version = get(preferred_versions, uuid, nothing) + preferred_version === nothing && continue + pkg_version = pkg.version + pkg_version isa VersionNumber || continue + pkg_version == preferred_version || continue + pkg.name === nothing && continue + if uuid in direct_requested_uuids + push!(direct_names, pkg.name) + else + indirect_count += 1 + end + end + sort!(direct_names) + unique!(direct_names) + return direct_names, indirect_count +end + +function maybe_print_preferred_loaded_note(io::IO, direct_names::Vector{String}, indirect_count::Int) + isempty(direct_names) && indirect_count == 0 && return + parts = String[] + if !isempty(direct_names) + push!(parts, join(direct_names, ", ")) + end + if indirect_count > 0 + dep_word = indirect_count == 1 ? "dependency" : "dependencies" + push!(parts, "$(indirect_count) $(dep_word)") + end + joined = length(parts) == 2 ? string(parts[1], " and ", parts[2]) : parts[1] + msg = if length(direct_names) + indirect_count > 1 + "was able to add the versions of $(joined) that are already loaded" + else + "was able to add the version of $(joined) that is already loaded" + end + printpkgstyle(io, :Resolve, msg; color = Base.info_color()) + return +end + +function apply_preferred_versions_to_direct!(pkgs::Vector{PackageSpec}, preferred_versions::Dict{UUID, VersionNumber}) + isempty(preferred_versions) && return + empty_spec = VersionSpec() + for pkg in pkgs + pkg.version == empty_spec || continue + uuid = pkg.uuid + uuid isa UUID || continue + pref_version = get(preferred_versions, uuid, nothing) + pref_version === nothing && continue + pkg.version = VersionSpec(pref_version) + end + return +end + get_or_make!(d::Dict{K, V}, k::K) where {K, V} = get!(d, k) do; V() end @@ -2072,8 +2153,17 @@ end function tiered_resolve( env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, julia_version, - try_all_installed::Bool + try_all_installed::Bool; preferred_versions::Dict{UUID, VersionNumber} = Dict{UUID, VersionNumber}() ) + if !isempty(preferred_versions) + # first try maintaining any loaded versions of the new packages + try # do not modify existing subgraph + @debug "tiered_resolve: trying PRESERVE_ALL with any loaded versions of new packages" + return targeted_resolve(env, registries, pkgs, PRESERVE_ALL, julia_version; preferred_versions) + catch err + err isa Resolve.ResolverError || rethrow() + end + end if try_all_installed try # do not modify existing subgraph and only add installed versions of the new packages @debug "tiered_resolve: trying PRESERVE_ALL_INSTALLED" @@ -2104,7 +2194,10 @@ function tiered_resolve( return targeted_resolve(env, registries, pkgs, PRESERVE_NONE, julia_version) end -function targeted_resolve(env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, preserve::PreserveLevel, julia_version) +function targeted_resolve( + env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, preserve::PreserveLevel, + julia_version; preferred_versions::Dict{UUID, VersionNumber} = Dict{UUID, VersionNumber}() + ) if preserve == PRESERVE_ALL || preserve == PRESERVE_ALL_INSTALLED pkgs = load_all_deps(env, pkgs; preserve) else @@ -2112,23 +2205,24 @@ function targeted_resolve(env::EnvCache, registries::Vector{Registry.RegistryIns end check_registered(registries, pkgs) - deps_map = resolve_versions!(env, registries, pkgs, julia_version, preserve == PRESERVE_ALL_INSTALLED) + deps_map = resolve_versions!(env, registries, pkgs, julia_version, preserve == PRESERVE_ALL_INSTALLED, preferred_versions) return pkgs, deps_map end function _resolve( io::IO, env::EnvCache, registries::Vector{Registry.RegistryInstance}, - pkgs::Vector{PackageSpec}, preserve::PreserveLevel, julia_version + pkgs::Vector{PackageSpec}, preserve::PreserveLevel, julia_version; + preferred_versions::Dict{UUID, VersionNumber} = Dict{UUID, VersionNumber}() ) usingstrategy = preserve != PRESERVE_TIERED ? " using $preserve" : "" printpkgstyle(io, :Resolving, "package versions$(usingstrategy)...") return try if preserve == PRESERVE_TIERED_INSTALLED - tiered_resolve(env, registries, pkgs, julia_version, true) + tiered_resolve(env, registries, pkgs, julia_version, true; preferred_versions) elseif preserve == PRESERVE_TIERED - tiered_resolve(env, registries, pkgs, julia_version, false) + tiered_resolve(env, registries, pkgs, julia_version, false; preferred_versions) else - targeted_resolve(env, registries, pkgs, preserve, julia_version) + targeted_resolve(env, registries, pkgs, preserve, julia_version; preferred_versions) end catch err @@ -2191,7 +2285,7 @@ end function add( ctx::Context, pkgs::Vector{PackageSpec}, new_git = Set{UUID}(); allow_autoprecomp::Bool = true, preserve::PreserveLevel = default_preserve(), platform::AbstractPlatform = HostPlatform(), - target::Symbol = :deps + target::Symbol = :deps, prefer_loaded_versions::Bool = true ) assert_can_add(ctx, pkgs) # load manifest data @@ -2235,11 +2329,34 @@ function add( return end + preferred_loaded_versions = Dict{UUID, VersionNumber}() + existing_manifest_uuids = Set{UUID}() + preferred_direct_note_names = String[] + preferred_indirect_note_count = 0 + direct_requested_uuids = Set{UUID}() foreach(pkg -> target_field[pkg.name] = pkg.uuid, pkgs) # update set of deps/weakdeps/extras + for pkg in pkgs + uuid = pkg.uuid + uuid isa UUID || continue + push!(direct_requested_uuids, uuid) + end + + if target == :deps && prefer_loaded_versions + preferred_loaded_versions = collect_preferred_loaded_versions(ctx.env) + existing_manifest_uuids = Set(keys(ctx.env.manifest)) + end if target == :deps # nothing to resolve/install if it's weak or extras # resolve - man_pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, preserve, ctx.julia_version) + apply_preferred_versions_to_direct!(pkgs, preferred_loaded_versions) + man_pkgs, deps_map = _resolve( + ctx.io, ctx.env, ctx.registries, pkgs, preserve, ctx.julia_version; + preferred_versions = preferred_loaded_versions + ) + preferred_direct_note_names, preferred_indirect_note_count = preferred_loaded_packages_usage( + man_pkgs, preferred_loaded_versions, existing_manifest_uuids, direct_requested_uuids + ) + maybe_print_preferred_loaded_note(ctx.io, preferred_direct_note_names, preferred_indirect_note_count) update_manifest!(ctx.env, man_pkgs, deps_map, ctx.julia_version, ctx.registries) new_apply = download_source(ctx) fixups_from_projectfile!(ctx) diff --git a/src/Pkg.jl b/src/Pkg.jl index b73c8f10ba..50bde1a20c 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -162,8 +162,8 @@ const PreserveLevel = Types.PreserveLevel # Define new variables so tab comleting Pkg. works. """ - Pkg.add(pkg::Union{String, Vector{String}}; preserve=PRESERVE_TIERED, target::Symbol=:deps) - Pkg.add(pkg::Union{PackageSpec, Vector{PackageSpec}}; preserve=PRESERVE_TIERED, target::Symbol=:deps) + Pkg.add(pkg::Union{String, Vector{String}}; preserve=PRESERVE_TIERED, target::Symbol=:deps, prefer_loaded_versions::Bool=true) + Pkg.add(pkg::Union{PackageSpec, Vector{PackageSpec}}; preserve=PRESERVE_TIERED, target::Symbol=:deps, prefer_loaded_versions::Bool=true) Add a package to the current project. This package will be available by using the `import` and `using` keywords in the Julia REPL, and if the current project is @@ -175,6 +175,16 @@ added automatically with a lower bound of the added version. To add as a weak dependency (in the `[weakdeps]` field) set the kwarg `target=:weakdeps`. To add as an extra dep (in the `[extras]` field) set `target=:extras`. +## Loaded Version Preference + +By default, when adding packages, Pkg will prefer versions of packages (and their dependencies) that are +already loaded in the current Julia session. This helps maintain compatibility with code already running +in your session. To disable this behavior and resolve versions independently of what's currently loaded, +set `prefer_loaded_versions=false`. + +!!! compat "Julia 1.13" + The `prefer_loaded_versions` kwarg requires at least Julia 1.13. + ## Resolution Tiers `Pkg` resolves the set of packages in your environment using a tiered algorithm. The `preserve` keyword argument allows you to key into a specific tier in the resolve algorithm. @@ -213,6 +223,7 @@ precompiled before, or the precompile cache has been deleted by the LRU cache st Pkg.add("Example") # Add a package from registry Pkg.add("Example", target=:weakdeps) # Add a package as a weak dependency Pkg.add("Example", target=:extras) # Add a package to the `[extras]` list +Pkg.add("Example"; prefer_loaded_versions=false) # Add a package, ignoring versions already loaded in this session Pkg.add("Example"; preserve=Pkg.PRESERVE_ALL) # Add the `Example` package and strictly preserve existing dependencies Pkg.add(name="Example", version="0.3") # Specify version; latest release in the 0.3 series Pkg.add(name="Example", version="0.3.1") # Specify version; exact release diff --git a/src/Resolve/graphtype.jl b/src/Resolve/graphtype.jl index 9e4b1b6425..4e2c95d299 100644 --- a/src/Resolve/graphtype.jl +++ b/src/Resolve/graphtype.jl @@ -234,6 +234,7 @@ mutable struct Graph newmsg::Vector{FieldValue} diff::Vector{FieldValue} cavfld::Vector{FieldValue} + preferred_versions::Dict{UUID, VersionNumber} function Graph( deps_compressed::Dict{UUID, Vector{Dict{VersionRange, Set{UUID}}}}, @@ -246,7 +247,8 @@ mutable struct Graph reqs::Requires, fixed::Dict{UUID, Fixed}, verbose::Bool = false, - julia_version::Union{VersionNumber, Nothing} = VERSION + julia_version::Union{VersionNumber, Nothing} = VERSION, + preferred_versions::Dict{UUID, VersionNumber} = Dict{UUID, VersionNumber}() ) # Tell the resolver about julia itself @@ -391,7 +393,7 @@ mutable struct Graph graph = new( data, gadj, gmsk, gconstr, adjdict, req_inds, fix_inds, ignored, solve_stack, spp, np, - FieldValue[], FieldValue[], FieldValue[] + FieldValue[], FieldValue[], FieldValue[], Dict{UUID, VersionNumber}(preferred_versions) ) _add_fixed!(graph, fixed) @@ -416,7 +418,8 @@ mutable struct Graph ignored = copy(graph.ignored) solve_stack = [([copy(gc0) for gc0 in sav_gconstr], copy(sav_ignored)) for (sav_gconstr, sav_ignored) in graph.solve_stack] - return new(data, gadj, gmsk, gconstr, adjdict, req_inds, fix_inds, ignored, solve_stack, spp, np) + preferred_versions = copy(graph.preferred_versions) + return new(data, gadj, gmsk, gconstr, adjdict, req_inds, fix_inds, ignored, solve_stack, spp, np, FieldValue[], FieldValue[], FieldValue[], preferred_versions) end end diff --git a/src/Resolve/maxsum.jl b/src/Resolve/maxsum.jl index 7fb71f12c7..c9b6eb528d 100644 --- a/src/Resolve/maxsum.jl +++ b/src/Resolve/maxsum.jl @@ -1,6 +1,10 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license const DEFAULT_MAX_TIME = "300" +# Prefer loaded versions, but not so strongly that it overrides compatibility constraints. +# This bonus is added to the version weight of already-loaded packages, making them more +# favorable than other versions but not creating hard requirements. +const PREFERRED_VERSION_WEIGHT_BONUS = VersionWeight(100, 0, 0) # Some parameters to drive the decimation process mutable struct MaxSumParams @@ -55,6 +59,19 @@ mutable struct Messages ## generate wveights (v0 == spp[p0] is the "uninstalled" state) vweight = [[VersionWeight(v0 < spp[p0] ? pvers[p0][v0] : v"0") for v0 in 1:spp[p0]] for p0 in 1:np] + preferred_versions = graph.preferred_versions + if !isempty(preferred_versions) + pkgs = graph.data.pkgs + vdict = graph.data.vdict + for p0 in 1:np + uuid = pkgs[p0] + pref_version = get(preferred_versions, uuid, nothing) + pref_version === nothing && continue + idx = get(vdict[p0], pref_version, 0) + (idx > 0 && idx < spp[p0]) || continue + vweight[p0][idx] += PREFERRED_VERSION_WEIGHT_BONUS + end + end # external fields: favor newest versions over older, and no-version over all; # explicit requirements use level l1 instead of l2 diff --git a/test/new.jl b/test/new.jl index 27b21c387f..7a6316723a 100644 --- a/test/new.jl +++ b/test/new.jl @@ -3922,4 +3922,28 @@ end end end +@testset "Pkg.add prefers loaded dependency versions" begin + isolate(loaded_depot = true) do + script = """ + using Pkg, Test + Pkg.activate(; temp = true) + io = IOBuffer() + Pkg.add(name = "Example", version = v"0.5.4", io = io) + add_output = String(take!(io)) + @test occursin("[7876af07] + Example v0.5.4", add_output) + using Example + Pkg.activate(; temp = true) + Pkg.add("Example", io = io) # v0.5.5 exists, but v0.5.4 is loaded + add_output = String(take!(io)) + @test occursin("was able to add the version of Example that is already loaded", add_output) + @test occursin("[7876af07] + Example v0.5.4", add_output) + """ + cmd = addenv( + `$(Base.julia_cmd()) --startup-file=no --project=$(dirname(@__DIR__)) -e $script`, + "JULIA_DEPOT_PATH" => join(DEPOT_PATH, Sys.iswindows() ? ";" : ":") + ) + @test Utils.show_output_if_command_errors(cmd) + end +end + end #module