Skip to content

Commit a807507

Browse files
IanButterworthIanButterworth
committed
prefer adding already loaded dep versions
1 parent bf67f63 commit a807507

File tree

4 files changed

+173
-18
lines changed

4 files changed

+173
-18
lines changed

‎src/Operations.jl‎

Lines changed: 123 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -708,7 +708,7 @@ end
708708
# all versioned packages should have a `tree_hash`
709709
function resolve_versions!(
710710
env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, julia_version,
711-
installed_only::Bool
711+
installed_only::Bool, preferred_versions::Dict{UUID, VersionNumber} = Dict{UUID, VersionNumber}()
712712
)
713713
installed_only = installed_only || OFFLINE_MODE[]
714714

@@ -777,7 +777,10 @@ function resolve_versions!(
777777
unbind_stdlibs = julia_version === VERSION
778778
reqs = Resolve.Requires(pkg.uuid => is_stdlib(pkg.uuid) && unbind_stdlibs ? VersionSpec("*") : VersionSpec(pkg.version) for pkg in pkgs)
779779
deps_map_compressed, compat_map_compressed, weak_deps_map_compressed, weak_compat_map_compressed, pkg_versions_map, pkg_versions_per_registry, uuid_to_name, reqs, fixed = deps_graph(env, registries, names, reqs, fixed, julia_version, installed_only)
780-
graph = Resolve.Graph(deps_map_compressed, compat_map_compressed, weak_deps_map_compressed, weak_compat_map_compressed, pkg_versions_map, pkg_versions_per_registry, uuid_to_name, reqs, fixed, false, julia_version)
780+
graph = Resolve.Graph(
781+
deps_map_compressed, compat_map_compressed, weak_deps_map_compressed, weak_compat_map_compressed,
782+
pkg_versions_map, pkg_versions_per_registry, uuid_to_name, reqs, fixed, false, julia_version, preferred_versions
783+
)
781784
Resolve.simplify_graph!(graph)
782785
vers = Resolve.resolve(graph)
783786

@@ -844,6 +847,84 @@ function resolve_versions!(
844847
return final_deps_map
845848
end
846849

850+
function collect_preferred_loaded_versions(env::EnvCache)
851+
preferred = Dict{UUID, VersionNumber}()
852+
for (pkgid, mod) in Base.loaded_modules
853+
pkgid isa Base.PkgId || continue
854+
pkg_uuid = pkgid.uuid
855+
pkg_uuid isa UUID || continue
856+
Types.is_stdlib(pkg_uuid) && continue
857+
haskey(env.manifest, pkg_uuid) && continue
858+
env.pkg !== nothing && pkg_uuid == env.pkg.uuid && continue
859+
version = Base.pkgversion(mod)
860+
version isa VersionNumber || continue
861+
preferred[pkg_uuid] = version
862+
end
863+
return preferred
864+
end
865+
866+
function preferred_loaded_packages_usage(
867+
pkgs::Vector{PackageSpec}, preferred_versions::Dict{UUID, VersionNumber}, manifest_uuids::Set{UUID},
868+
direct_requested_uuids::Set{UUID}
869+
)
870+
(isempty(preferred_versions) || isempty(pkgs)) && return String[], 0
871+
direct_names = String[]
872+
indirect_count = 0
873+
for pkg in pkgs
874+
uuid = pkg.uuid
875+
uuid isa UUID || continue
876+
uuid in manifest_uuids && continue
877+
preferred_version = get(preferred_versions, uuid, nothing)
878+
preferred_version === nothing && continue
879+
pkg_version = pkg.version
880+
pkg_version isa VersionNumber || continue
881+
pkg_version == preferred_version || continue
882+
pkg.name === nothing && continue
883+
if uuid in direct_requested_uuids
884+
push!(direct_names, pkg.name)
885+
else
886+
indirect_count += 1
887+
end
888+
end
889+
sort!(direct_names)
890+
unique!(direct_names)
891+
return direct_names, indirect_count
892+
end
893+
894+
function maybe_print_preferred_loaded_note(io::IO, direct_names::Vector{String}, indirect_count::Int)
895+
isempty(direct_names) && indirect_count == 0 && return
896+
parts = String[]
897+
if !isempty(direct_names)
898+
push!(parts, join(direct_names, ", "))
899+
end
900+
if indirect_count > 0
901+
dep_word = indirect_count == 1 ? "dependency" : "dependencies"
902+
push!(parts, "$(indirect_count) $(dep_word)")
903+
end
904+
joined = length(parts) == 2 ? string(parts[1], " and ", parts[2]) : parts[1]
905+
msg = if length(direct_names) + indirect_count > 1
906+
"Preferring versions of $(joined) that are already loaded"
907+
else
908+
"Preferring the version of $(joined) that is already loaded"
909+
end
910+
printpkgstyle(io, :Resolve, msg; color = Base.info_color())
911+
return
912+
end
913+
914+
function apply_preferred_versions_to_direct!(pkgs::Vector{PackageSpec}, preferred_versions::Dict{UUID, VersionNumber})
915+
isempty(preferred_versions) && return
916+
empty_spec = VersionSpec()
917+
for pkg in pkgs
918+
pkg.version == empty_spec || continue
919+
uuid = pkg.uuid
920+
uuid isa UUID || continue
921+
pref_version = get(preferred_versions, uuid, nothing)
922+
pref_version === nothing && continue
923+
pkg.version = VersionSpec(pref_version)
924+
end
925+
return
926+
end
927+
847928
get_or_make!(d::Dict{K, V}, k::K) where {K, V} = get!(d, k) do;
848929
V()
849930
end
@@ -2072,63 +2153,67 @@ end
20722153

20732154
function tiered_resolve(
20742155
env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, julia_version,
2075-
try_all_installed::Bool
2156+
try_all_installed::Bool; preferred_versions::Dict{UUID, VersionNumber} = Dict{UUID, VersionNumber}()
20762157
)
20772158
if try_all_installed
20782159
try # do not modify existing subgraph and only add installed versions of the new packages
20792160
@debug "tiered_resolve: trying PRESERVE_ALL_INSTALLED"
2080-
return targeted_resolve(env, registries, pkgs, PRESERVE_ALL_INSTALLED, julia_version)
2161+
return targeted_resolve(env, registries, pkgs, PRESERVE_ALL_INSTALLED, julia_version; preferred_versions)
20812162
catch err
20822163
err isa Resolve.ResolverError || rethrow()
20832164
end
20842165
end
20852166
try # do not modify existing subgraph
20862167
@debug "tiered_resolve: trying PRESERVE_ALL"
2087-
return targeted_resolve(env, registries, pkgs, PRESERVE_ALL, julia_version)
2168+
return targeted_resolve(env, registries, pkgs, PRESERVE_ALL, julia_version; preferred_versions)
20882169
catch err
20892170
err isa Resolve.ResolverError || rethrow()
20902171
end
20912172
try # do not modify existing direct deps
20922173
@debug "tiered_resolve: trying PRESERVE_DIRECT"
2093-
return targeted_resolve(env, registries, pkgs, PRESERVE_DIRECT, julia_version)
2174+
return targeted_resolve(env, registries, pkgs, PRESERVE_DIRECT, julia_version; preferred_versions)
20942175
catch err
20952176
err isa Resolve.ResolverError || rethrow()
20962177
end
20972178
try
20982179
@debug "tiered_resolve: trying PRESERVE_SEMVER"
2099-
return targeted_resolve(env, registries, pkgs, PRESERVE_SEMVER, julia_version)
2180+
return targeted_resolve(env, registries, pkgs, PRESERVE_SEMVER, julia_version; preferred_versions)
21002181
catch err
21012182
err isa Resolve.ResolverError || rethrow()
21022183
end
21032184
@debug "tiered_resolve: trying PRESERVE_NONE"
2104-
return targeted_resolve(env, registries, pkgs, PRESERVE_NONE, julia_version)
2185+
return targeted_resolve(env, registries, pkgs, PRESERVE_NONE, julia_version; preferred_versions)
21052186
end
21062187

2107-
function targeted_resolve(env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, preserve::PreserveLevel, julia_version)
2188+
function targeted_resolve(
2189+
env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, preserve::PreserveLevel,
2190+
julia_version; preferred_versions::Dict{UUID, VersionNumber} = Dict{UUID, VersionNumber}()
2191+
)
21082192
if preserve == PRESERVE_ALL || preserve == PRESERVE_ALL_INSTALLED
21092193
pkgs = load_all_deps(env, pkgs; preserve)
21102194
else
21112195
pkgs = load_direct_deps(env, pkgs; preserve)
21122196
end
21132197
check_registered(registries, pkgs)
21142198

2115-
deps_map = resolve_versions!(env, registries, pkgs, julia_version, preserve == PRESERVE_ALL_INSTALLED)
2199+
deps_map = resolve_versions!(env, registries, pkgs, julia_version, preserve == PRESERVE_ALL_INSTALLED, preferred_versions)
21162200
return pkgs, deps_map
21172201
end
21182202

21192203
function _resolve(
21202204
io::IO, env::EnvCache, registries::Vector{Registry.RegistryInstance},
2121-
pkgs::Vector{PackageSpec}, preserve::PreserveLevel, julia_version
2205+
pkgs::Vector{PackageSpec}, preserve::PreserveLevel, julia_version;
2206+
preferred_versions::Dict{UUID, VersionNumber} = Dict{UUID, VersionNumber}()
21222207
)
21232208
usingstrategy = preserve != PRESERVE_TIERED ? " using $preserve" : ""
21242209
printpkgstyle(io, :Resolving, "package versions$(usingstrategy)...")
21252210
return try
21262211
if preserve == PRESERVE_TIERED_INSTALLED
2127-
tiered_resolve(env, registries, pkgs, julia_version, true)
2212+
tiered_resolve(env, registries, pkgs, julia_version, true; preferred_versions)
21282213
elseif preserve == PRESERVE_TIERED
2129-
tiered_resolve(env, registries, pkgs, julia_version, false)
2214+
tiered_resolve(env, registries, pkgs, julia_version, false; preferred_versions)
21302215
else
2131-
targeted_resolve(env, registries, pkgs, preserve, julia_version)
2216+
targeted_resolve(env, registries, pkgs, preserve, julia_version; preferred_versions)
21322217
end
21332218
catch err
21342219

@@ -2235,11 +2320,34 @@ function add(
22352320
return
22362321
end
22372322

2323+
preferred_loaded_versions = Dict{UUID, VersionNumber}()
2324+
existing_manifest_uuids = Set{UUID}()
2325+
preferred_direct_note_names = String[]
2326+
preferred_indirect_note_count = 0
2327+
direct_requested_uuids = Set{UUID}()
22382328
foreach(pkg -> target_field[pkg.name] = pkg.uuid, pkgs) # update set of deps/weakdeps/extras
2329+
for pkg in pkgs
2330+
uuid = pkg.uuid
2331+
uuid isa UUID || continue
2332+
push!(direct_requested_uuids, uuid)
2333+
end
2334+
2335+
if target == :deps
2336+
preferred_loaded_versions = collect_preferred_loaded_versions(ctx.env)
2337+
existing_manifest_uuids = Set(keys(ctx.env.manifest))
2338+
end
22392339

22402340
if target == :deps # nothing to resolve/install if it's weak or extras
22412341
# resolve
2242-
man_pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, preserve, ctx.julia_version)
2342+
apply_preferred_versions_to_direct!(pkgs, preferred_loaded_versions)
2343+
man_pkgs, deps_map = _resolve(
2344+
ctx.io, ctx.env, ctx.registries, pkgs, preserve, ctx.julia_version;
2345+
preferred_versions = preferred_loaded_versions
2346+
)
2347+
preferred_direct_note_names, preferred_indirect_note_count = preferred_loaded_packages_usage(
2348+
man_pkgs, preferred_loaded_versions, existing_manifest_uuids, direct_requested_uuids
2349+
)
2350+
maybe_print_preferred_loaded_note(ctx.io, preferred_direct_note_names, preferred_indirect_note_count)
22432351
update_manifest!(ctx.env, man_pkgs, deps_map, ctx.julia_version, ctx.registries)
22442352
new_apply = download_source(ctx)
22452353
fixups_from_projectfile!(ctx)

‎src/Resolve/graphtype.jl‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,7 @@ mutable struct Graph
234234
newmsg::Vector{FieldValue}
235235
diff::Vector{FieldValue}
236236
cavfld::Vector{FieldValue}
237+
preferred_versions::Dict{UUID, VersionNumber}
237238

238239
function Graph(
239240
deps_compressed::Dict{UUID, Vector{Dict{VersionRange, Set{UUID}}}},
@@ -246,7 +247,8 @@ mutable struct Graph
246247
reqs::Requires,
247248
fixed::Dict{UUID, Fixed},
248249
verbose::Bool = false,
249-
julia_version::Union{VersionNumber, Nothing} = VERSION
250+
julia_version::Union{VersionNumber, Nothing} = VERSION,
251+
preferred_versions::Dict{UUID, VersionNumber} = Dict{UUID, VersionNumber}()
250252
)
251253

252254
# Tell the resolver about julia itself
@@ -391,7 +393,7 @@ mutable struct Graph
391393

392394
graph = new(
393395
data, gadj, gmsk, gconstr, adjdict, req_inds, fix_inds, ignored, solve_stack, spp, np,
394-
FieldValue[], FieldValue[], FieldValue[]
396+
FieldValue[], FieldValue[], FieldValue[], Dict{UUID, VersionNumber}(preferred_versions)
395397
)
396398

397399
_add_fixed!(graph, fixed)
@@ -416,7 +418,8 @@ mutable struct Graph
416418
ignored = copy(graph.ignored)
417419
solve_stack = [([copy(gc0) for gc0 in sav_gconstr], copy(sav_ignored)) for (sav_gconstr, sav_ignored) in graph.solve_stack]
418420

419-
return new(data, gadj, gmsk, gconstr, adjdict, req_inds, fix_inds, ignored, solve_stack, spp, np)
421+
preferred_versions = copy(graph.preferred_versions)
422+
return new(data, gadj, gmsk, gconstr, adjdict, req_inds, fix_inds, ignored, solve_stack, spp, np, FieldValue[], FieldValue[], FieldValue[], preferred_versions)
420423
end
421424
end
422425

‎src/Resolve/maxsum.jl‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# This file is a part of Julia. License is MIT: https://julialang.org/license
22

33
const DEFAULT_MAX_TIME = "300"
4+
const PREFERRED_VERSION_WEIGHT_BONUS = VersionWeight(typemax(Int32))
45

56
# Some parameters to drive the decimation process
67
mutable struct MaxSumParams
@@ -55,6 +56,19 @@ mutable struct Messages
5556

5657
## generate wveights (v0 == spp[p0] is the "uninstalled" state)
5758
vweight = [[VersionWeight(v0 < spp[p0] ? pvers[p0][v0] : v"0") for v0 in 1:spp[p0]] for p0 in 1:np]
59+
preferred_versions = graph.preferred_versions
60+
if !isempty(preferred_versions)
61+
pkgs = graph.data.pkgs
62+
vdict = graph.data.vdict
63+
for p0 in 1:np
64+
uuid = pkgs[p0]
65+
pref_version = get(preferred_versions, uuid, nothing)
66+
pref_version === nothing && continue
67+
idx = get(vdict[p0], pref_version, 0)
68+
(idx > 0 && idx < spp[p0]) || continue
69+
vweight[p0][idx] += PREFERRED_VERSION_WEIGHT_BONUS
70+
end
71+
end
5872

5973
# external fields: favor newest versions over older, and no-version over all;
6074
# explicit requirements use level l1 instead of l2

‎test/new.jl‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3922,4 +3922,34 @@ end
39223922
end
39233923
end
39243924

3925+
@testset "Pkg.add prefers loaded dependency versions" begin
3926+
isolate(loaded_depot = true) do
3927+
mktempdir() do tmp
3928+
pkg_path = copy_test_package(tmp, "PackageWithDependency"; use_pkg = false)
3929+
script = """
3930+
using Pkg
3931+
Pkg.activate(; temp = true)
3932+
Pkg.add(name = "Example", version = v"0.5.1")
3933+
using Example
3934+
@test pkgversion(Example) == v"0.5.1"
3935+
Pkg.activate(; temp = true)
3936+
io = IOBuffer()
3937+
Pkg.add("Example", io = io)
3938+
add_output = String(take!(io))
3939+
@test occursin("Preferring the version of Example that is already loaded", add_output)
3940+
@test occursin("direct dep Example", add_output)
3941+
deps = Pkg.dependencies()
3942+
example_dep = get(deps, exuuid, nothing)
3943+
@test example_dep !== nothing
3944+
@test example_dep.version == pkgversion(Example)
3945+
"""
3946+
cmd = addenv(
3947+
`$(Base.julia_cmd()) --startup-file=no --project=$(dirname(@__DIR__)) -e $script`,
3948+
"JULIA_DEPOT_PATH" => join(DEPOT_PATH, Sys.iswindows() ? ";" : ":")
3949+
)
3950+
@test Utils.show_output_if_command_errors(cmd)
3951+
end
3952+
end
3953+
end
3954+
39253955
end #module

0 commit comments

Comments
 (0)