From 83e13631e712384340ca5dff8c390f4dc6ad2479 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 23 Nov 2024 15:44:12 -0500 Subject: [PATCH 001/154] Run CI on backport branch too (#4094) --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 11f1643502..290596a0eb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,10 +4,12 @@ on: branches: - 'master' - 'release-*' + - 'backports-release-*' push: branches: - 'master' - 'release-*' + - 'backports-release-*' tags: '*' defaults: run: From 814949ed2334afcaf51e437a5fa4002d1ae7eaaa Mon Sep 17 00:00:00 2001 From: Felix Hagemann Date: Mon, 2 Dec 2024 16:07:31 +0100 Subject: [PATCH 002/154] Increase version of `StaticArrays` in `why` tests (#4077) --- test/new.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/new.jl b/test/new.jl index 5b10b72965..4d54ef2f1c 100644 --- a/test/new.jl +++ b/test/new.jl @@ -1585,7 +1585,7 @@ end @testset "why" begin isolate() do - Pkg.add(name = "StaticArrays", version = "1.5.0") + Pkg.add(name = "StaticArrays", version = "1.5.20") io = IOBuffer() Pkg.why("StaticArrays"; io) From b610661204a2b0e866847a4377ef4779965f9950 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Mon, 2 Dec 2024 10:30:35 -0500 Subject: [PATCH 003/154] rename FORMER_STDLIBS -> UPGRADABLE_STDLIBS (#4070) --- src/Types.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Types.jl b/src/Types.jl index 859b93221a..2d01527073 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -460,8 +460,8 @@ is_project_uuid(env::EnvCache, uuid::UUID) = project_uuid(env) == uuid # Context # ########### -const FORMER_STDLIBS = ["DelimitedFiles", "Statistics"] -const FORMER_STDLIBS_UUIDS = Set{UUID}() +const UPGRADABLE_STDLIBS = ["DelimitedFiles", "Statistics"] +const UPGRADABLE_STDLIBS_UUIDS = Set{UUID}() const STDLIB = Ref{DictStdLibs}() function load_stdlib() stdlib = DictStdLibs() @@ -473,8 +473,8 @@ function load_stdlib() v_str = get(project, "version", nothing)::Union{String, Nothing} version = isnothing(v_str) ? nothing : VersionNumber(v_str) nothing === uuid && continue - if name in FORMER_STDLIBS - push!(FORMER_STDLIBS_UUIDS, UUID(uuid)) + if name in UPGRADABLE_STDLIBS + push!(UPGRADABLE_STDLIBS_UUIDS, UUID(uuid)) continue end deps = UUID.(values(get(project, "deps", Dict{String,Any}()))) @@ -499,7 +499,7 @@ end is_stdlib(uuid::UUID) = uuid in keys(stdlib_infos()) # Includes former stdlibs function is_or_was_stdlib(uuid::UUID, julia_version::Union{VersionNumber, Nothing}) - return is_stdlib(uuid, julia_version) || uuid in FORMER_STDLIBS_UUIDS + return is_stdlib(uuid, julia_version) || uuid in UPGRADABLE_STDLIBS_UUIDS end From cd75456a8786a83f7cdb9a5f97bbf90337c3284a Mon Sep 17 00:00:00 2001 From: Nils Gudat Date: Mon, 2 Dec 2024 17:18:06 +0000 Subject: [PATCH 004/154] Fix heading (#4102) --- docs/src/environments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/environments.md b/docs/src/environments.md index 54fa4e9fe9..fd16427e10 100644 --- a/docs/src/environments.md +++ b/docs/src/environments.md @@ -1,4 +1,4 @@ -# [**4.** Working with Environment](@id Working-with-Environments) +# [**4.** Working with Environments](@id Working-with-Environments) The following discusses Pkg's interaction with environments. For more on the role, environments play in code loading, including the "stack" of environments from which code can be loaded, see [this section in the Julia manual](https://docs.julialang.org/en/v1/manual/code-loading/#Environments-1). From d84a1a38b6466fa7400e9ad2874a0ef963a10456 Mon Sep 17 00:00:00 2001 From: Timothy Date: Tue, 3 Dec 2024 21:50:53 +0800 Subject: [PATCH 005/154] Allow use of a url and subdir in [sources] (#4039) --- docs/src/toml-files.md | 1 + src/API.jl | 6 ++++++ src/project.jl | 2 +- test/sources.jl | 6 ++++++ .../WithSources/TestMonorepo/Project.toml | 13 +++++++++++++ .../WithSources/TestMonorepo/src/TestMonorepo.jl | 5 +++++ .../WithSources/TestMonorepo/test/runtests.jl | 1 + 7 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 test/test_packages/WithSources/TestMonorepo/Project.toml create mode 100644 test/test_packages/WithSources/TestMonorepo/src/TestMonorepo.jl create mode 100644 test/test_packages/WithSources/TestMonorepo/test/runtests.jl diff --git a/docs/src/toml-files.md b/docs/src/toml-files.md index 79496e0321..262c1b5767 100644 --- a/docs/src/toml-files.md +++ b/docs/src/toml-files.md @@ -100,6 +100,7 @@ corresponding manifest file. ```toml [sources] Example = {url = "https://github.com/JuliaLang/Example.jl", rev = "custom_branch"} +WithinMonorepo = {url = "https://github.org/author/BigProject", subdir = "SubPackage"} SomeDependency = {path = "deps/SomeDependency.jl"} ``` diff --git a/src/API.jl b/src/API.jl index 5e5723a8b7..2218f248b3 100644 --- a/src/API.jl +++ b/src/API.jl @@ -195,6 +195,9 @@ function update_source_if_set(project, pkg) if pkg.path !== nothing source["path"] = pkg.path end + if pkg.subdir !== nothing + source["subdir"] = pkg.subdir + end path, repo = get_path_repo(project, pkg.name) if path !== nothing pkg.path = path @@ -205,6 +208,9 @@ function update_source_if_set(project, pkg) if repo.rev !== nothing pkg.repo.rev = repo.rev end + if repo.subdir !== nothing + pkg.repo.subdir = repo.subdir + end end function develop(ctx::Context, pkgs::Vector{PackageSpec}; shared::Bool=true, diff --git a/src/project.jl b/src/project.jl index f7a7e83757..1b559898c2 100644 --- a/src/project.jl +++ b/src/project.jl @@ -92,7 +92,7 @@ read_project_compat(raw, project::Project) = read_project_sources(::Nothing, project::Project) = Dict{String,Any}() function read_project_sources(raw::Dict{String,Any}, project::Project) - valid_keys = ("path", "url", "rev") + valid_keys = ("path", "url", "rev", "subdir") sources = Dict{String,Any}() for (name, source) in raw if !(source isa AbstractDict) diff --git a/test/sources.jl b/test/sources.jl index 311b203f00..1497464aab 100644 --- a/test/sources.jl +++ b/test/sources.jl @@ -33,6 +33,12 @@ temp_pkg_dir() do project_path end end + cd(joinpath(dir, "WithSources", "TestMonorepo")) do + with_current_env() do + Pkg.test() + end + end + cd(joinpath(dir, "WithSources", "TestProject")) do with_current_env() do Pkg.test() diff --git a/test/test_packages/WithSources/TestMonorepo/Project.toml b/test/test_packages/WithSources/TestMonorepo/Project.toml new file mode 100644 index 0000000000..b6a31a7b51 --- /dev/null +++ b/test/test_packages/WithSources/TestMonorepo/Project.toml @@ -0,0 +1,13 @@ +name = "TestMonorepo" +uuid = "864d8eef-2526-4817-933e-34008eadd182" +authors = ["KristofferC "] +version = "0.1.0" + +[extras] +Example = "d359f271-ef68-451f-b4fc-6b43e571086c" + +[sources] +Example = {url = "https://github.com/JuliaLang/Pkg.jl", subdir = "test/test_packages/Example"} + +[targets] +test = ["Example"] diff --git a/test/test_packages/WithSources/TestMonorepo/src/TestMonorepo.jl b/test/test_packages/WithSources/TestMonorepo/src/TestMonorepo.jl new file mode 100644 index 0000000000..e4d52f12ff --- /dev/null +++ b/test/test_packages/WithSources/TestMonorepo/src/TestMonorepo.jl @@ -0,0 +1,5 @@ +module TestMonorepo + +greet() = print("Hello World!") + +end diff --git a/test/test_packages/WithSources/TestMonorepo/test/runtests.jl b/test/test_packages/WithSources/TestMonorepo/test/runtests.jl new file mode 100644 index 0000000000..3e04fee8cc --- /dev/null +++ b/test/test_packages/WithSources/TestMonorepo/test/runtests.jl @@ -0,0 +1 @@ +using Example From f9ad1c01a05e507c323661dc60469a68b5788727 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Mon, 9 Dec 2024 13:44:40 -0500 Subject: [PATCH 006/154] Fix a couple of bad JULIA_DEPOT_PATH settings (#4110) --- test/new.jl | 2 +- test/repl.jl | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/new.jl b/test/new.jl index 4d54ef2f1c..1b78751b71 100644 --- a/test/new.jl +++ b/test/new.jl @@ -2998,7 +2998,7 @@ end @testset "relative depot path" begin isolate(loaded_depot=false) do mktempdir() do tmp - withenv("JULIA_DEPOT_PATH" => tmp) do + withenv("JULIA_DEPOT_PATH" => tmp * (Sys.iswindows() ? ";" : ":")) do Base.init_depot_path() cp(joinpath(@__DIR__, "test_packages", "BasicSandbox"), joinpath(tmp, "BasicSandbox")) git_init_and_commit(joinpath(tmp, "BasicSandbox")) diff --git a/test/repl.jl b/test/repl.jl index b0d729dd92..ce817eb20c 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -738,7 +738,9 @@ end tmp_55850 = mktempdir() tmp_sym_link = joinpath(tmp_55850, "sym") symlink(tmp_55850, tmp_sym_link; dir_target=true) - withenv("JULIA_DEPOT_PATH" => tmp_sym_link * (Sys.iswindows() ? ";" : ":"), "JULIA_LOAD_PATH" => nothing) do + depot_path = join([tmp_sym_link, Base.DEPOT_PATH...], Sys.iswindows() ? ";" : ":") + # include the symlink in the depot path and include the regular default depot so we don't precompile this Pkg again + withenv("JULIA_DEPOT_PATH" => depot_path, "JULIA_LOAD_PATH" => nothing) do prompt = readchomp(`$(Base.julia_cmd()[1]) --project=$(dirname(@__DIR__)) --startup-file=no -e "using Pkg, REPL; Pkg.activate(io=devnull); REPLExt = Base.get_extension(Pkg, :REPLExt); print(REPLExt.promptf())"`) @test prompt == "(@v$(VERSION.major).$(VERSION.minor)) pkg> " end From e7c37f34293ab12051258828884755ea116b77df Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Mon, 9 Dec 2024 14:01:04 -0500 Subject: [PATCH 007/154] status: highlight when deps have different loaded versions (#4109) --- CHANGELOG.md | 1 + src/Operations.jl | 19 +++++++++++++++++++ test/new.jl | 15 +++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 056a6f1f36..7f39463873 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Pkg v1.12 Release Notes - Pkg now has support for "workspaces" which is a way to resolve multiple project files into a single manifest. The functions `Pkg.status`, `Pkg.why`, `Pkg.instantiate`, `Pkg.precompile` (and their REPL variants) have been updated to take a `workspace` option. Read more about this feature in the manual about the TOML-files. +- `status` now shows when different versions/sources of dependencies are loaded than that which is expected by the manifest ([#4109]) Pkg v1.11 Release Notes ======================= diff --git a/src/Operations.jl b/src/Operations.jl index 6d6bc94558..4ade9ae4a6 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -2736,6 +2736,25 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie printstyled(io, pkg_str; color=Base.warn_color()) end end + # show if loaded version and version in the manifest doesn't match + pkg_spec = something(pkg.new, pkg.old) + pkgid = Base.PkgId(pkg.uuid, pkg_spec.name) + m = get(Base.loaded_modules, pkgid, nothing) + if m isa Module && pkg_spec.version !== nothing + loaded_path = pathof(m) + env_path = Base.locate_package(pkgid) # nothing if not installed + if loaded_path !== nothing && env_path !== nothing &&!samefile(loaded_path, env_path) + loaded_version = pkgversion(m) + env_version = pkg_spec.version + if loaded_version !== env_version + printstyled(io, " [loaded: v$loaded_version]"; color=:light_yellow) + else + loaded_version_str = loaded_version === nothing ? "" : " (v$loaded_version)" + env_version_str = env_version === nothing ? "" : " (v$env_version)" + printstyled(io, " [loaded: `$loaded_path`$loaded_version_str expected `$env_path`$env_version_str]"; color=:light_yellow) + end + end + end if extensions && !diff && pkg.extinfo !== nothing println(io) diff --git a/test/new.jl b/test/new.jl index 1b78751b71..96debeac6f 100644 --- a/test/new.jl +++ b/test/new.jl @@ -3243,4 +3243,19 @@ end end end +@testset "status showing incompatible loaded deps" begin + cmd = addenv(`$(Base.julia_cmd()) --color=no --startup-file=no -e " + using Pkg + Pkg.activate(temp=true) + Pkg.add(Pkg.PackageSpec(name=\"Example\", version=v\"0.5.4\")) + using Example + Pkg.activate(temp=true) + Pkg.add(Pkg.PackageSpec(name=\"Example\", version=v\"0.5.5\")) + "`) + iob = IOBuffer() + run(pipeline(cmd, stderr=iob, stdout=iob)) + out = String(take!(iob)) + @test occursin("[loaded: v0.5.4]", out) +end + end #module From 47ad7e39cfb02fefca9e38fe288541420319dff1 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 16 Dec 2024 15:28:04 +0100 Subject: [PATCH 008/154] add some explicit precompile statements for `Pkg.status`. For some reason, these don't seem to "take" on our precompile workload. (#4114) Before: ``` julia> @time @eval Pkg.status() 0.470141 seconds (2.28 M allocations: 151.859 MiB, 2.91% gc time, 59.31% compilation time: 11% of which was recompilation) ``` After ``` julia> @time @eval Pkg.status() 0.220179 seconds (1.69 M allocations: 122.680 MiB, 6.14% gc time, 25.84% compilation time: 74% of which was recompilation) ``` Co-authored-by: KristofferC --- src/precompile.jl | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/precompile.jl b/src/precompile.jl index 761f64efda..c5283d7c51 100644 --- a/src/precompile.jl +++ b/src/precompile.jl @@ -159,6 +159,29 @@ let Base.precompile(Tuple{Type{Pkg.REPLMode.QString}, String, Bool}) Base.precompile(Tuple{typeof(Pkg.REPLMode.parse_package), Array{Pkg.REPLMode.QString, 1}, Base.Dict{Symbol, Any}}) Base.precompile(Tuple{Type{Pkg.REPLMode.Command}, Pkg.REPLMode.CommandSpec, Base.Dict{Symbol, Any}, Array{Pkg.Types.PackageSpec, 1}}) + + # Manually added from trace compiling Pkg.status. + Base.precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:color,), Tuple{Symbol}}, typeof(Base.printstyled), Base.IOContext{Base.GenericIOBuffer{Memory{UInt8}}}, Char}) + Base.precompile(Tuple{typeof(Base.join), Base.GenericIOBuffer{Memory{UInt8}}, Tuple{UInt64}, Char}) + Base.precompile(Tuple{typeof(Base.empty), Base.Dict{Any, Any}, Type{String}, Type{Base.UUID}}) + Base.precompile(Tuple{typeof(Base.join), Base.GenericIOBuffer{Memory{UInt8}}, Tuple{UInt32}, Char}) + Base.precompile(Tuple{typeof(Base.unsafe_read), Base.PipeEndpoint, Ptr{UInt8}, UInt64}) + Base.precompile(Tuple{typeof(Base.readbytes!), Base.PipeEndpoint, Array{UInt8, 1}, Int64}) + Base.precompile(Tuple{typeof(Base.closewrite), Base.PipeEndpoint}) + Base.precompile(Tuple{typeof(Base.convert), Type{Base.Dict{String, Union{Array{String, 1}, String}}}, Base.Dict{String, Any}}) + Base.precompile(Tuple{typeof(Base.map), Function, Array{Any, 1}}) + Base.precompile(Tuple{Type{Array{Dates.DateTime, 1}}, UndefInitializer, Tuple{Int64}}) + Base.precompile(Tuple{typeof(Base.maximum), Array{Dates.DateTime, 1}}) + Base.precompile(Tuple{Type{Pair{A, B} where B where A}, String, Dates.DateTime}) + Base.precompile(Tuple{typeof(Base.map), Function, Array{Base.Dict{String, Dates.DateTime}, 1}}) + Base.precompile(Tuple{typeof(TOML.Internals.Printer.is_array_of_tables), Array{Base.Dict{String, Dates.DateTime}, 1}}) + Base.precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:indent, :sorted, :by, :inline_tables), Tuple{Int64, Bool, typeof(Base.identity), Base.IdSet{Base.Dict{String, V} where V}}}, typeof(TOML.Internals.Printer.print_table), Nothing, Base.IOStream, Base.Dict{String, Dates.DateTime}, Array{String, 1}}) + Base.precompile(Tuple{typeof(Base.deepcopy_internal), Base.Dict{String, Base.UUID}, Base.IdDict{Any, Any}}) + Base.precompile(Tuple{typeof(Base.deepcopy_internal), Base.Dict{String, Union{Array{String, 1}, String}}, Base.IdDict{Any, Any}}) + Base.precompile(Tuple{typeof(Base.deepcopy_internal), Base.Dict{String, Array{String, 1}}, Base.IdDict{Any, Any}}) + Base.precompile(Tuple{typeof(Base.deepcopy_internal), Base.Dict{String, Base.Dict{String, String}}, Base.IdDict{Any, Any}}) + Base.precompile(Tuple{typeof(Base.deepcopy_internal), Tuple{String}, Base.IdDict{Any, Any}}) + Base.precompile(Tuple{Type{Memory{Pkg.Types.PackageSpec}}, UndefInitializer, Int64}) end copy!(DEPOT_PATH, original_depot_path) copy!(LOAD_PATH, original_load_path) From a1cfc6d1bf0db6c99498482906db363f941bf09d Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 17 Dec 2024 13:01:05 +0100 Subject: [PATCH 009/154] Improve latency for `Pkg.add` (#4115) * precompile registry update Before ``` julia> @time @eval Pkg.Registry.update() Updating registry at `~/.julia/registries/General.toml` 1.006864 seconds (2.18 M allocations: 148.068 MiB, 1.42% gc time, 43.47% compilation time: 6% of which was recompilation) ``` After ``` julia> @time @eval Pkg.Registry.update() Updating registry at `~/.julia/registries/General.toml` 0.677679 seconds (1.75 M allocations: 125.226 MiB, 2.03% gc time, 14.63% compilation time: 39% of which was recompilation) ``` * add some explicit precompile statements for `Pkg.add` For some reason, these don't seem to "take" on our precompile workload. Before: ``` 1.379429 seconds (3.39 M allocations: 211.476 MiB, 2.36% gc time, 60.64% compilation time: 38% of which was recompilation) ``` After: ``` 0.963989 seconds (2.83 M allocations: 182.510 MiB, 1.67% gc time, 42.22% compilation time: 75% of which was recompilation) ``` * remove call to update registry --------- Co-authored-by: KristofferC --- src/precompile.jl | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/precompile.jl b/src/precompile.jl index c5283d7c51..3a7e2ea32f 100644 --- a/src/precompile.jl +++ b/src/precompile.jl @@ -182,6 +182,32 @@ let Base.precompile(Tuple{typeof(Base.deepcopy_internal), Base.Dict{String, Base.Dict{String, String}}, Base.IdDict{Any, Any}}) Base.precompile(Tuple{typeof(Base.deepcopy_internal), Tuple{String}, Base.IdDict{Any, Any}}) Base.precompile(Tuple{Type{Memory{Pkg.Types.PackageSpec}}, UndefInitializer, Int64}) + + # Manually added from trace compiling Pkg.add + # Why needed? Something with constant prop overspecialization? + Base.precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:io, :update_cooldown), Tuple{Base.IOContext{IO}, Dates.Day}}, typeof(Pkg.Registry.update)}) + + Base.precompile(Tuple{Type{Memory{Pkg.Types.PackageSpec}}, UndefInitializer, Int64}) + Base.precompile(Tuple{typeof(Base.hash), Tuple{String, UInt64}, UInt64}) + Base.precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:context,), Tuple{Base.TTY}}, typeof(Base.sprint), Function, Tuple{Pkg.Versions.VersionSpec}}) + Base.precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:context,), Tuple{Base.TTY}}, typeof(Base.sprint), Function, Tuple{String}}) + Base.precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:context,), Tuple{Base.TTY}}, typeof(Base.sprint), Function, Tuple{Base.VersionNumber}}) + Base.precompile(Tuple{typeof(Base.join), Base.IOContext{Base.GenericIOBuffer{Memory{UInt8}}}, Tuple{String, UInt64}, Char}) + Base.precompile(Tuple{typeof(Base.vcat), Base.BitArray{2}, Base.BitArray{2}}) + Base.precompile(Tuple{typeof(Base.vcat), Base.BitArray{2}}) + Base.precompile(Tuple{typeof(Base.vcat), Base.BitArray{2}, Base.BitArray{2}, Base.BitArray{2}}) + Base.precompile(Tuple{typeof(Base.vcat), Base.BitArray{2}, Base.BitArray{2}, Base.BitArray{2}, Vararg{Base.BitArray{2}}}) + Base.precompile(Tuple{typeof(Base.vcat), Base.BitArray{1}, Base.BitArray{1}}) + Base.precompile(Tuple{typeof(Base.vcat), Base.BitArray{1}, Base.BitArray{1}, Base.BitArray{1}, Vararg{Base.BitArray{1}}}) + Base.precompile(Tuple{typeof(Base.:(==)), Base.Dict{String, Any}, Base.Dict{String, Any}}) + Base.precompile(Tuple{typeof(Base.join), Base.GenericIOBuffer{Memory{UInt8}}, Tuple{String}, Char}) + Base.precompile(Tuple{typeof(Base.values), Base.Dict{String, Array{Base.Dict{String, Any}, 1}}}) + Base.precompile(Tuple{typeof(Base.all), Base.Generator{Base.ValueIterator{Base.Dict{String, Array{Base.Dict{String, Any}, 1}}}, TOML.Internals.Printer.var"#5#6"}}) + Base.precompile(Tuple{typeof(TOML.Internals.Printer.is_array_of_tables), Array{Base.Dict{String, Any}, 1}}) + Base.precompile(Tuple{Type{Array{Dates.DateTime, 1}}, UndefInitializer, Tuple{Int64}}) + Base.precompile(Tuple{Type{Pair{A, B} where B where A}, String, Dates.DateTime}) + Base.precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:internal_call, :strict, :warn_loaded, :timing, :_from_loading, :configs, :manifest, :io), Tuple{Bool, Bool, Bool, Bool, Bool, Pair{Base.Cmd, Base.CacheFlags}, Bool, Base.TTY}}, typeof(Base.Precompilation.precompilepkgs), Array{String, 1}}) + ################ end copy!(DEPOT_PATH, original_depot_path) copy!(LOAD_PATH, original_load_path) From b995782c1c5b9618d2012c2e68f067da355c839f Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 17 Dec 2024 22:22:56 +0100 Subject: [PATCH 010/154] avoid writing and reparsing the artifact usage file once per artifact file in the project deps (#4117) --- src/Operations.jl | 5 ++--- src/Types.jl | 13 ++++++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/Operations.jl b/src/Operations.jl index 4ade9ae4a6..a4e2c77553 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -982,9 +982,8 @@ function download_artifacts(ctx::Context; end end - for f in used_artifact_tomls - write_env_usage(f, "artifact_usage.toml") - end + + write_env_usage(used_artifact_tomls, "artifact_usage.toml") end function check_artifacts_downloaded(pkg_root::String; platform::AbstractPlatform=HostPlatform()) diff --git a/src/Types.jl b/src/Types.jl index 2d01527073..d9c8741b27 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -606,9 +606,14 @@ function workspace_resolve_hash(env::EnvCache) return bytes2hex(sha1(str)) end -function write_env_usage(source_file::AbstractString, usage_filepath::AbstractString) + +write_env_usage(source_file::AbstractString, usage_filepath::AbstractString) = + write_env_usage([source_file], usage_filepath) + +function write_env_usage(source_files, usage_filepath::AbstractString) # Don't record ghost usage - !isfile(source_file) && return + source_files = filter(isfile, source_files) + isempty(source_files) && return # Ensure that log dir exists !ispath(logdir()) && mkpath(logdir()) @@ -630,7 +635,9 @@ function write_env_usage(source_file::AbstractString, usage_filepath::AbstractSt end # record new usage - usage[source_file] = [Dict("time" => timestamp)] + for source_file in source_files + usage[source_file] = [Dict("time" => timestamp)] + end # keep only latest usage info for k in keys(usage) From c7e611bc89826bc462c4b2a308f1a71dbb617145 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sun, 29 Dec 2024 19:54:32 -0500 Subject: [PATCH 011/154] support before/after openssl switch (#4123) --- test/new.jl | 10 + .../{Manifest.toml => Manifest_MbedTLS.toml} | 0 .../Manifest_OpenSSL.toml | 258 ++++++++++++++++++ 3 files changed, 268 insertions(+) rename test/test_packages/ShouldPreserveSemver/{Manifest.toml => Manifest_MbedTLS.toml} (100%) create mode 100644 test/test_packages/ShouldPreserveSemver/Manifest_OpenSSL.toml diff --git a/test/new.jl b/test/new.jl index 96debeac6f..fee012dd3d 100644 --- a/test/new.jl +++ b/test/new.jl @@ -964,6 +964,16 @@ end @test Pkg.dependencies()[ordered_collections].version == v"1.0.1" # sanity check # SEMVER copy_test_package(tmp, "ShouldPreserveSemver"; use_pkg=false) + + # Support julia versions before & after the MbedTLS > OpenSSL switch + OpenSSL_pkgid = Base.PkgId(Base.UUID("458c3c95-2e84-50aa-8efc-19380b2a3a95"), "OpenSSL_jll") + manifest_to_use = if haskey(Base.loaded_modules, OpenSSL_pkgid) + joinpath(tmp, "ShouldPreserveSemver", "Manifest_OpenSSL.toml") + else + joinpath(tmp, "ShouldPreserveSemver", "Manifest_MbedTLS.toml") + end + mv(manifest_to_use, joinpath(tmp, "ShouldPreserveSemver", "Manifest.toml")) + Pkg.activate(joinpath(tmp, "ShouldPreserveSemver")) light_graphs = UUID("093fc24a-ae57-5d10-9952-331d41423f4d") meta_graphs = UUID("626554b9-1ddb-594c-aa3c-2596fe9399a5") diff --git a/test/test_packages/ShouldPreserveSemver/Manifest.toml b/test/test_packages/ShouldPreserveSemver/Manifest_MbedTLS.toml similarity index 100% rename from test/test_packages/ShouldPreserveSemver/Manifest.toml rename to test/test_packages/ShouldPreserveSemver/Manifest_MbedTLS.toml diff --git a/test/test_packages/ShouldPreserveSemver/Manifest_OpenSSL.toml b/test/test_packages/ShouldPreserveSemver/Manifest_OpenSSL.toml new file mode 100644 index 0000000000..f1fe5150bd --- /dev/null +++ b/test/test_packages/ShouldPreserveSemver/Manifest_OpenSSL.toml @@ -0,0 +1,258 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.9.0-DEV" +manifest_format = "2.0" +project_hash = "9af0d7a4d60a77b1a42f518d7da50edc4261ffcb" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" + +[[deps.Arpack]] +deps = ["BinaryProvider", "Libdl", "LinearAlgebra"] +git-tree-sha1 = "07a2c077bdd4b6d23a40342a8a108e2ee5e58ab6" +uuid = "7d9fca2a-8960-54d3-9f78-7d1dccf2cb97" +version = "0.3.1" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[deps.BinaryProvider]] +deps = ["Libdl", "Logging", "SHA"] +git-tree-sha1 = "c7361ce8a2129f20b0e05a89f7070820cfed6648" +uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232" +version = "0.5.6" + +[[deps.CSTParser]] +deps = ["Tokenize"] +git-tree-sha1 = "c69698c3d4a7255bc1b4bc2afc09f59db910243b" +uuid = "00ebfdb7-1f24-5e51-bd34-a7502290713f" +version = "0.6.2" + +[[deps.CodecZlib]] +deps = ["BinaryProvider", "Libdl", "TranscodingStreams"] +git-tree-sha1 = "05916673a2627dd91b4969ff8ba6941bc85a960e" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.6.0" + +[[deps.Compat]] +deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] +git-tree-sha1 = "84aa74986c5b9b898b0d1acaf3258741ee64754f" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "2.1.0" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "0.5.2+0" + +[[deps.DataStructures]] +deps = ["InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "0809951a1774dc724da22d26e4289bbaab77809a" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.17.0" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[deps.DelimitedFiles]] +deps = ["Mmap"] +git-tree-sha1 = "19b1417ff479c07e523fcbf2fd735a3fde3d1ab3" +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" +version = "1.9.0" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.3" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.9.1+0" + +[[deps.LibGit2]] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "OpenSSL_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.3+0" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[deps.LightGraphs]] +deps = ["Arpack", "Base64", "CodecZlib", "DataStructures", "DelimitedFiles", "Distributed", "LinearAlgebra", "Markdown", "Random", "SharedArrays", "SimpleTraits", "SparseArrays", "Statistics", "Test"] +git-tree-sha1 = "e7e380a7c009019df1203bf400894aa04ee37ba0" +uuid = "093fc24a-ae57-5d10-9952-331d41423f4d" +version = "1.0.1" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[deps.MacroTools]] +deps = ["CSTParser", "Compat", "DataStructures", "Test", "Tokenize"] +git-tree-sha1 = "d6e9dedb8c92c3465575442da456aec15a89ff76" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.1" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.0.15+1" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2024.11.26" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.20+0" + +[[deps.OrderedCollections]] +deps = ["Random", "Serialization", "Test"] +git-tree-sha1 = "c4c13474d23c60d20a67b217f1d7f22a40edf8f1" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.1.0" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.8.0" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[deps.Random]] +deps = ["SHA", "Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[deps.SharedArrays]] +deps = ["Distributed", "Mmap", "Random", "Serialization"] +uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" + +[[deps.SimpleTraits]] +deps = ["InteractiveUtils", "MacroTools"] +git-tree-sha1 = "05bbf4484b975782e5e54bb0750f21f7f2f66171" +uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d" +version = "0.9.0" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[deps.Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "83850190e0f902ae1673d63ae349fc2a36dc6afb" +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.11" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "Pkg", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "5.10.1+0" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.0" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.Tokenize]] +git-tree-sha1 = "dfcdbbfb2d0370716c815cbd6f8a364efb6f42cf" +uuid = "0796e94c-ce3b-5d07-9a54-7f471281c624" +version = "0.5.6" + +[[deps.TranscodingStreams]] +deps = ["Random", "Test"] +git-tree-sha1 = "7c53c35547de1c5b9d46a4797cf6d8253807108c" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.9.5" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.12+3" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.1.1+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.48.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+0" From 739a64a0b1d4a8d79c703e99c73bd0433b32ebfa Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 31 Dec 2024 21:00:17 -0500 Subject: [PATCH 012/154] check source path is found when getting package_info (#4126) --- src/API.jl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/API.jl b/src/API.jl index 2218f248b3..834fbdcef5 100644 --- a/src/API.jl +++ b/src/API.jl @@ -63,6 +63,11 @@ function package_info(env::EnvCache, pkg::PackageSpec, entry::PackageEntry)::Pac git_source = pkg.repo.source === nothing ? nothing : isurl(pkg.repo.source::String) ? pkg.repo.source::String : Operations.project_rel_path(env, pkg.repo.source::String) + _source_path = Operations.source_path(env.manifest_file, pkg) + if _source_path === nothing + @debug "Manifest file $(env.manifest_file) contents:\n$(read(env.manifest_file, String))" + pkgerror("could not find source path for package $(err_rep(pkg)) based on $(env.manifest_file)") + end info = PackageInfo( name = pkg.name, version = pkg.version != VersionSpec() ? pkg.version : nothing, @@ -74,7 +79,7 @@ function package_info(env::EnvCache, pkg::PackageSpec, entry::PackageEntry)::Pac is_tracking_registry = Operations.is_tracking_registry(pkg), git_revision = pkg.repo.rev, git_source = git_source, - source = Operations.project_rel_path(env, Operations.source_path(env.manifest_file, pkg)), + source = Operations.project_rel_path(env, _source_path), dependencies = copy(entry.deps), #TODO is copy needed? ) return info From 8d3cf02e5e35913c89440c3b5c6678c1a806e975 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Wed, 1 Jan 2025 13:53:52 -0500 Subject: [PATCH 013/154] Fix OpenSSL_jll detection (#4127) --- test/new.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/new.jl b/test/new.jl index fee012dd3d..7e0a51d82b 100644 --- a/test/new.jl +++ b/test/new.jl @@ -967,7 +967,7 @@ end # Support julia versions before & after the MbedTLS > OpenSSL switch OpenSSL_pkgid = Base.PkgId(Base.UUID("458c3c95-2e84-50aa-8efc-19380b2a3a95"), "OpenSSL_jll") - manifest_to_use = if haskey(Base.loaded_modules, OpenSSL_pkgid) + manifest_to_use = if Base.is_stdlib(OpenSSL_pkgid) joinpath(tmp, "ShouldPreserveSemver", "Manifest_OpenSSL.toml") else joinpath(tmp, "ShouldPreserveSemver", "Manifest_MbedTLS.toml") From 1eb09be2ae808f5c4a50a421710df5881513bf7e Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 2 Jan 2025 13:08:23 +0100 Subject: [PATCH 014/154] slightly improve memory in resolver code (#4119) --- src/Resolve/graphtype.jl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Resolve/graphtype.jl b/src/Resolve/graphtype.jl index f2cffce50d..a019ae2d18 100644 --- a/src/Resolve/graphtype.jl +++ b/src/Resolve/graphtype.jl @@ -268,12 +268,17 @@ mutable struct Graph p1 == p0 && error("Package $(pkgID(pkgs[p0], uuid_to_name)) version $vn has a dependency with itself") # check conflicts instead of intersecting? # (intersecting is used by fixed packages though...) - req_p1 = get!(VersionSpec, req, p1) - req[p1] = req_p1 ∩ vs + req_p1 = get(req, p1, nothing) + if req_p1 == nothing + req[p1] = vs + else + req[p1] = req_p1 ∩ vs + end end # Translate the requirements into bit masks # Hot code, measure performance before changing req_msk = Dict{Int,BitVector}() + sizehint!(req_msk, length(req)) maybe_weak = haskey(compat_weak, uuid0) && haskey(compat_weak[uuid0], vn) for (p1, vs) in req pv = pvers[p1] From bc9fb21b1f2d72038491eff938673fc5fbc99445 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 10 Jan 2025 12:43:53 -0500 Subject: [PATCH 015/154] Fix JLL version fix fix (#4130) --- src/Operations.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Operations.jl b/src/Operations.jl index a4e2c77553..632fb454dc 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -507,6 +507,13 @@ function resolve_versions!(env::EnvCache, registries::Vector{Registry.RegistryIn old_v = get(jll_fix, uuid, nothing) # We only fixup a JLL if the old major/minor/patch matches the new major/minor/patch if old_v !== nothing && Base.thispatch(old_v) == Base.thispatch(vers_fix[uuid]) + new_v = vers_fix[uuid] + if old_v != new_v + compat_map[uuid][old_v] = compat_map[uuid][new_v] + # Note that we don't delete!(compat_map[uuid], old_v) because we want to keep the compat info around + # in case there's JLL version confusion between the sysimage pkgorigins version and manifest + # but that issue hasn't been fully specified, so keep it to be cautious + end vers_fix[uuid] = old_v end end From 2609a94116b0426afb4ab60c0e2ef2e6d628ea0e Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Thu, 16 Jan 2025 08:39:03 -0500 Subject: [PATCH 016/154] don't set up callbacks if using cli git (#4128) --- src/GitTools.jl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/GitTools.jl b/src/GitTools.jl index 02fae614ea..03cc08adff 100644 --- a/src/GitTools.jl +++ b/src/GitTools.jl @@ -97,16 +97,6 @@ function clone(io::IO, url, source_path; header=nothing, credentials=nothing, kw printpkgstyle(io, :Cloning, header === nothing ? "git-repo `$url`" : header) bar = MiniProgressBar(header = "Fetching:", color = Base.info_color()) fancyprint = can_fancyprint(io) - callbacks = if fancyprint - LibGit2.Callbacks( - :transfer_progress => ( - @cfunction(transfer_progress, Cint, (Ptr{LibGit2.TransferProgress}, Any)), - bar, - ) - ) - else - LibGit2.Callbacks() - end fancyprint && start_progress(io, bar) if credentials === nothing credentials = LibGit2.CachedCredentials() @@ -121,6 +111,16 @@ function clone(io::IO, url, source_path; header=nothing, credentials=nothing, kw end return LibGit2.GitRepo(source_path) else + callbacks = if fancyprint + LibGit2.Callbacks( + :transfer_progress => ( + @cfunction(transfer_progress, Cint, (Ptr{LibGit2.TransferProgress}, Any)), + bar, + ) + ) + else + LibGit2.Callbacks() + end mkpath(source_path) return LibGit2.clone(url, source_path; callbacks=callbacks, credentials=credentials, kwargs...) end From df1931a9310dc3f9f0d6c53b5f23ac15e47ee6e7 Mon Sep 17 00:00:00 2001 From: Rob Vermaas Date: Thu, 16 Jan 2025 17:06:36 +0100 Subject: [PATCH 017/154] Use copy_test_packages in some places, and ensure that the copied test package is user writable. This allows running the Pkg tests as part of Base.runtests of an installed Julia installation, where sources might be readonly. (#4136) --- test/extensions.jl | 5 ++--- test/new.jl | 10 ++++------ test/project_manifest.jl | 7 +++---- test/sources.jl | 5 ++--- test/utils.jl | 12 ++++++++++++ test/workspaces.jl | 5 ++--- 6 files changed, 25 insertions(+), 19 deletions(-) diff --git a/test/extensions.jl b/test/extensions.jl index f7d7ab26b9..05a2a6bac6 100644 --- a/test/extensions.jl +++ b/test/extensions.jl @@ -98,9 +98,8 @@ using UUIDs isolate(loaded_depot=false) do mktempdir() do dir Pkg.Registry.add("General") - path = joinpath(@__DIR__, "test_packages", "TestWeakDepProject") - cp(path, joinpath(dir, "TestWeakDepProject")) - Pkg.activate(joinpath(dir, "TestWeakDepProject")) + path = copy_test_package(dir, "TestWeakDepProject") + Pkg.activate(path) Pkg.resolve() @test Pkg.dependencies()[UUID("2ab3a3ac-af41-5b50-aa03-7779005ae688")].version == v"0.3.26" diff --git a/test/new.jl b/test/new.jl index 7e0a51d82b..46f91168f7 100644 --- a/test/new.jl +++ b/test/new.jl @@ -3224,9 +3224,8 @@ end temp_pkg_dir() do project_path @testset "test entryfile entries" begin mktempdir() do dir - path = abspath(joinpath(dirname(pathof(Pkg)), "../test", "test_packages", "ProjectPath")) - cp(path, joinpath(dir, "ProjectPath")) - cd(joinpath(dir, "ProjectPath")) do + path = copy_test_package(dir, "ProjectPath") + cd(path) do with_current_env() do Pkg.resolve() @test success(run(`$(Base.julia_cmd()) --startup-file=no --project -e 'using ProjectPath'`)) @@ -3238,9 +3237,8 @@ temp_pkg_dir() do project_path end @testset "test resolve with tree hash" begin mktempdir() do dir - path = abspath(joinpath(@__DIR__, "../test", "test_packages", "ResolveWithRev")) - cp(path, joinpath(dir, "ResolveWithRev")) - cd(joinpath(dir, "ResolveWithRev")) do + path = copy_test_package(dir, "ResolveWithRev") + cd(path) do with_current_env() do @test !isfile("Manifest.toml") @test !isdir(joinpath(DEPOT_PATH[1], "packages", "Example")) diff --git a/test/project_manifest.jl b/test/project_manifest.jl index ab6a6cc99f..41974ed612 100644 --- a/test/project_manifest.jl +++ b/test/project_manifest.jl @@ -7,9 +7,8 @@ using ..Utils temp_pkg_dir() do project_path @testset "test Project.toml manifest" begin mktempdir() do dir - path = abspath(joinpath(dirname(pathof(Pkg)), "../test", "test_packages", "monorepo")) - cp(path, joinpath(dir, "monorepo")) - cd(joinpath(dir, "monorepo")) do + path = copy_test_package(dir, "monorepo") + cd(path) do with_current_env() do Pkg.develop(path="packages/B") end @@ -60,4 +59,4 @@ temp_pkg_dir() do project_path end end -end # module \ No newline at end of file +end # module diff --git a/test/sources.jl b/test/sources.jl index 1497464aab..e17c40d6c3 100644 --- a/test/sources.jl +++ b/test/sources.jl @@ -7,9 +7,8 @@ using ..Utils temp_pkg_dir() do project_path @testset "test Project.toml [sources]" begin mktempdir() do dir - path = abspath(joinpath(dirname(pathof(Pkg)), "../test", "test_packages", "WithSources")) - cp(path, joinpath(dir, "WithSources")) - cd(joinpath(dir, "WithSources")) do + path = copy_test_package(dir, "WithSources") + cd(path) do with_current_env() do Pkg.resolve() @test !isempty(Pkg.project().sources["Example"]) diff --git a/test/utils.jl b/test/utils.jl index ee19cb9bcb..7bbc606ade 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -270,9 +270,21 @@ function git_init_package(tmp, path) return pkgpath end +function ensure_test_package_user_writable(dir) + for (root, _, files) in walkdir(dir) + chmod(root, filemode(root) | 0o200 | 0o100) + + for file in files + filepath = joinpath(root, file) + chmod(filepath, filemode(filepath) | 0o200) + end + end +end + function copy_test_package(tmpdir::String, name::String; use_pkg=true) target = joinpath(tmpdir, name) cp(joinpath(@__DIR__, "test_packages", name), target) + ensure_test_package_user_writable(target) use_pkg || return target # The known Pkg UUID, and whatever UUID we're currently using for testing diff --git a/test/workspaces.jl b/test/workspaces.jl index acb41be7bd..33b75d2baa 100644 --- a/test/workspaces.jl +++ b/test/workspaces.jl @@ -148,9 +148,8 @@ end @testset "test resolve with tree hash" begin mktempdir() do dir - path = abspath(joinpath(@__DIR__, "../test", "test_packages", "WorkspaceTestInstantiate")) - cp(path, joinpath(dir, "WorkspaceTestInstantiate")) - cd(joinpath(dir, "WorkspaceTestInstantiate")) do + path = copy_test_package(dir, "WorkspaceTestInstantiate") + cd(path) do with_current_env() do @test !isfile("Manifest.toml") @test !isfile("test/Manifest.toml") From 938e9b24eebf1bb19613e8941dc8700e68249578 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 24 Jan 2025 16:15:56 +0100 Subject: [PATCH 018/154] app support in Pkg (#3772) This provides initial support for defining "apps" in Julia which can be run from the terminal and installed via Pkg. --- docs/src/apps.md | 59 ++++ ext/REPLExt/completions.jl | 17 + src/Apps/Apps.jl | 387 ++++++++++++++++++++++ src/Operations.jl | 18 +- src/Pkg.jl | 1 + src/REPLMode/REPLMode.jl | 2 +- src/REPLMode/argument_parsers.jl | 9 + src/REPLMode/command_declarations.jl | 67 ++++ src/Types.jl | 13 +- src/manifest.jl | 25 ++ src/project.jl | 16 +- test/apps.jl | 41 +++ test/runtests.jl | 3 +- test/test_packages/Rot13.jl/Manifest.toml | 10 + test/test_packages/Rot13.jl/Project.toml | 6 + test/test_packages/Rot13.jl/src/Rot13.jl | 17 + 16 files changed, 677 insertions(+), 14 deletions(-) create mode 100644 docs/src/apps.md create mode 100644 src/Apps/Apps.jl create mode 100644 test/apps.jl create mode 100644 test/test_packages/Rot13.jl/Manifest.toml create mode 100644 test/test_packages/Rot13.jl/Project.toml create mode 100644 test/test_packages/Rot13.jl/src/Rot13.jl diff --git a/docs/src/apps.md b/docs/src/apps.md new file mode 100644 index 0000000000..1fccbb8fbd --- /dev/null +++ b/docs/src/apps.md @@ -0,0 +1,59 @@ +# [**?.** Apps](@id Apps) + +!!! note + The app support in Pkg is currently considered experimental and some functionality and API may change. + + Some inconveniences that can be encountered are: + - You need to manually make `~/.julia/bin` available on the PATH environment. + - The path to the julia executable used is the same as the one used to install the app. If this + julia installation gets removed, you might need to reinstall the app. + - You can only have one app installed per package. + +Apps are Julia packages that are intended to be run as a "standalone programs" (by e.g. typing the name of the app in the terminal possibly together with some arguments or flags/options). +This is in contrast to most Julia packages that are used as "libraries" and are loaded by other files or in the Julia REPL. + +## Creating a Julia app + +A Julia app is structured similar to a standard Julia library with the following additions: + +- A `@main` entry point in the package module (see the [Julia help on `@main`](https://docs.julialang.org/en/v1/manual/command-line-interface/#The-Main.main-entry-point) for details) +- An `[app]` section in the `Project.toml` file listing the executable names that the package provides. + +A very simple example of an app that prints the reversed input arguments would be: + +```julia +# src/MyReverseApp.jl +module MyReverseApp + +function (@main)(ARGS) + for arg in ARGS + print(stdout, reverse(arg), " ") + end + return +end + +end # module +``` + +```toml +# Project.toml + +# standard fields here + +[apps] +reverse = {} +``` +The empty table `{}` is to allow for giving metadata about the app but it is currently unused. + +After installing this app one could run: + +``` +$ reverse some input string +emos tupni gnirts +``` + +directly in the terminal. + +## Installing Julia apps + +The installation of Julia apps are similar to installing julia libraries but instead of using e.g. `Pkg.add` or `pkg> add` one uses `Pkg.Apps.add` or `pkg> app add` (`develop` is also available). diff --git a/ext/REPLExt/completions.jl b/ext/REPLExt/completions.jl index eca5e11218..2a5e625116 100644 --- a/ext/REPLExt/completions.jl +++ b/ext/REPLExt/completions.jl @@ -156,6 +156,23 @@ function complete_add_dev(options, partial, i1, i2; hint::Bool) return comps, idx, !isempty(comps) end +# TODO: Move +import Pkg: Operations, Types, Apps +function complete_installed_apps(options, partial; hint) + manifest = try + Types.read_manifest(joinpath(Apps.app_env_folder(), "AppManifest.toml")) + catch err + err isa PkgError || rethrow() + return String[] + end + apps = String[] + for (uuid, entry) in manifest.deps + append!(apps, keys(entry.apps)) + push!(apps, entry.name) + end + return unique!(apps) +end + ######################## # COMPLETION INTERFACE # ######################## diff --git a/src/Apps/Apps.jl b/src/Apps/Apps.jl new file mode 100644 index 0000000000..3661c3c2e6 --- /dev/null +++ b/src/Apps/Apps.jl @@ -0,0 +1,387 @@ +module Apps + +using Pkg +using Pkg.Versions +using Pkg.Types: AppInfo, PackageSpec, Context, EnvCache, PackageEntry, Manifest, handle_repo_add!, handle_repo_develop!, write_manifest, write_project, + pkgerror, projectfile_path, manifestfile_path +using Pkg.Operations: print_single, source_path, update_package_add +using Pkg.API: handle_package_input! +using TOML, UUIDs +import Pkg.Registry + +app_env_folder() = joinpath(first(DEPOT_PATH), "environments", "apps") +app_manifest_file() = joinpath(app_env_folder(), "AppManifest.toml") +julia_bin_path() = joinpath(first(DEPOT_PATH), "bin") + +app_context() = Context(env=EnvCache(joinpath(app_env_folder(), "Project.toml"))) + + +function rm_shim(name; kwargs...) + Base.rm(joinpath(julia_bin_path(), name * (Sys.iswindows() ? ".bat" : "")); kwargs...) +end + +function get_project(sourcepath) + project_file = projectfile_path(sourcepath) + + isfile(project_file) || error("Project file not found: $project_file") + + project = Pkg.Types.read_project(project_file) + isempty(project.apps) && error("No apps found in Project.toml for package $(project.name) at version $(project.version)") + return project +end + + +function overwrite_file_if_different(file, content) + if !isfile(file) || read(file, String) != content + mkpath(dirname(file)) + write(file, content) + end +end + +function get_max_version_register(pkg::PackageSpec, regs) + max_v = nothing + tree_hash = nothing + for reg in regs + if get(reg, pkg.uuid, nothing) !== nothing + reg_pkg = get(reg, pkg.uuid, nothing) + reg_pkg === nothing && continue + pkg_info = Registry.registry_info(reg_pkg) + for (version, info) in pkg_info.version_info + info.yanked && continue + if pkg.version isa VersionNumber + pkg.version == version || continue + else + version in pkg.version || continue + end + if max_v === nothing || version > max_v + max_v = version + tree_hash = info.git_tree_sha1 + end + end + end + end + if max_v === nothing + error("Suitable package version for $(pkg.name) not found in any registries.") + end + return (max_v, tree_hash) +end + + +################## +# Main Functions # +################## + +function _resolve(manifest::Manifest, pkgname=nothing) + for (uuid, pkg) in manifest.deps + if pkgname !== nothing && pkg.name !== pkgname + continue + end + + projectfile = joinpath(app_env_folder(), pkg.name, "Project.toml") + sourcepath = source_path(app_manifest_file(), pkg) + + # TODO: Add support for existing manifest + # Create a manifest with the manifest entry + Pkg.activate(joinpath(app_env_folder(), pkg.name)) do + ctx = Context() + if isempty(ctx.env.project.deps) + ctx.env.project.deps[pkg.name] = uuid + end + if isempty(ctx.env.manifest) + ctx.env.manifest.deps[uuid] = pkg + Pkg.resolve(ctx) + else + Pkg.instantiate(ctx) + end + end + + # TODO: Julia path + generate_shims_for_apps(pkg.name, pkg.apps, dirname(projectfile), joinpath(Sys.BINDIR, "julia")) + end + + write_manifest(manifest, app_manifest_file()) +end + + +function add(pkg::Vector{PackageSpec}) + for p in pkg + add(p) + end +end + + +function add(pkg::PackageSpec) + handle_package_input!(pkg) + + ctx = app_context() + manifest = ctx.env.manifest + new = false + + # Download package + if pkg.repo.source !== nothing || pkg.repo.rev !== nothing + entry = Pkg.API.manifest_info(ctx.env.manifest, pkg.uuid) + pkg = update_package_add(ctx, pkg, entry, false) + new = handle_repo_add!(ctx, pkg) + else + pkgs = [pkg] + Pkg.Operations.registry_resolve!(ctx.registries, pkgs) + Pkg.Operations.ensure_resolved(ctx, manifest, pkgs, registry=true) + + pkg.version, pkg.tree_hash = get_max_version_register(pkg, ctx.registries) + + new = Pkg.Operations.download_source(ctx, pkgs) + end + + sourcepath = source_path(ctx.env.manifest_file, pkg) + project = get_project(sourcepath) + # TODO: Wrong if package itself has a sourcepath? + + entry = PackageEntry(;apps = project.apps, name = pkg.name, version = project.version, tree_hash = pkg.tree_hash, path = pkg.path, repo = pkg.repo, uuid=pkg.uuid) + manifest.deps[pkg.uuid] = entry + + _resolve(manifest, pkg.name) + precompile(pkg.name) + + @info "For package: $(pkg.name) installed apps $(join(keys(project.apps), ","))" +end + +function develop(pkg::Vector{PackageSpec}) + for p in pkg + develop(p) + end +end + +function develop(pkg::PackageSpec) + if pkg.path !== nothing + pkg.path == abspath(pkg.path) + end + handle_package_input!(pkg) + ctx = app_context() + handle_repo_develop!(ctx, pkg, #=shared =# true) + sourcepath = abspath(source_path(ctx.env.manifest_file, pkg)) + project = get_project(sourcepath) + + # Seems like the `.repo.source` field is not cleared. + # At least repo-url is still in the manifest after doing a dev with a path + # Figure out why for normal dev this is not needed. + # XXX: Why needed? + if pkg.path !== nothing + pkg.repo.source = nothing + end + + + entry = PackageEntry(;apps = project.apps, name = pkg.name, version = project.version, tree_hash = pkg.tree_hash, path = sourcepath, repo = pkg.repo, uuid=pkg.uuid) + manifest = ctx.env.manifest + manifest.deps[pkg.uuid] = entry + + _resolve(manifest, pkg.name) + precompile(pkg.name) + @info "For package: $(pkg.name) installed apps: $(join(keys(project.apps), ","))" +end + +function status(pkgs_or_apps::Vector) + if isempty(pkgs_or_apps) + status() + else + for pkg_or_app in pkgs_or_apps + if pkg_or_app isa String + pkg_or_app = PackageSpec(pkg_or_app) + end + status(pkg_or_app) + end + end +end + +function status(pkg_or_app::Union{PackageSpec, Nothing}=nothing) + # TODO: Sort. + pkg_or_app = pkg_or_app === nothing ? nothing : pkg_or_app.name + manifest = Pkg.Types.read_manifest(joinpath(app_env_folder(), "AppManifest.toml")) + deps = Pkg.Operations.load_manifest_deps(manifest) + + is_pkg = pkg_or_app !== nothing && any(dep -> dep.name == pkg_or_app, values(manifest.deps)) + + for dep in deps + info = manifest.deps[dep.uuid] + if is_pkg && dep.name !== pkg_or_app + continue + end + if !is_pkg && pkg_or_app !== nothing + if !(pkg_or_app in keys(info.apps)) + continue + end + end + + printstyled("[", string(dep.uuid)[1:8], "] "; color = :light_black) + print_single(stdout, dep) + println() + for (appname, appinfo) in info.apps + if !is_pkg && pkg_or_app !== nothing && appname !== pkg_or_app + continue + end + julia_cmd = contractuser(appinfo.julia_command) + printstyled(" $(appname)", color=:green) + printstyled(" $(julia_cmd) \n", color=:gray) + end + end +end + +function precompile(pkg::Union{Nothing, String}=nothing) + manifest = Pkg.Types.read_manifest(joinpath(app_env_folder(), "AppManifest.toml")) + deps = Pkg.Operations.load_manifest_deps(manifest) + for dep in deps + # TODO: Parallel app compilation..? + info = manifest.deps[dep.uuid] + if pkg !== nothing && info.name !== pkg + continue + end + Pkg.activate(joinpath(app_env_folder(), info.name)) do + Pkg.instantiate() + Pkg.precompile() + end + end +end + + +function require_not_empty(pkgs, f::Symbol) + pkgs === nothing && return + isempty(pkgs) && pkgerror("app $f requires at least one package") +end + +rm(pkgs_or_apps::String) = rm([pkgs_or_apps]) +function rm(pkgs_or_apps::Union{Vector, Nothing}) + if pkgs_or_apps === nothing + rm(nothing) + else + for pkg_or_app in pkgs_or_apps + if pkg_or_app isa String + pkg_or_app = PackageSpec(pkg_or_app) + end + rm(pkg_or_app) + end + end +end + +function rm(pkg_or_app::Union{PackageSpec, Nothing}=nothing) + pkg_or_app = pkg_or_app === nothing ? nothing : pkg_or_app.name + + require_not_empty(pkg_or_app, :rm) + + manifest = Pkg.Types.read_manifest(joinpath(app_env_folder(), "AppManifest.toml")) + dep_idx = findfirst(dep -> dep.name == pkg_or_app, manifest.deps) + if dep_idx !== nothing + dep = manifest.deps[dep_idx] + @info "Deleting all apps for package $(dep.name)" + delete!(manifest.deps, dep.uuid) + for (appname, appinfo) in dep.apps + @info "Deleted $(appname)" + rm_shim(appname; force=true) + end + if dep.path === nothing + Base.rm(joinpath(app_env_folder(), dep.name); recursive=true) + end + else + for (uuid, pkg) in manifest.deps + app_idx = findfirst(app -> app.name == pkg_or_app, pkg.apps) + if app_idx !== nothing + app = pkg.apps[app_idx] + @info "Deleted app $(app.name)" + delete!(pkg.apps, app.name) + rm_shim(app.name; force=true) + end + if isempty(pkg.apps) + delete!(manifest.deps, uuid) + Base.rm(joinpath(app_env_folder(), pkg.name); recursive=true) + end + end + end + + Pkg.Types.write_manifest(manifest, app_manifest_file()) + return +end + +for f in (:develop, :add) + @eval begin + $f(pkg::Union{AbstractString, PackageSpec}; kwargs...) = $f([pkg]; kwargs...) + $f(pkgs::Vector{<:AbstractString}; kwargs...) = $f([PackageSpec(pkg) for pkg in pkgs]; kwargs...) + function $f(; name::Union{Nothing,AbstractString}=nothing, uuid::Union{Nothing,String,UUID}=nothing, + version::Union{VersionNumber, String, VersionSpec, Nothing}=nothing, + url=nothing, rev=nothing, path=nothing, subdir=nothing, kwargs...) + pkg = PackageSpec(; name, uuid, version, url, rev, path, subdir) + if all(isnothing, [name,uuid,version,url,rev,path,subdir]) + $f(PackageSpec[]; kwargs...) + else + $f(pkg; kwargs...) + end + end + function $f(pkgs::Vector{<:NamedTuple}; kwargs...) + $f([PackageSpec(;pkg...) for pkg in pkgs]; kwargs...) + end + end +end + + +######### +# Shims # +######### + +const SHIM_COMMENT = Sys.iswindows() ? "REM " : "#" +const SHIM_VERSION = 1.0 +const SHIM_HEADER = """$SHIM_COMMENT This file is generated by the Julia package manager. + $SHIM_COMMENT Shim version: $SHIM_VERSION""" + + +function generate_shims_for_apps(pkgname, apps, env, julia) + for (_, app) in apps + generate_shim(pkgname, app, env, julia) + end +end + +function generate_shim(pkgname, app::AppInfo, env, julia) + filename = app.name * (Sys.iswindows() ? ".bat" : "") + julia_bin_filename = joinpath(julia_bin_path(), filename) + mkpath(dirname(filename)) + content = if Sys.iswindows() + windows_shim(pkgname, julia, env) + else + bash_shim(pkgname, julia, env) + end + overwrite_file_if_different(julia_bin_filename, content) + if Sys.isunix() + chmod(julia_bin_filename, 0o755) + end +end + + +function bash_shim(pkgname, julia::String, env) + return """ + #!/usr/bin/env bash + + $SHIM_HEADER + + export JULIA_LOAD_PATH=$(repr(env)) + export JULIA_DEPOT_PATH=$(repr(join(DEPOT_PATH, ':'))) + exec $julia \\ + --startup-file=no \\ + -m $(pkgname) \\ + "\$@" + """ +end + +function windows_shim(pkgname, julia::String, env) + return """ + @echo off + + $SHIM_HEADER + + setlocal + set JULIA_LOAD_PATH=$env + set JULIA_DEPOT_PATH=$(join(DEPOT_PATH, ';')) + + $julia ^ + --startup-file=no ^ + -m $(pkgname) ^ + %* + """ +end + +end diff --git a/src/Operations.jl b/src/Operations.jl index 632fb454dc..4c47468a26 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -1020,9 +1020,11 @@ function find_urls(registries::Vector{Registry.RegistryInstance}, uuid::UUID) end -function download_source(ctx::Context; readonly=true) - pkgs_to_install = NamedTuple{(:pkg, :urls, :path), Tuple{PackageEntry, Set{String}, String}}[] - for pkg in values(ctx.env.manifest) +download_source(ctx::Context; readonly=true) = download_source(ctx, values(ctx.env.manifest); readonly) + +function download_source(ctx::Context, pkgs; readonly=true) + pkgs_to_install = NamedTuple{(:pkg, :urls, :path), Tuple{eltype(pkgs), Set{String}, String}}[] + for pkg in pkgs tracking_registered_version(pkg, ctx.julia_version) || continue path = source_path(ctx.env.manifest_file, pkg, ctx.julia_version) path === nothing && continue @@ -1107,7 +1109,7 @@ function download_source(ctx::Context; readonly=true) fancyprint = can_fancyprint(ctx.io) try for i in 1:length(pkgs_to_install) - pkg::PackageEntry, exc_or_success, bt_or_pathurls = take!(results) + pkg::eltype(pkgs), exc_or_success, bt_or_pathurls = take!(results) exc_or_success isa Exception && pkgerror("Error when installing package $(pkg.name):\n", sprint(Base.showerror, exc_or_success, bt_or_pathurls)) success, (urls, path) = exc_or_success, bt_or_pathurls @@ -1138,7 +1140,6 @@ function download_source(ctx::Context; readonly=true) # Use LibGit2 to download any remaining packages # ################################################## for (pkg, urls, path) in missed_packages - uuid = pkg.uuid install_git(ctx.io, pkg.uuid, pkg.name, pkg.tree_hash, urls, path) readonly && set_readonly(path) vstr = if pkg.version !== nothing @@ -1491,8 +1492,8 @@ function rm(ctx::Context, pkgs::Vector{PackageSpec}; mode::PackageMode) show_update(ctx.env, ctx.registries; io=ctx.io) end -update_package_add(ctx::Context, pkg::PackageSpec, ::Nothing, source_path, source_repo, is_dep::Bool) = pkg -function update_package_add(ctx::Context, pkg::PackageSpec, entry::PackageEntry, source_path, source_repo, is_dep::Bool) +update_package_add(ctx::Context, pkg::PackageSpec, ::Nothing, is_dep::Bool) = pkg +function update_package_add(ctx::Context, pkg::PackageSpec, entry::PackageEntry, is_dep::Bool) if entry.pinned if pkg.version == VersionSpec() println(ctx.io, "`$(pkg.name)` is pinned at `v$(entry.version)`: maintaining pinned version") @@ -1636,8 +1637,7 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}, new_git=Set{UUID}(); delete!(ctx.env.project.weakdeps, pkg.name) entry = manifest_info(ctx.env.manifest, pkg.uuid) is_dep = any(uuid -> uuid == pkg.uuid, [uuid for (name, uuid) in ctx.env.project.deps]) - source_path, source_repo = get_path_repo(ctx.env.project, pkg.name) - pkgs[i] = update_package_add(ctx, pkg, entry, source_path, source_repo, is_dep) + pkgs[i] = update_package_add(ctx, pkg, entry, is_dep) end names = (p.name for p in pkgs) diff --git a/src/Pkg.jl b/src/Pkg.jl index d6260607dd..1c29d61c01 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -72,6 +72,7 @@ include("BinaryPlatforms_compat.jl") include("Artifacts.jl") include("Operations.jl") include("API.jl") +include("Apps/Apps.jl") include("REPLMode/REPLMode.jl") import .REPLMode: @pkg_str diff --git a/src/REPLMode/REPLMode.jl b/src/REPLMode/REPLMode.jl index aba9ef4dd8..2abd52e5cc 100644 --- a/src/REPLMode/REPLMode.jl +++ b/src/REPLMode/REPLMode.jl @@ -7,7 +7,7 @@ module REPLMode using Markdown, UUIDs, Dates import ..casesensitive_isdir, ..OFFLINE_MODE, ..linewrap, ..pathrepr -using ..Types, ..Operations, ..API, ..Registry, ..Resolve +using ..Types, ..Operations, ..API, ..Registry, ..Resolve, ..Apps import ..stdout_f, ..stderr_f diff --git a/src/REPLMode/argument_parsers.jl b/src/REPLMode/argument_parsers.jl index c0f284a4b0..5d909e91ba 100644 --- a/src/REPLMode/argument_parsers.jl +++ b/src/REPLMode/argument_parsers.jl @@ -204,6 +204,15 @@ function parse_registry(word::AbstractString; add=false)::RegistrySpec return registry end +# +# # Apps +# +function parse_app_add(raw_args::Vector{QString}, options) + return parse_package(raw_args, options; add_or_dev=true) +end + + + # # # Other # diff --git a/src/REPLMode/command_declarations.jl b/src/REPLMode/command_declarations.jl index cb00dfb260..a9d9cc3304 100644 --- a/src/REPLMode/command_declarations.jl +++ b/src/REPLMode/command_declarations.jl @@ -582,4 +582,71 @@ pkg> registry status """, ] ], #registry +"app" => CommandDeclaration[ + PSA[:name => "status", + :short_name => "st", + :api => Apps.status, + :should_splat => false, + :arg_count => 0 => Inf, + :arg_parser => parse_package, + :completions => :complete_installed_apps, + :description => "show status of apps", + :help => md""" + show status of apps + """ +], +PSA[:name => "add", + :api => Apps.add, + :should_splat => false, + :arg_count => 0 => Inf, + :arg_parser => parse_app_add, + :completions => :complete_add_dev, + :description => "add app", + :help => md""" + app add pkg + +Adds the apps for packages `pkg...` or apps `app...`. +``` +""", +], +PSA[:name => "remove", + :short_name => "rm", + :api => Apps.rm, + :should_splat => false, + :arg_count => 0 => Inf, + :arg_parser => parse_package, + :completions => :complete_installed_apps, + :description => "remove packages from project or manifest", + :help => md""" + app [rm|remove] pkg ... + app [rm|remove] app ... + + Remove the apps for package `pkg`. + """ +], +PSA[:name => "develop", + :short_name => "dev", + :api => Apps.develop, + :should_splat => false, + :arg_count => 1 => Inf, + :arg_parser => (x,y) -> parse_package(x,y; add_or_dev=true), + :completions => :complete_add_dev, + :description => "develop a package and install all the apps in it", + :help => md""" + app [dev|develop] pkg[=uuid] ... + app [dev|develop] path + +Same as `develop` but also installs all the apps in the package. +This allows one to edit their app and have the changes immediately be reflected in the app. + +**Examples** +```jl +pkg> app develop Example +pkg> app develop https://github.com/JuliaLang/Example.jl +pkg> app develop ~/mypackages/Example +pkg> app develop --local Example +``` +""" +], # app +] ] #command_declarations diff --git a/src/Types.jl b/src/Types.jl index d9c8741b27..4d7a6f4b24 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -179,7 +179,7 @@ function projectfile_path(env_path::String; strict=false) end function manifestfile_path(env_path::String; strict=false) - for name in Base.manifest_names + for name in (Base.manifest_names..., "AppManifest.toml") maybe_file = joinpath(env_path, name) isfile(maybe_file) && return maybe_file end @@ -233,6 +233,12 @@ end Base.:(==)(t1::Compat, t2::Compat) = t1.val == t2.val Base.hash(t::Compat, h::UInt) = hash(t.val, h) +struct AppInfo + name::String + julia_command::Union{String, Nothing} + julia_version::Union{VersionNumber, Nothing} + other::Dict{String,Any} +end Base.@kwdef mutable struct Project other::Dict{String,Any} = Dict{String,Any}() # Fields @@ -251,6 +257,7 @@ Base.@kwdef mutable struct Project exts::Dict{String,Union{Vector{String}, String}} = Dict{String,String}() extras::Dict{String,UUID} = Dict{String,UUID}() targets::Dict{String,Vector{String}} = Dict{String,Vector{String}}() + apps::Dict{String, AppInfo} = Dict{String, AppInfo}() compat::Dict{String,Compat} = Dict{String,Compat}() sources::Dict{String,Dict{String, String}} = Dict{String,Dict{String, String}}() workspace::Dict{String, Any} = Dict{String, Any}() @@ -272,6 +279,7 @@ Base.@kwdef mutable struct PackageEntry weakdeps::Dict{String,UUID} = Dict{String,UUID}() exts::Dict{String,Union{Vector{String}, String}} = Dict{String,String}() uuid::Union{Nothing, UUID} = nothing + apps::Dict{String, AppInfo} = Dict{String, AppInfo}() # used by AppManifest.toml other::Union{Dict,Nothing} = nothing end Base.:(==)(t1::PackageEntry, t2::PackageEntry) = t1.name == t2.name && @@ -284,7 +292,8 @@ Base.:(==)(t1::PackageEntry, t2::PackageEntry) = t1.name == t2.name && t1.deps == t2.deps && t1.weakdeps == t2.weakdeps && t1.exts == t2.exts && - t1.uuid == t2.uuid + t1.uuid == t2.uuid && + t1.apps == t2.apps # omits `other` Base.hash(x::PackageEntry, h::UInt) = foldr(hash, [x.name, x.version, x.path, x.entryfile, x.pinned, x.repo, x.tree_hash, x.deps, x.weakdeps, x.exts, x.uuid], init=h) # omits `other` diff --git a/src/manifest.jl b/src/manifest.jl index db04bdbe7f..b8042ff312 100644 --- a/src/manifest.jl +++ b/src/manifest.jl @@ -81,6 +81,20 @@ function read_deps(raw::Dict{String, Any})::Dict{String,UUID} return deps end +read_apps(::Nothing) = Dict{String, AppInfo}() +read_apps(::Any) = pkgerror("Expected `apps` field to be a Dict") +function read_apps(apps::Dict) + appinfos = Dict{String, AppInfo}() + for (appname, app) in apps + appinfo = AppInfo(appname::String, + app["julia_command"]::String, + VersionNumber(app["julia_version"]::String), + app) + appinfos[appinfo.name] = appinfo + end + return appinfos +end + struct Stage1 uuid::UUID entry::PackageEntry @@ -182,6 +196,7 @@ function Manifest(raw::Dict{String, Any}, f_or_io::Union{String, IO})::Manifest entry.uuid = uuid deps = read_deps(get(info::Dict, "deps", nothing)::Union{Nothing, Dict{String, Any}, Vector{String}}) weakdeps = read_deps(get(info::Dict, "weakdeps", nothing)::Union{Nothing, Dict{String, Any}, Vector{String}}) + entry.apps = read_apps(get(info::Dict, "apps", nothing)::Union{Nothing, Dict{String, Any}}) entry.exts = get(Dict{String, String}, info, "extensions") catch # TODO: Should probably not unconditionally log something @@ -310,6 +325,15 @@ function destructure(manifest::Manifest)::Dict if !isempty(entry.exts) entry!(new_entry, "extensions", entry.exts) end + + if !isempty(entry.apps) + new_entry["apps"] = Dict{String,Any}() + for (appname, appinfo) in entry.apps + julia_command = @something appinfo.julia_command joinpath(Sys.BINDIR, "julia" * (Sys.iswindows() ? ".exe" : "")) + julia_version = @something appinfo.julia_version VERSION + new_entry["apps"][appname] = Dict{String,Any}("julia_command" => julia_command, "julia_version" => julia_version) + end + end if manifest.manifest_format.major == 1 push!(get!(raw, entry.name, Dict{String,Any}[]), new_entry) elseif manifest.manifest_format.major == 2 @@ -344,6 +368,7 @@ function write_manifest(io::IO, raw_manifest::Dict) end function write_manifest(raw_manifest::Dict, manifest_file::AbstractString) str = sprint(write_manifest, raw_manifest) + mkpath(dirname(manifest_file)) write(manifest_file, str) end diff --git a/src/project.jl b/src/project.jl index 1b559898c2..a856ddafe0 100644 --- a/src/project.jl +++ b/src/project.jl @@ -74,6 +74,19 @@ end read_project_targets(raw, project::Project) = pkgerror("Expected `targets` section to be a key-value list") +read_project_apps(::Nothing, project::Project) = Dict{String,Any}() +function read_project_apps(raw::Dict{String,Any}, project::Project) + other = raw + appinfos = Dict{String,AppInfo}() + for (name, info) in raw + info isa Dict{String,Any} || pkgerror(""" + Expected value for app `$name` to be a dictionary. + """) + appinfos[name] = AppInfo(name, nothing, nothing, other) + end + return appinfos +end + read_project_compat(::Nothing, project::Project) = Dict{String,Compat}() function read_project_compat(raw::Dict{String,Any}, project::Project) compat = Dict{String,Compat}() @@ -196,6 +209,7 @@ function Project(raw::Dict; file=nothing) project.compat = read_project_compat(get(raw, "compat", nothing), project) project.targets = read_project_targets(get(raw, "targets", nothing), project) project.workspace = read_project_workspace(get(raw, "workspace", nothing), project) + project.apps = read_project_apps(get(raw, "apps", nothing), project) # Handle deps in both [deps] and [weakdeps] project._deps_weak = Dict(intersect(project.deps, project.weakdeps)) @@ -261,7 +275,6 @@ project_key_order(key::String) = something(findfirst(x -> x == key, _project_key_order), length(_project_key_order) + 1) function write_project(env::EnvCache) - mkpath(dirname(env.project_file)) write_project(env.project, env.project_file) end write_project(project::Project, project_file::AbstractString) = @@ -282,5 +295,6 @@ function write_project(io::IO, project::Dict) end function write_project(project::Dict, project_file::AbstractString) str = sprint(write_project, project) + mkpath(dirname(project_file)) write(project_file, str) end diff --git a/test/apps.jl b/test/apps.jl new file mode 100644 index 0000000000..f47c1a039a --- /dev/null +++ b/test/apps.jl @@ -0,0 +1,41 @@ +module AppsTests + +import ..Pkg # ensure we are using the correct Pkg +using ..Utils + +using Test + +@testset "Apps" begin + +isolate(loaded_depot=true) do + sep = Sys.iswindows() ? ';' : ':' + Pkg.Apps.develop(path=joinpath(@__DIR__, "test_packages", "Rot13.jl")) + current_path = ENV["PATH"] + exename = Sys.iswindows() ? "rot13.bat" : "rot13" + withenv("PATH" => string(joinpath(first(DEPOT_PATH), "bin"), sep, current_path)) do + @test contains(Sys.which("$exename"), first(DEPOT_PATH)) + @test read(`$exename test`, String) == "grfg\n" + Pkg.Apps.rm("Rot13") + @test Sys.which(exename) == nothing + end +end + +isolate(loaded_depot=true) do + mktempdir() do tmpdir + sep = Sys.iswindows() ? ';' : ':' + path = git_init_package(tmpdir, joinpath(@__DIR__, "test_packages", "Rot13.jl")) + Pkg.Apps.add(path=path) + exename = Sys.iswindows() ? "rot13.bat" : "rot13" + current_path = ENV["PATH"] + withenv("PATH" => string(joinpath(first(DEPOT_PATH), "bin"), sep, current_path)) do + @test contains(Sys.which(exename), first(DEPOT_PATH)) + @test read(`$exename test`, String) == "grfg\n" + Pkg.Apps.rm("Rot13") + @test Sys.which(exename) == nothing + end + end +end + +end + +end # module diff --git a/test/runtests.jl b/test/runtests.jl index bb4a0b86e8..f912ff06b1 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -87,7 +87,8 @@ Logging.with_logger((islogging || Pkg.DEFAULT_IO[] == devnull) ? Logging.Console "manifests.jl", "project_manifest.jl", "sources.jl", - "workspaces.jl" + "workspaces.jl", + "apps.jl", ] @info "==== Testing `test/$f`" flush(Pkg.DEFAULT_IO[]) diff --git a/test/test_packages/Rot13.jl/Manifest.toml b/test/test_packages/Rot13.jl/Manifest.toml new file mode 100644 index 0000000000..bbef702fb7 --- /dev/null +++ b/test/test_packages/Rot13.jl/Manifest.toml @@ -0,0 +1,10 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.12.0-DEV" +manifest_format = "2.0" +project_hash = "2610b29b73f9f9432fb181a7f9f7c5c9e3de5557" + +[[deps.Rot13]] +path = "." +uuid = "43ef800a-eac4-47f4-949b-25107b932e8f" +version = "0.1.0" diff --git a/test/test_packages/Rot13.jl/Project.toml b/test/test_packages/Rot13.jl/Project.toml new file mode 100644 index 0000000000..5aca368746 --- /dev/null +++ b/test/test_packages/Rot13.jl/Project.toml @@ -0,0 +1,6 @@ +name = "Rot13" +uuid = "43ef800a-eac4-47f4-949b-25107b932e8f" +version = "0.1.0" + +[apps] +rot13 = {} diff --git a/test/test_packages/Rot13.jl/src/Rot13.jl b/test/test_packages/Rot13.jl/src/Rot13.jl new file mode 100644 index 0000000000..facbb92527 --- /dev/null +++ b/test/test_packages/Rot13.jl/src/Rot13.jl @@ -0,0 +1,17 @@ +module Rot13 + +function rot13(c::Char) + shft = islowercase(c) ? 'a' : 'A' + isletter(c) ? c = shft + (c - shft + 13) % 26 : c +end + +rot13(str::AbstractString) = map(rot13, str) + +function (@main)(ARGS) + for arg in ARGS + println(rot13(arg)) + end + return 0 +end + +end # module Rot13 From a3626bf29431e1a0d8856890f2169008b55df8cd Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 28 Jan 2025 15:07:26 +0100 Subject: [PATCH 019/154] add `PREV_ENV_PATH` to globals getting reset and move the reset to before precompilation is finished instead of `__init__` (#4142) Co-authored-by: KristofferC --- src/Pkg.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Pkg.jl b/src/Pkg.jl index 1c29d61c01..9d8d5aa36a 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -783,8 +783,6 @@ This function can be used in tests to verify that the manifest is synchronized w const is_manifest_current = API.is_manifest_current function __init__() - DEFAULT_IO[] = nothing - Pkg.UPDATED_REGISTRY_THIS_SESSION[] = false if !isassigned(Base.PKG_PRECOMPILE_HOOK) # allows Base to use Pkg.precompile during loading # disable via `Base.PKG_PRECOMPILE_HOOK[] = Returns(nothing)` @@ -867,4 +865,9 @@ end include("precompile.jl") +# Reset globals that might have been mutated during precompilation. +DEFAULT_IO[] = nothing +Pkg.UPDATED_REGISTRY_THIS_SESSION[] = false +PREV_ENV_PATH[] = "" + end # module From ecdf6aa38f76bdf52505e2c418c0a2e43d3643bb Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 28 Jan 2025 15:11:31 +0100 Subject: [PATCH 020/154] rename rot13 app in test to avoid name conflicts on systems where such a binary is already installed (#4143) Co-authored-by: KristofferC --- test/apps.jl | 4 ++-- test/test_packages/Rot13.jl/Project.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/apps.jl b/test/apps.jl index f47c1a039a..64e2b264a0 100644 --- a/test/apps.jl +++ b/test/apps.jl @@ -11,7 +11,7 @@ isolate(loaded_depot=true) do sep = Sys.iswindows() ? ';' : ':' Pkg.Apps.develop(path=joinpath(@__DIR__, "test_packages", "Rot13.jl")) current_path = ENV["PATH"] - exename = Sys.iswindows() ? "rot13.bat" : "rot13" + exename = Sys.iswindows() ? "juliarot13.bat" : "juliarot13" withenv("PATH" => string(joinpath(first(DEPOT_PATH), "bin"), sep, current_path)) do @test contains(Sys.which("$exename"), first(DEPOT_PATH)) @test read(`$exename test`, String) == "grfg\n" @@ -25,7 +25,7 @@ isolate(loaded_depot=true) do sep = Sys.iswindows() ? ';' : ':' path = git_init_package(tmpdir, joinpath(@__DIR__, "test_packages", "Rot13.jl")) Pkg.Apps.add(path=path) - exename = Sys.iswindows() ? "rot13.bat" : "rot13" + exename = Sys.iswindows() ? "juliarot13.bat" : "juliarot13" current_path = ENV["PATH"] withenv("PATH" => string(joinpath(first(DEPOT_PATH), "bin"), sep, current_path)) do @test contains(Sys.which(exename), first(DEPOT_PATH)) diff --git a/test/test_packages/Rot13.jl/Project.toml b/test/test_packages/Rot13.jl/Project.toml index 5aca368746..fe3ae6389c 100644 --- a/test/test_packages/Rot13.jl/Project.toml +++ b/test/test_packages/Rot13.jl/Project.toml @@ -3,4 +3,4 @@ uuid = "43ef800a-eac4-47f4-949b-25107b932e8f" version = "0.1.0" [apps] -rot13 = {} +juliarot13 = {} From 6091533bc93c722591d3e7ee53d26012cb4d5aa4 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 29 Jan 2025 15:11:38 +0100 Subject: [PATCH 021/154] fix ambiguity in apps `rm` (#4144) --- src/Apps/Apps.jl | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/Apps/Apps.jl b/src/Apps/Apps.jl index 3661c3c2e6..f8f9b49140 100644 --- a/src/Apps/Apps.jl +++ b/src/Apps/Apps.jl @@ -243,21 +243,19 @@ end function require_not_empty(pkgs, f::Symbol) - pkgs === nothing && return - isempty(pkgs) && pkgerror("app $f requires at least one package") + + if pkgs == nothing || isempty(pkgs) + pkgerror("app $f requires at least one package") + end end rm(pkgs_or_apps::String) = rm([pkgs_or_apps]) -function rm(pkgs_or_apps::Union{Vector, Nothing}) - if pkgs_or_apps === nothing - rm(nothing) - else - for pkg_or_app in pkgs_or_apps - if pkg_or_app isa String - pkg_or_app = PackageSpec(pkg_or_app) - end - rm(pkg_or_app) +function rm(pkgs_or_apps::Vector) + for pkg_or_app in pkgs_or_apps + if pkg_or_app isa String + pkg_or_app = PackageSpec(pkg_or_app) end + rm(pkg_or_app) end end From 7aeec766cf637e2bc2af161eba8abd3a4b68d025 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Mon, 3 Feb 2025 10:35:41 -0500 Subject: [PATCH 022/154] Update LineEdit internal for hints (#4146) --- ext/REPLExt/REPLExt.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/REPLExt/REPLExt.jl b/ext/REPLExt/REPLExt.jl index 92e636cd2f..b7942b4a95 100644 --- a/ext/REPLExt/REPLExt.jl +++ b/ext/REPLExt/REPLExt.jl @@ -154,7 +154,7 @@ function create_mode(repl::REPL.AbstractREPL, main::LineEdit.Prompt) end else LineEdit.edit_insert(s, ';') - LineEdit.check_for_hint(s) && LineEdit.refresh_line(s) + LineEdit.check_show_hint(s) end end end @@ -181,7 +181,7 @@ function repl_init(repl::REPL.LineEditREPL) end else LineEdit.edit_insert(s, ']') - LineEdit.check_for_hint(s) && LineEdit.refresh_line(s) + LineEdit.check_show_hint(s) end end ) From 834c62d07bbb696739f7fa2605268869ac986b2b Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 5 Feb 2025 13:50:40 +0100 Subject: [PATCH 023/154] reset caching of STDLIBs reading after precompilation is done (#4148) --- src/Pkg.jl | 1 + src/Types.jl | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Pkg.jl b/src/Pkg.jl index 9d8d5aa36a..8c7837d10d 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -869,5 +869,6 @@ include("precompile.jl") DEFAULT_IO[] = nothing Pkg.UPDATED_REGISTRY_THIS_SESSION[] = false PREV_ENV_PATH[] = "" +Types.STDLIB[] = nothing end # module diff --git a/src/Types.jl b/src/Types.jl index 4d7a6f4b24..5f72f49c96 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -471,7 +471,7 @@ is_project_uuid(env::EnvCache, uuid::UUID) = project_uuid(env) == uuid const UPGRADABLE_STDLIBS = ["DelimitedFiles", "Statistics"] const UPGRADABLE_STDLIBS_UUIDS = Set{UUID}() -const STDLIB = Ref{DictStdLibs}() +const STDLIB = Ref{Union{DictStdLibs, Nothing}}(nothing) function load_stdlib() stdlib = DictStdLibs() for name in readdir(stdlib_dir()) @@ -500,7 +500,7 @@ function stdlibs() return Dict(uuid => (info.name, info.version) for (uuid, info) in stdlib_infos()) end function stdlib_infos() - if !isassigned(STDLIB) + if STDLIB[] === nothing STDLIB[] = load_stdlib() end return STDLIB[] From 5e6c457dd2f37b496ffee69e7830906bdc5d2cb0 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Mon, 10 Feb 2025 20:09:54 -0500 Subject: [PATCH 024/154] Add Aqua tests. Add missing compats (#4150) --- Project.toml | 16 +++++++++++++++ test/Project.toml | 2 ++ test/aqua.jl | 2 ++ test/runtests.jl | 50 +++++++++++++++++++++++++++-------------------- 4 files changed, 49 insertions(+), 21 deletions(-) create mode 100644 test/aqua.jl diff --git a/Project.toml b/Project.toml index 4ddbbefd00..fe6c8ccc9d 100644 --- a/Project.toml +++ b/Project.toml @@ -32,4 +32,20 @@ REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" REPLExt = "REPL" [compat] +Artifacts = "1.11" +Dates = "1.11" +Downloads = "1.6" +FileWatching = "1.11" +LibGit2 = "1.11" +Libdl = "1.11" +Logging = "1.11" +Markdown = "1.11" +Printf = "1.11" +Random = "1.11" +REPL = "1.11" +SHA = "0.7" +TOML = "1" +Tar = "1.10" +UUIDs = "1.11" julia = "1.12" +p7zip_jll = "17.5" diff --git a/test/Project.toml b/test/Project.toml index 0922624374..4760acba31 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,4 +1,5 @@ [deps] +Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" HistoricalStdlibVersions = "6df8b67a-e8a0-4029-b4b7-ac196fe72102" LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" @@ -15,4 +16,5 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] +Aqua = "0.8.10" HistoricalStdlibVersions = "2" diff --git a/test/aqua.jl b/test/aqua.jl new file mode 100644 index 0000000000..c5aeb90392 --- /dev/null +++ b/test/aqua.jl @@ -0,0 +1,2 @@ +using Aqua +Aqua.test_all(Pkg) diff --git a/test/runtests.jl b/test/runtests.jl index f912ff06b1..91eea997b0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -67,29 +67,37 @@ Logging.with_logger((islogging || Pkg.DEFAULT_IO[] == devnull) ? Logging.Console Utils.check_init_reg() + test_files = [ + "new.jl", + "pkg.jl", + "repl.jl", + "api.jl", + "registry.jl", + "subdir.jl", + "extensions.jl", + "artifacts.jl", + "binaryplatforms.jl", + "platformengines.jl", + "sandbox.jl", + "resolve.jl", + "misc.jl", + "force_latest_compatible_version.jl", + "manifests.jl", + "project_manifest.jl", + "sources.jl", + "workspaces.jl", + "apps.jl", + ] + + # Only test these if the test deps are available (they aren't typically via `Base.runtests`) + Aqua_pkgid = Base.PkgId(Base.UUID("4c88cf16-eb10-579e-8560-4a9242c79595"), "Aqua") + if Base.locate_package(Aqua_pkgid) !== nothing + push!(test_files, "aqua.jl") + end + @testset "Pkg" begin try - @testset "$f" for f in [ - "new.jl", - "pkg.jl", - "repl.jl", - "api.jl", - "registry.jl", - "subdir.jl", - "extensions.jl", - "artifacts.jl", - "binaryplatforms.jl", - "platformengines.jl", - "sandbox.jl", - "resolve.jl", - "misc.jl", - "force_latest_compatible_version.jl", - "manifests.jl", - "project_manifest.jl", - "sources.jl", - "workspaces.jl", - "apps.jl", - ] + @testset "$f" for f in test_files @info "==== Testing `test/$f`" flush(Pkg.DEFAULT_IO[]) include(f) From ebfc041327c6b2219ef898aca0ceb9a7192f2662 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Tue, 11 Feb 2025 06:47:16 -0800 Subject: [PATCH 025/154] Fix artifact directories not having traversal permissions (#4075) It turns out that Windows requires the executable bit set if the `BYPASS_TRAVERSE_CHECKING` privilege is not attached to a user's account. It's simply more correct to ensure that our directories have this bit set, and unfortunately our `filemode()` function call is not complete on Windows and does not include this bit, so we manually add it in on Windows. --- src/Artifacts.jl | 9 ++++++++- test/artifacts.jl | 12 ++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/Artifacts.jl b/src/Artifacts.jl index 957d14aab9..a80eb63a28 100644 --- a/src/Artifacts.jl +++ b/src/Artifacts.jl @@ -84,7 +84,14 @@ function _mv_temp_artifact_dir(temp_dir::String, new_path::String)::Nothing err = ccall(:jl_fs_rename, Int32, (Cstring, Cstring), temp_dir, new_path) if err ≥ 0 # rename worked - chmod(new_path, filemode(dirname(new_path))) + new_path_mode = filemode(dirname(new_path)) + if Sys.iswindows() + # If this is Windows, ensure the directory mode is executable, + # as `filemode()` is incomplete. Some day, that may not be the + # case, there exists a test that will fail if this is changes. + new_path_mode |= 0o111 + end + chmod(new_path, new_path_mode) set_readonly(new_path) return else diff --git a/test/artifacts.jl b/test/artifacts.jl index 605c3b26f8..fb9c6e1a66 100644 --- a/test/artifacts.jl +++ b/test/artifacts.jl @@ -823,4 +823,16 @@ end end end +if Sys.iswindows() + @testset "filemode(dir) non-executable on windows" begin + mktempdir() do dir + touch(joinpath(dir, "foo")) + @test !isempty(readdir(dir)) + # This technically should be true, the fact that it's not is + # a wrinkle of libuv, it would be nice to fix it and so if we + # do, this test will let us know. + @test filemode(dir) & 0o001 == 0 + end + end +end end # module From 0333d55c5a020d053cb0ef003d65f569f83aee02 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Thu, 13 Feb 2025 10:01:19 -0500 Subject: [PATCH 026/154] Fixes for julia_version. Expand and consolidate julia_version tests. (#4151) --- src/Operations.jl | 54 +++-- test/historical_stdlib_version.jl | 321 ++++++++++++++++++++++++++++++ test/new.jl | 138 ------------- test/resolve.jl | 48 ----- test/runtests.jl | 28 +-- 5 files changed, 362 insertions(+), 227 deletions(-) create mode 100644 test/historical_stdlib_version.jl diff --git a/src/Operations.jl b/src/Operations.jl index 4c47468a26..4e4c8fe44e 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -222,21 +222,42 @@ end # This has to be done after the packages have been downloaded # since we need access to the Project file to read the information # about extensions -function fixups_from_projectfile!(env::EnvCache) +function fixups_from_projectfile!(ctx::Context) + env = ctx.env for pkg in values(env.manifest) - # isfile_casesenstive within locate_project_file used to error on Windows if given a - # relative path so abspath it to be extra safe https://github.com/JuliaLang/julia/pull/55220 - project_file = Base.locate_project_file(abspath(source_path(env.manifest_file, pkg))) - if project_file isa String && isfile(project_file) - p = Types.read_project(project_file) - pkg.weakdeps = p.weakdeps - pkg.exts = p.exts - pkg.entryfile = p.entryfile - for (name, _) in p.weakdeps + if ctx.julia_version !== VERSION && is_stdlib(pkg.uuid, ctx.julia_version) + # Special handling for non-current julia_version resolving given the source for historical stdlibs + # isn't available at this stage as Pkg thinks it should not be needed, so rely on STDLIBS_BY_VERSION + stdlibs = Types.get_last_stdlibs(ctx.julia_version) + p = stdlibs[pkg.uuid] + pkg.weakdeps = Dict{String, Base.UUID}(stdlibs[uuid].name => uuid for uuid in p.weakdeps) + # pkg.exts = p.exts # TODO: STDLIBS_BY_VERSION doesn't record this + # pkg.entryfile = p.entryfile # TODO: STDLIBS_BY_VERSION doesn't record this + for (name, _) in pkg.weakdeps if !haskey(p.deps, name) delete!(pkg.deps, name) end end + else + # normal mode based on project files. + # isfile_casesenstive within locate_project_file used to error on Windows if given a + # relative path so abspath it to be extra safe https://github.com/JuliaLang/julia/pull/55220 + sourcepath = source_path(env.manifest_file, pkg) + if sourcepath === nothing + pkgerror("could not find source path for package $(pkg.name) based on manifest $(env.manifest_file)") + end + project_file = Base.locate_project_file(abspath(sourcepath)) + if project_file isa String && isfile(project_file) + p = Types.read_project(project_file) + pkg.weakdeps = p.weakdeps + pkg.exts = p.exts + pkg.entryfile = p.entryfile + for (name, _) in p.weakdeps + if !haskey(p.deps, name) + delete!(pkg.deps, name) + end + end + end end end prune_manifest(env) @@ -1658,7 +1679,7 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}, new_git=Set{UUID}(); man_pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, preserve, ctx.julia_version) update_manifest!(ctx.env, man_pkgs, deps_map, ctx.julia_version) new_apply = download_source(ctx) - fixups_from_projectfile!(ctx.env) + fixups_from_projectfile!(ctx) # After downloading resolutionary packages, search for (Julia)Artifacts.toml files # and ensure they are all downloaded and unpacked as well: @@ -1705,7 +1726,7 @@ function develop(ctx::Context, pkgs::Vector{PackageSpec}, new_git::Set{UUID}; pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, preserve, ctx.julia_version) update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version) new_apply = download_source(ctx) - fixups_from_projectfile!(ctx.env) + fixups_from_projectfile!(ctx) download_artifacts(ctx; platform=platform, julia_version=ctx.julia_version) write_env(ctx.env) # write env before building show_update(ctx.env, ctx.registries; io=ctx.io) @@ -1846,7 +1867,7 @@ function up(ctx::Context, pkgs::Vector{PackageSpec}, level::UpgradeLevel; end update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version) new_apply = download_source(ctx) - fixups_from_projectfile!(ctx.env) + fixups_from_projectfile!(ctx) download_artifacts(ctx, julia_version=ctx.julia_version) write_env(ctx.env; skip_writing_project) # write env before building show_update(ctx.env, ctx.registries; io=ctx.io, hidden_upgrades_info = true) @@ -1892,7 +1913,7 @@ function pin(ctx::Context, pkgs::Vector{PackageSpec}) update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version) new = download_source(ctx) - fixups_from_projectfile!(ctx.env) + fixups_from_projectfile!(ctx) download_artifacts(ctx; julia_version=ctx.julia_version) write_env(ctx.env) # write env before building show_update(ctx.env, ctx.registries; io=ctx.io) @@ -1940,7 +1961,7 @@ function free(ctx::Context, pkgs::Vector{PackageSpec}; err_if_free=true) update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version) new = download_source(ctx) - fixups_from_projectfile!(ctx.env) + fixups_from_projectfile!(ctx) download_artifacts(ctx) write_env(ctx.env) # write env before building show_update(ctx.env, ctx.registries; io=ctx.io) @@ -2549,8 +2570,7 @@ end function is_package_downloaded(manifest_file::String, pkg::PackageSpec; platform=HostPlatform()) sourcepath = source_path(manifest_file, pkg) - identifier = pkg.name !== nothing ? pkg.name : pkg.uuid - (sourcepath === nothing) && pkgerror("Could not locate the source code for the $(identifier) package. Are you trying to use a manifest generated by a different version of Julia?") + sourcepath === nothing && return false isdir(sourcepath) || return false check_artifacts_downloaded(sourcepath; platform) || return false return true diff --git a/test/historical_stdlib_version.jl b/test/historical_stdlib_version.jl new file mode 100644 index 0000000000..4b7288ced8 --- /dev/null +++ b/test/historical_stdlib_version.jl @@ -0,0 +1,321 @@ +module HistoricalStdlibVersionsTests +using ..Pkg +using Pkg.Types: is_stdlib +using Pkg.Artifacts: artifact_meta, artifact_path +using Base.BinaryPlatforms: HostPlatform, Platform, platforms_match +using Test +using TOML + +ENV["HISTORICAL_STDLIB_VERSIONS_AUTO_REGISTER"]="false" +using HistoricalStdlibVersions + +include("utils.jl") +using .Utils + +@testset "is_stdlib() across versions" begin + HistoricalStdlibVersions.register!() + + networkoptions_uuid = Base.UUID("ca575930-c2e3-43a9-ace4-1e988b2c1908") + pkg_uuid = Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f") + mbedtls_jll_uuid = Base.UUID("c8ffd9c3-330d-5841-b78e-0817d7145fa1") + + # Test NetworkOptions across multiple versions (It became an stdlib in v1.6+, and was registered) + @test is_stdlib(networkoptions_uuid) + @test is_stdlib(networkoptions_uuid, v"1.6") + @test !is_stdlib(networkoptions_uuid, v"1.5") + @test !is_stdlib(networkoptions_uuid, v"1.0.0") + @test !is_stdlib(networkoptions_uuid, v"0.7") + @test !is_stdlib(networkoptions_uuid, nothing) + + # Pkg is an unregistered stdlib and has always been an stdlib + @test is_stdlib(pkg_uuid) + @test is_stdlib(pkg_uuid, v"1.0") + @test is_stdlib(pkg_uuid, v"1.6") + @test is_stdlib(pkg_uuid, v"999.999.999") + @test is_stdlib(pkg_uuid, v"0.7") + @test is_stdlib(pkg_uuid, nothing) + + # MbedTLS_jll stopped being a stdlib in 1.12 + @test !is_stdlib(mbedtls_jll_uuid) + @test !is_stdlib(mbedtls_jll_uuid, v"1.12") + @test is_stdlib(mbedtls_jll_uuid, v"1.11") + @test is_stdlib(mbedtls_jll_uuid, v"1.10") + + HistoricalStdlibVersions.unregister!() + # Test that we can probe for stdlibs for the current version with no STDLIBS_BY_VERSION, + # but that we throw a PkgError if we ask for a particular julia version. + @test is_stdlib(networkoptions_uuid) + @test_throws Pkg.Types.PkgError is_stdlib(networkoptions_uuid, v"1.6") +end + + +@testset "Pkg.add() with julia_version" begin + HistoricalStdlibVersions.register!() + + # A package with artifacts that went from normal package -> stdlib + gmp_jll_uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d" + # A package that has always only ever been an stdlib + linalg_uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + # A package that went from normal package - >stdlib + networkoptions_uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" + + function get_manifest_block(name) + manifest_path = joinpath(dirname(Base.active_project()), "Manifest.toml") + @test isfile(manifest_path) + deps = Base.get_deps(TOML.parsefile(manifest_path)) + @test haskey(deps, name) + return only(deps[name]) + end + + isolate(loaded_depot=true) do + # Next, test that if we ask for `v1.5` it DOES have a version, and that GMP_jll installs v6.1.X + Pkg.add(["NetworkOptions", "GMP_jll"]; julia_version=v"1.5") + no_block = get_manifest_block("NetworkOptions") + @test haskey(no_block, "uuid") + @test no_block["uuid"] == networkoptions_uuid + @test haskey(no_block, "version") + + gmp_block = get_manifest_block("GMP_jll") + @test haskey(gmp_block, "uuid") + @test gmp_block["uuid"] == gmp_jll_uuid + @test haskey(gmp_block, "version") + @test startswith(gmp_block["version"], "6.1.2") + + # Test that the artifact of GMP_jll contains the right library + @test haskey(gmp_block, "git-tree-sha1") + gmp_jll_dir = Pkg.Operations.find_installed("GMP_jll", Base.UUID(gmp_jll_uuid), Base.SHA1(gmp_block["git-tree-sha1"])) + @test isdir(gmp_jll_dir) + artifacts_toml = joinpath(gmp_jll_dir, "Artifacts.toml") + @test isfile(artifacts_toml) + meta = artifact_meta("GMP", artifacts_toml) + + # `meta` can be `nothing` on some of our newer platforms; we _know_ this should + # not be the case on the following platforms, so we check these explicitly to + # ensure that we haven't accidentally broken something, and then we gate some + # following tests on whether or not `meta` is `nothing`: + for arch in ("x86_64", "i686"), os in ("linux", "mac", "windows") + if platforms_match(HostPlatform(), Platform(arch, os)) + @test meta !== nothing + end + end + + # These tests require a matching platform artifact for this old version of GMP_jll, + # which is not the case on some of our newer platforms. + if meta !== nothing + gmp_artifact_path = artifact_path(Base.SHA1(meta["git-tree-sha1"])) + @test isdir(gmp_artifact_path) + + # On linux, we can check the filename to ensure it's grabbing the correct library + if Sys.islinux() + libgmp_filename = joinpath(gmp_artifact_path, "lib", "libgmp.so.10.3.2") + @test isfile(libgmp_filename) + end + end + end + + # Next, test that if we ask for `v1.6`, GMP_jll gets `v6.2.0`, and for `v1.7`, it gets `v6.2.1` + function do_gmp_test(julia_version, gmp_version) + isolate(loaded_depot=true) do + Pkg.add("GMP_jll"; julia_version) + gmp_block = get_manifest_block("GMP_jll") + @test haskey(gmp_block, "uuid") + @test gmp_block["uuid"] == gmp_jll_uuid + @test haskey(gmp_block, "version") + @test startswith(gmp_block["version"], string(gmp_version)) + end + end + do_gmp_test(v"1.6", v"6.2.0") + do_gmp_test(v"1.7", v"6.2.1") + + isolate(loaded_depot=true) do + # Next, test that if we ask for `nothing`, NetworkOptions has a `version` but `LinearAlgebra` does not. + Pkg.add(["LinearAlgebra", "NetworkOptions"]; julia_version=nothing) + no_block = get_manifest_block("NetworkOptions") + @test haskey(no_block, "uuid") + @test no_block["uuid"] == networkoptions_uuid + @test haskey(no_block, "version") + linalg_block = get_manifest_block("LinearAlgebra") + @test haskey(linalg_block, "uuid") + @test linalg_block["uuid"] == linalg_uuid + @test !haskey(linalg_block, "version") + end + + isolate(loaded_depot=true) do + # Next, test that stdlibs do not get dependencies from the registry + # NOTE: this test depends on the fact that in Julia v1.6+ we added + # "fake" JLLs that do not depend on Pkg while the "normal" p7zip_jll does. + # A future p7zip_jll in the registry may not depend on Pkg, so be sure + # to verify your assumptions when updating this test. + Pkg.add("p7zip_jll") + p7zip_jll_uuid = Base.UUID("3f19e933-33d8-53b3-aaab-bd5110c3b7a0") + @test !("Pkg" in keys(Pkg.dependencies()[p7zip_jll_uuid].dependencies)) + end + + HistoricalStdlibVersions.unregister!() +end + +@testset "Resolving for another version of Julia" begin + HistoricalStdlibVersions.register!() + temp_pkg_dir() do dir + function find_by_name(versions, name) + idx = findfirst(p -> p.name == name, versions) + if idx === nothing + return nothing + end + return versions[idx] + end + + # First, we're going to resolve for specific versions of Julia, ensuring we get the right dep versions: + Pkg.Registry.download_default_registries(Pkg.stdout_f()) + ctx = Pkg.Types.Context(;julia_version=v"1.5") + versions, deps = Pkg.Operations._resolve(ctx.io, ctx.env, ctx.registries, [ + Pkg.Types.PackageSpec(name="MPFR_jll", uuid=Base.UUID("3a97d323-0669-5f0c-9066-3539efd106a3")), + ], Pkg.Types.PRESERVE_TIERED, ctx.julia_version) + gmp = find_by_name(versions, "GMP_jll") + @test gmp !== nothing + @test gmp.version.major == 6 && gmp.version.minor == 1 + ctx = Pkg.Types.Context(;julia_version=v"1.6") + versions, deps = Pkg.Operations._resolve(ctx.io, ctx.env, ctx.registries, [ + Pkg.Types.PackageSpec(name="MPFR_jll", uuid=Base.UUID("3a97d323-0669-5f0c-9066-3539efd106a3")), + ], Pkg.Types.PRESERVE_TIERED, ctx.julia_version) + gmp = find_by_name(versions, "GMP_jll") + @test gmp !== nothing + @test gmp.version.major == 6 && gmp.version.minor == 2 + + # We'll also test resolving an "impossible" manifest; one that requires two package versions that + # are not both loadable by the same Julia: + ctx = Pkg.Types.Context(;julia_version=nothing) + versions, deps = Pkg.Operations._resolve(ctx.io, ctx.env, ctx.registries, [ + # This version of GMP only works on Julia v1.6 + Pkg.Types.PackageSpec(name="GMP_jll", uuid=Base.UUID("781609d7-10c4-51f6-84f2-b8444358ff6d"), version=v"6.2.0"), + # This version of MPFR only works on Julia v1.5 + Pkg.Types.PackageSpec(name="MPFR_jll", uuid=Base.UUID("3a97d323-0669-5f0c-9066-3539efd106a3"), version=v"4.0.2"), + ], Pkg.Types.PRESERVE_TIERED, ctx.julia_version) + gmp = find_by_name(versions, "GMP_jll") + @test gmp !== nothing + @test gmp.version.major == 6 && gmp.version.minor == 2 + mpfr = find_by_name(versions, "MPFR_jll") + @test mpfr !== nothing + @test mpfr.version.major == 4 && mpfr.version.minor == 0 + end + HistoricalStdlibVersions.unregister!() +end + +HelloWorldC_jll_UUID = Base.UUID("dca1746e-5efc-54fc-8249-22745bc95a49") +GMP_jll_UUID = Base.UUID("781609d7-10c4-51f6-84f2-b8444358ff6d") +OpenBLAS_jll_UUID = Base.UUID("4536629a-c528-5b80-bd46-f80d51c5b363") +libcxxwrap_julia_jll_UUID = Base.UUID("3eaa8342-bff7-56a5-9981-c04077f7cee7") +libblastrampoline_jll_UUID = Base.UUID("8e850b90-86db-534c-a0d3-1478176c7d93") + +isolate(loaded_depot=true) do + @testset "Elliot and Mosè's mini Pkg test suite" begin # https://github.com/JuliaPackaging/JLLPrefixes.jl/issues/6 + HistoricalStdlibVersions.register!() + @testset "Standard add" begin + Pkg.activate(temp=true) + # Standard add (non-stdlib, flexible version) + Pkg.add(; name="HelloWorldC_jll") + @test haskey(Pkg.dependencies(), HelloWorldC_jll_UUID) + + Pkg.activate(temp=true) + # Standard add (non-stdlib, url and rev) + Pkg.add(; name="HelloWorldC_jll", url="https://github.com/JuliaBinaryWrappers/HelloWorldC_jll.jl", rev="0b4959a49385d4bb00efd281447dc19348ebac08") + @test Pkg.dependencies()[Base.UUID("dca1746e-5efc-54fc-8249-22745bc95a49")].git_revision === "0b4959a49385d4bb00efd281447dc19348ebac08" + + Pkg.activate(temp=true) + # Standard add (non-stdlib, specified version) + Pkg.add(; name="HelloWorldC_jll", version=v"1.0.10+1") + @test Pkg.dependencies()[Base.UUID("dca1746e-5efc-54fc-8249-22745bc95a49")].version === v"1.0.10+1" + + Pkg.activate(temp=true) + # Standard add (non-stdlib, versionspec) + Pkg.add(; name="HelloWorldC_jll", version=Pkg.Types.VersionSpec("1.0.10")) + @test Pkg.dependencies()[Base.UUID("dca1746e-5efc-54fc-8249-22745bc95a49")].version === v"1.0.10+1" + end + + @testset "Julia-version-dependent add" begin + Pkg.activate(temp=true) + # Julia-version-dependent add (non-stdlib, flexible version) + Pkg.add(; name="libcxxwrap_julia_jll", julia_version=v"1.7") + @test Pkg.dependencies()[libcxxwrap_julia_jll_UUID].version >= v"0.14.0+0" + + Pkg.activate(temp=true) + # Julia-version-dependent add (non-stdlib, specified version) + Pkg.add(; name="libcxxwrap_julia_jll", version=v"0.9.4+0", julia_version=v"1.7") + @test Pkg.dependencies()[libcxxwrap_julia_jll_UUID].version === v"0.9.4+0" + + Pkg.activate(temp=true) + Pkg.add(; name="libcxxwrap_julia_jll", version=v"0.8.8+1", julia_version=v"1.9") + # FIXME? Pkg.dependencies() complains here that mbedtls_jll isn't installed so can't be used here. + # Perhaps Pkg.dependencies() should just return state and not error if source isn't installed? + @test_skip Pkg.dependencies()[libcxxwrap_julia_jll_UUID].version === v"0.9.4+0" + for pkgspec in Pkg.Operations.load_all_deps_loadable(Pkg.Types.Context().env) + if pkgspec.uuid == libcxxwrap_julia_jll_UUID + @test pkgspec.version === v"0.8.8+1" + end + end + end + + @testset "Stdlib add" begin + Pkg.activate(temp=true) + # Stdlib add (current julia version) + Pkg.add(; name="GMP_jll") + @test Pkg.dependencies()[GMP_jll_UUID].version >= v"6.3.0+2" # v1.13.0-DEV + + Pkg.activate(temp=true) + # Make sure the source of GMP_jll is installed + Pkg.add([PackageSpec("GMP_jll")]; julia_version=v"1.6") + src = Pkg.Operations.find_installed( + "GMP_jll", + Base.UUID("781609d7-10c4-51f6-84f2-b8444358ff6d"), + Base.SHA1("40388878122d491a2e55b0e730196098595d8a90") + ) + @test src isa String + # issue https://github.com/JuliaLang/Pkg.jl/issues/2930 + @test_broken isdir(src) + @test_broken isfile(joinpath(src, "Artifacts.toml")) + + Pkg.activate(temp=true) + # Stdlib add (other julia version) + Pkg.add(; name="GMP_jll", julia_version=v"1.7") + @test Pkg.dependencies()[GMP_jll_UUID].version === v"6.2.1+1" + + # Stdlib add (other julia version, with specific version bound) + # Note, this doesn't work properly, it adds but doesn't install any artifacts. + # Technically speaking, this is probably okay from Pkg's perspective, since + # we're asking Pkg to resolve according to what Julia v1.7 would do.... and + # Julia v1.7 would not install anything because it's a stdlib! However, we + # would sometimes like to resolve the latest version of GMP_jll for Julia v1.7 + # then install that. If we have to manually work around that and look up what + # GMP_jll for Julia v1.7 is, then ask for that version explicitly, that's ok. + + Pkg.activate(temp=true) + Pkg.add(; name="GMP_jll", julia_version=v"1.7") + + # This is expected to fail, that version can't live with `julia_version = v"1.7"` + @test_throws Pkg.Resolve.ResolverError Pkg.add(; name="GMP_jll", version=v"6.2.0+5", julia_version=v"1.7") + + Pkg.activate(temp=true) + # Stdlib add (julia_version == nothing) + # Note: this is currently known to be broken, we get the wrong GMP_jll! + Pkg.add(; name="GMP_jll", version=v"6.2.1+1", julia_version=nothing) + @test_broken Pkg.dependencies()[GMP_jll_UUID].version === v"6.2.1+1" + end + + @testset "julia_version = nothing" begin + Pkg.activate(temp=true) + # Stdlib add (impossible constraints due to julia version compat, so + # must pass `julia_version=nothing`). In this case, we always fully + # specify versions, but if we don't, it's okay to just give us whatever + # the resolver prefers + Pkg.add([ + PackageSpec(;name="OpenBLAS_jll", version=v"0.3.13"), + PackageSpec(;name="libblastrampoline_jll", version=v"5.1.1"), + ]; julia_version=nothing) + @test v"0.3.14" > Pkg.dependencies()[OpenBLAS_jll_UUID].version >= v"0.3.13" + @test v"5.1.2" > Pkg.dependencies()[libblastrampoline_jll_UUID].version >= v"5.1.1" + end + HistoricalStdlibVersions.unregister!() + end +end + +end # module diff --git a/test/new.jl b/test/new.jl index 46f91168f7..e100c83833 100644 --- a/test/new.jl +++ b/test/new.jl @@ -7,7 +7,6 @@ using Pkg.Resolve: ResolverError import Pkg.Artifacts: artifact_meta, artifact_path import Base.BinaryPlatforms: HostPlatform, Platform, platforms_match using ..Utils -import ..HistoricalStdlibVersions using Logging general_uuid = UUID("23338594-aafe-5451-b93e-139f81909106") # UUID for `General` @@ -3020,143 +3019,6 @@ end end end -using Pkg.Types: is_stdlib -@testset "is_stdlib() across versions" begin - HistoricalStdlibVersions.register!() - - networkoptions_uuid = UUID("ca575930-c2e3-43a9-ace4-1e988b2c1908") - pkg_uuid = UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f") - - # Test NetworkOptions across multiple versions (It became an stdlib in v1.6+, and was registered) - @test is_stdlib(networkoptions_uuid) - @test is_stdlib(networkoptions_uuid, v"1.6") - @test !is_stdlib(networkoptions_uuid, v"1.5") - @test !is_stdlib(networkoptions_uuid, v"1.0.0") - @test !is_stdlib(networkoptions_uuid, v"0.7") - @test !is_stdlib(networkoptions_uuid, nothing) - - # Pkg is an unregistered stdlib and has always been an stdlib - @test is_stdlib(pkg_uuid) - @test is_stdlib(pkg_uuid, v"1.0") - @test is_stdlib(pkg_uuid, v"1.6") - @test is_stdlib(pkg_uuid, v"999.999.999") - @test is_stdlib(pkg_uuid, v"0.7") - @test is_stdlib(pkg_uuid, nothing) - - HistoricalStdlibVersions.unregister!() - # Test that we can probe for stdlibs for the current version with no STDLIBS_BY_VERSION, - # but that we throw a PkgError if we ask for a particular julia version. - @test is_stdlib(networkoptions_uuid) - @test_throws Pkg.Types.PkgError is_stdlib(networkoptions_uuid, v"1.6") -end - - -@testset "Pkg.add() with julia_version" begin - HistoricalStdlibVersions.register!() - - # A package with artifacts that went from normal package -> stdlib - gmp_jll_uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d" - # A package that has always only ever been an stdlib - linalg_uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" - # A package that went from normal package - >stdlib - networkoptions_uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" - - function get_manifest_block(name) - manifest_path = joinpath(dirname(Base.active_project()), "Manifest.toml") - @test isfile(manifest_path) - deps = Base.get_deps(TOML.parsefile(manifest_path)) - @test haskey(deps, name) - return only(deps[name]) - end - - isolate(loaded_depot=true) do - # Next, test that if we ask for `v1.5` it DOES have a version, and that GMP_jll installs v6.1.X - Pkg.add(["NetworkOptions", "GMP_jll"]; julia_version=v"1.5") - no_block = get_manifest_block("NetworkOptions") - @test haskey(no_block, "uuid") - @test no_block["uuid"] == networkoptions_uuid - @test haskey(no_block, "version") - - gmp_block = get_manifest_block("GMP_jll") - @test haskey(gmp_block, "uuid") - @test gmp_block["uuid"] == gmp_jll_uuid - @test haskey(gmp_block, "version") - @test startswith(gmp_block["version"], "6.1.2") - - # Test that the artifact of GMP_jll contains the right library - @test haskey(gmp_block, "git-tree-sha1") - gmp_jll_dir = Pkg.Operations.find_installed("GMP_jll", Base.UUID(gmp_jll_uuid), Base.SHA1(gmp_block["git-tree-sha1"])) - @test isdir(gmp_jll_dir) - artifacts_toml = joinpath(gmp_jll_dir, "Artifacts.toml") - @test isfile(artifacts_toml) - meta = artifact_meta("GMP", artifacts_toml) - - # `meta` can be `nothing` on some of our newer platforms; we _know_ this should - # not be the case on the following platforms, so we check these explicitly to - # ensure that we haven't accidentally broken something, and then we gate some - # following tests on whether or not `meta` is `nothing`: - for arch in ("x86_64", "i686"), os in ("linux", "mac", "windows") - if platforms_match(HostPlatform(), Platform(arch, os)) - @test meta !== nothing - end - end - - # These tests require a matching platform artifact for this old version of GMP_jll, - # which is not the case on some of our newer platforms. - if meta !== nothing - gmp_artifact_path = artifact_path(Base.SHA1(meta["git-tree-sha1"])) - @test isdir(gmp_artifact_path) - - # On linux, we can check the filename to ensure it's grabbing the correct library - if Sys.islinux() - libgmp_filename = joinpath(gmp_artifact_path, "lib", "libgmp.so.10.3.2") - @test isfile(libgmp_filename) - end - end - end - - # Next, test that if we ask for `v1.6`, GMP_jll gets `v6.2.0`, and for `v1.7`, it gets `v6.2.1` - function do_gmp_test(julia_version, gmp_version) - isolate(loaded_depot=true) do - Pkg.add("GMP_jll"; julia_version) - gmp_block = get_manifest_block("GMP_jll") - @test haskey(gmp_block, "uuid") - @test gmp_block["uuid"] == gmp_jll_uuid - @test haskey(gmp_block, "version") - @test startswith(gmp_block["version"], string(gmp_version)) - end - end - do_gmp_test(v"1.6", v"6.2.0") - do_gmp_test(v"1.7", v"6.2.1") - - isolate(loaded_depot=true) do - # Next, test that if we ask for `nothing`, NetworkOptions has a `version` but `LinearAlgebra` does not. - Pkg.add(["LinearAlgebra", "NetworkOptions"]; julia_version=nothing) - no_block = get_manifest_block("NetworkOptions") - @test haskey(no_block, "uuid") - @test no_block["uuid"] == networkoptions_uuid - @test haskey(no_block, "version") - linalg_block = get_manifest_block("LinearAlgebra") - @test haskey(linalg_block, "uuid") - @test linalg_block["uuid"] == linalg_uuid - @test !haskey(linalg_block, "version") - end - - isolate(loaded_depot=true) do - # Next, test that stdlibs do not get dependencies from the registry - # NOTE: this test depends on the fact that in Julia v1.6+ we added - # "fake" JLLs that do not depend on Pkg while the "normal" p7zip_jll does. - # A future p7zip_jll in the registry may not depend on Pkg, so be sure - # to verify your assumptions when updating this test. - Pkg.add("p7zip_jll") - p7zip_jll_uuid = UUID("3f19e933-33d8-53b3-aaab-bd5110c3b7a0") - @test !("Pkg" in keys(Pkg.dependencies()[p7zip_jll_uuid].dependencies)) - end - - HistoricalStdlibVersions.unregister!() -end - - @testset "Issue #2931" begin isolate(loaded_depot=false) do temp_pkg_dir() do path diff --git a/test/resolve.jl b/test/resolve.jl index 91907e2d10..2fc2e98fe7 100644 --- a/test/resolve.jl +++ b/test/resolve.jl @@ -9,7 +9,6 @@ using Pkg.Types: VersionBound using UUIDs using Pkg.Resolve import Pkg.Resolve: VersionWeight, add_reqs!, simplify_graph!, ResolverError, ResolverTimeoutError, Fixed, Requires -import ..HistoricalStdlibVersions include("utils.jl") using .Utils @@ -737,53 +736,6 @@ end @test_throws ResolverError resolve_tst(deps_data, reqs_data) end -@testset "Resolving for another version of Julia" begin - HistoricalStdlibVersions.register!() - temp_pkg_dir() do dir - function find_by_name(versions, name) - idx = findfirst(p -> p.name == name, versions) - if idx === nothing - return nothing - end - return versions[idx] - end - - # First, we're going to resolve for specific versions of Julia, ensuring we get the right dep versions: - Pkg.Registry.download_default_registries(Pkg.stdout_f()) - ctx = Pkg.Types.Context(;julia_version=v"1.5") - versions, deps = Pkg.Operations._resolve(ctx.io, ctx.env, ctx.registries, [ - Pkg.Types.PackageSpec(name="MPFR_jll", uuid=Base.UUID("3a97d323-0669-5f0c-9066-3539efd106a3")), - ], Pkg.Types.PRESERVE_TIERED, ctx.julia_version) - gmp = find_by_name(versions, "GMP_jll") - @test gmp !== nothing - @test gmp.version.major == 6 && gmp.version.minor == 1 - ctx = Pkg.Types.Context(;julia_version=v"1.6") - versions, deps = Pkg.Operations._resolve(ctx.io, ctx.env, ctx.registries, [ - Pkg.Types.PackageSpec(name="MPFR_jll", uuid=Base.UUID("3a97d323-0669-5f0c-9066-3539efd106a3")), - ], Pkg.Types.PRESERVE_TIERED, ctx.julia_version) - gmp = find_by_name(versions, "GMP_jll") - @test gmp !== nothing - @test gmp.version.major == 6 && gmp.version.minor == 2 - - # We'll also test resolving an "impossible" manifest; one that requires two package versions that - # are not both loadable by the same Julia: - ctx = Pkg.Types.Context(;julia_version=nothing) - versions, deps = Pkg.Operations._resolve(ctx.io, ctx.env, ctx.registries, [ - # This version of GMP only works on Julia v1.6 - Pkg.Types.PackageSpec(name="GMP_jll", uuid=Base.UUID("781609d7-10c4-51f6-84f2-b8444358ff6d"), version=v"6.2.0"), - # This version of MPFR only works on Julia v1.5 - Pkg.Types.PackageSpec(name="MPFR_jll", uuid=Base.UUID("3a97d323-0669-5f0c-9066-3539efd106a3"), version=v"4.0.2"), - ], Pkg.Types.PRESERVE_TIERED, ctx.julia_version) - gmp = find_by_name(versions, "GMP_jll") - @test gmp !== nothing - @test gmp.version.major == 6 && gmp.version.minor == 2 - mpfr = find_by_name(versions, "MPFR_jll") - @test mpfr !== nothing - @test mpfr.version.major == 4 && mpfr.version.minor == 0 - end - HistoricalStdlibVersions.unregister!() -end - @testset "Stdlib resolve smoketest" begin # All stdlibs should be installable and resolvable temp_pkg_dir() do dir diff --git a/test/runtests.jl b/test/runtests.jl index 91eea997b0..7163cc844f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -20,7 +20,6 @@ if realpath(dirname(dirname(Base.pathof(Pkg)))) != realpath(dirname(@__DIR__)) end ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0 -ENV["HISTORICAL_STDLIB_VERSIONS_AUTO_REGISTER"]="false" logdir = get(ENV, "JULIA_TEST_VERBOSE_LOGS_DIR", nothing) ### Send all Pkg output to a file called Pkg.log @@ -37,29 +36,6 @@ end include("utils.jl") Logging.with_logger((islogging || Pkg.DEFAULT_IO[] == devnull) ? Logging.ConsoleLogger(Pkg.DEFAULT_IO[]) : Logging.current_logger()) do - # Because julia CI doesn't run stdlib tests via `Pkg.test` test deps must be manually installed if missing - if Base.find_package("HistoricalStdlibVersions") === nothing - @debug "Installing HistoricalStdlibVersions for Pkg tests" - iob = IOBuffer() - Pkg.activate(; temp = true) - try - # Needed for custom julia version resolve tests - # Don't use the toplevel PKg.add() command to avoid accidentally installing another copy of the registry - spec = Pkg.PackageSpec( - name="HistoricalStdlibVersions", - url="https://github.com/JuliaPackaging/HistoricalStdlibVersions.jl", - rev="5879c5f690795208481c60b904f4af4e8c1eeef8", #= version="2.0.0", =# - uuid="6df8b67a-e8a0-4029-b4b7-ac196fe72102") - Pkg.API.handle_package_input!(spec) - Pkg.add(Pkg.API.Context(), [spec], io=iob) - catch - println(String(take!(iob))) - rethrow() - end - end - - @eval import HistoricalStdlibVersions - if (server = Pkg.pkg_server()) !== nothing && Sys.which("curl") !== nothing s = read(`curl -sLI $(server)`, String); @info "Pkg Server metadata:\n$s" @@ -90,6 +66,10 @@ Logging.with_logger((islogging || Pkg.DEFAULT_IO[] == devnull) ? Logging.Console ] # Only test these if the test deps are available (they aren't typically via `Base.runtests`) + HSV_pkgid = Base.PkgId(Base.UUID("6df8b67a-e8a0-4029-b4b7-ac196fe72102"), "HistoricalStdlibVersions") + if Base.locate_package(HSV_pkgid) !== nothing + push!(test_files, "historical_stdlib_version.jl") + end Aqua_pkgid = Base.PkgId(Base.UUID("4c88cf16-eb10-579e-8560-4a9242c79595"), "Aqua") if Base.locate_package(Aqua_pkgid) !== nothing push!(test_files, "aqua.jl") From c54ee594d1c16270d4f67e09788839d9618faf18 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Thu, 13 Feb 2025 17:48:08 -0500 Subject: [PATCH 027/154] tests: make sure we're in an active project and that it's clean (#4155) --- test/runtests.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/runtests.jl b/test/runtests.jl index 7163cc844f..1d1d258174 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -76,6 +76,7 @@ Logging.with_logger((islogging || Pkg.DEFAULT_IO[] == devnull) ? Logging.Console end @testset "Pkg" begin + Pkg.activate(; temp=true) # make sure we're in an active project and that it's clean try @testset "$f" for f in test_files @info "==== Testing `test/$f`" From 92595ec6727bbb88df6eba856b71ec670281735c Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sun, 16 Feb 2025 08:28:50 -0500 Subject: [PATCH 028/154] Add more tests for BinaryBuilder usage (#4156) --- test/historical_stdlib_version.jl | 42 +++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/test/historical_stdlib_version.jl b/test/historical_stdlib_version.jl index 4b7288ced8..e19574155d 100644 --- a/test/historical_stdlib_version.jl +++ b/test/historical_stdlib_version.jl @@ -302,17 +302,37 @@ isolate(loaded_depot=true) do end @testset "julia_version = nothing" begin - Pkg.activate(temp=true) - # Stdlib add (impossible constraints due to julia version compat, so - # must pass `julia_version=nothing`). In this case, we always fully - # specify versions, but if we don't, it's okay to just give us whatever - # the resolver prefers - Pkg.add([ - PackageSpec(;name="OpenBLAS_jll", version=v"0.3.13"), - PackageSpec(;name="libblastrampoline_jll", version=v"5.1.1"), - ]; julia_version=nothing) - @test v"0.3.14" > Pkg.dependencies()[OpenBLAS_jll_UUID].version >= v"0.3.13" - @test v"5.1.2" > Pkg.dependencies()[libblastrampoline_jll_UUID].version >= v"5.1.1" + @testset "stdlib add" begin + Pkg.activate(temp=true) + # Stdlib add (impossible constraints due to julia version compat, so + # must pass `julia_version=nothing`). In this case, we always fully + # specify versions, but if we don't, it's okay to just give us whatever + # the resolver prefers + Pkg.add([ + PackageSpec(;name="OpenBLAS_jll", version=v"0.3.13"), + PackageSpec(;name="libblastrampoline_jll", version=v"5.1.1"), + ]; julia_version=nothing) + @test v"0.3.14" > Pkg.dependencies()[OpenBLAS_jll_UUID].version >= v"0.3.13" + @test v"5.1.2" > Pkg.dependencies()[libblastrampoline_jll_UUID].version >= v"5.1.1" + end + @testset "non-stdlib JLL add" begin + platform = Platform("x86_64", "linux"; libc="musl") + # specific version vs. compat spec + @testset for version in (v"3.24.3+0", "3.24.3") + dependencies = [PackageSpec(; name="CMake_jll", version = version)] + @testset "with context (using private Pkg.add method)" begin + Pkg.activate(temp=true) + ctx = Pkg.Types.Context(; julia_version=nothing) + mydeps = deepcopy(dependencies) + foreach(Pkg.API.handle_package_input!, mydeps) + Pkg.add(ctx, mydeps; platform) + end + @testset "with julia_version" begin + Pkg.activate(temp=true) + Pkg.add(deepcopy(dependencies); platform, julia_version=nothing) + end + end + end end HistoricalStdlibVersions.unregister!() end From 14831edf4fb6b9db6b8407301c66b33c46b450ad Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sun, 23 Feb 2025 02:03:49 -0500 Subject: [PATCH 029/154] enable windows tree_hash checking (#4167) Given TODO and https://github.com/JuliaLang/julia/issues/33212 is fixed (closed) --- src/Operations.jl | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/Operations.jl b/src/Operations.jl index 4e4c8fe44e..06254731eb 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -750,15 +750,12 @@ function install_archive( unpacked = joinpath(dir, dirs[1]) end # Assert that the tarball unpacked to the tree sha we wanted - # TODO: Enable on Windows when tree_hash handles - # executable bits correctly, see JuliaLang/julia #33212. - if !Sys.iswindows() - if SHA1(GitTools.tree_hash(unpacked)) != hash - @warn "tarball content does not match git-tree-sha1" - url_success = false - end - url_success || continue + if SHA1(GitTools.tree_hash(unpacked)) != hash + @warn "tarball content does not match git-tree-sha1" + url_success = false end + url_success || continue + # Move content to version path !isdir(version_path) && mkpath(version_path) mv(unpacked, version_path; force=true) From f5174898e9c1814c0963b89c40cffa050451a8b5 Mon Sep 17 00:00:00 2001 From: Jesper Stemann Andersen Date: Tue, 25 Feb 2025 02:31:52 +0100 Subject: [PATCH 030/154] Changed Pkg.test to prioritise julia_args --threads over current process threads (#4141) Co-authored-by: Ian Butterworth --- src/Operations.jl | 12 ++-- test/new.jl | 56 +++++++++++++++++++ test/test_packages/.gitignore | 1 + test/test_packages/TestThreads/Project.toml | 2 + .../TestThreads/src/TestThreads.jl | 2 + .../TestThreads/test/runtests.jl | 7 +++ 6 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 test/test_packages/.gitignore create mode 100644 test/test_packages/TestThreads/Project.toml create mode 100644 test/test_packages/TestThreads/src/TestThreads.jl create mode 100644 test/test_packages/TestThreads/test/runtests.jl diff --git a/src/Operations.jl b/src/Operations.jl index 06254731eb..d5e079a687 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -1982,14 +1982,18 @@ end function get_threads_spec() - if Threads.nthreads(:interactive) > 0 + if haskey(ENV, "JULIA_NUM_THREADS") + # if set, prefer JULIA_NUM_THREADS because this is passed to the test worker via --threads + # which takes precedence in the worker + ENV["JULIA_NUM_THREADS"] + elseif Threads.nthreads(:interactive) > 0 "$(Threads.nthreads(:default)),$(Threads.nthreads(:interactive))" else "$(Threads.nthreads(:default))" end end -function gen_subprocess_flags(source_path::String; coverage, julia_args) +function gen_subprocess_flags(source_path::String; coverage, julia_args::Cmd) coverage_arg = if coverage isa Bool # source_path is the package root, not "src" so "ext" etc. is included coverage ? string("@", source_path) : "none" @@ -2327,7 +2331,7 @@ function test(ctx::Context, pkgs::Vector{PackageSpec}; test_fn !== nothing && test_fn() sandbox_ctx = Context(;io=ctx.io) status(sandbox_ctx.env, sandbox_ctx.registries; mode=PKGMODE_COMBINED, io=sandbox_ctx.io, ignore_indent = false, show_usagetips = false) - flags = gen_subprocess_flags(source_path; coverage,julia_args) + flags = gen_subprocess_flags(source_path; coverage, julia_args) if should_autoprecompile() cacheflags = Base.CacheFlags(parse(UInt8, read(`$(Base.julia_cmd()) $(flags) --eval 'show(ccall(:jl_cache_flags, UInt8, ()))'`, String))) @@ -2337,7 +2341,7 @@ function test(ctx::Context, pkgs::Vector{PackageSpec}; printpkgstyle(ctx.io, :Testing, "Running tests...") flush(ctx.io) code = gen_test_code(source_path; test_args) - cmd = `$(Base.julia_cmd()) $(flags) --threads=$(get_threads_spec()) --eval $code` + cmd = `$(Base.julia_cmd()) --threads=$(get_threads_spec()) $(flags) --eval $code` p, interrupted = subprocess_handler(cmd, ctx.io, "Tests interrupted. Exiting the test process") if success(p) printpkgstyle(ctx.io, :Testing, pkg.name * " tests passed ") diff --git a/test/new.jl b/test/new.jl index e100c83833..2540f99bc8 100644 --- a/test/new.jl +++ b/test/new.jl @@ -2014,6 +2014,62 @@ end Pkg.test("TestArguments"; test_args=`a b`, julia_args=`--quiet --check-bounds=no`) Pkg.test("TestArguments"; test_args=["a", "b"], julia_args=["--quiet", "--check-bounds=no"]) end end + + @testset "threads" begin + mktempdir() do dir + path = copy_test_package(dir, "TestThreads") + cd(path) do + with_current_env() do + default_nthreads_default = Threads.nthreads(:default) + default_nthreads_interactive = Threads.nthreads(:interactive) + other_nthreads_default = default_nthreads_default == 1 ? 2 : 1 + other_nthreads_interactive = default_nthreads_interactive == 0 ? 1 : 0 + @testset "default" begin + withenv( + "EXPECTED_NUM_THREADS_DEFAULT" => "$default_nthreads_default", + "EXPECTED_NUM_THREADS_INTERACTIVE" => "$default_nthreads_interactive", + ) do + Pkg.test("TestThreads") + end + end + @testset "JULIA_NUM_THREADS=other_nthreads_default" begin + withenv( + "EXPECTED_NUM_THREADS_DEFAULT" => "$other_nthreads_default", + "EXPECTED_NUM_THREADS_INTERACTIVE" => "$default_nthreads_interactive", + "JULIA_NUM_THREADS" => "$other_nthreads_default", + ) do + Pkg.test("TestThreads") + end + end + @testset "JULIA_NUM_THREADS=other_nthreads_default,other_nthreads_interactive" begin + withenv( + "EXPECTED_NUM_THREADS_DEFAULT" => "$other_nthreads_default", + "EXPECTED_NUM_THREADS_INTERACTIVE" => "$other_nthreads_interactive", + "JULIA_NUM_THREADS" => "$other_nthreads_default,$other_nthreads_interactive", + ) do + Pkg.test("TestThreads") + end + end + @testset "--threads=other_nthreads_default" begin + withenv( + "EXPECTED_NUM_THREADS_DEFAULT" => "$other_nthreads_default", + "EXPECTED_NUM_THREADS_INTERACTIVE" => "$default_nthreads_interactive", + ) do + Pkg.test("TestThreads"; julia_args=`--threads=$other_nthreads_default`) + end + end + @testset "--threads=other_nthreads_default,other_nthreads_interactive" begin + withenv( + "EXPECTED_NUM_THREADS_DEFAULT" => "$other_nthreads_default", + "EXPECTED_NUM_THREADS_INTERACTIVE" => "$other_nthreads_interactive", + ) do + Pkg.test("TestThreads"; julia_args=`--threads=$other_nthreads_default,$other_nthreads_interactive`) + end + end + end + end + end + end end # diff --git a/test/test_packages/.gitignore b/test/test_packages/.gitignore new file mode 100644 index 0000000000..ba39cc531e --- /dev/null +++ b/test/test_packages/.gitignore @@ -0,0 +1 @@ +Manifest.toml diff --git a/test/test_packages/TestThreads/Project.toml b/test/test_packages/TestThreads/Project.toml new file mode 100644 index 0000000000..35e36aed33 --- /dev/null +++ b/test/test_packages/TestThreads/Project.toml @@ -0,0 +1,2 @@ +name = "TestThreads" +uuid = "79df5fe7-ed23-44ca-b7b9-b3881e57664d" diff --git a/test/test_packages/TestThreads/src/TestThreads.jl b/test/test_packages/TestThreads/src/TestThreads.jl new file mode 100644 index 0000000000..11d357747f --- /dev/null +++ b/test/test_packages/TestThreads/src/TestThreads.jl @@ -0,0 +1,2 @@ +module TestThreads +end diff --git a/test/test_packages/TestThreads/test/runtests.jl b/test/test_packages/TestThreads/test/runtests.jl new file mode 100644 index 0000000000..43b8df8628 --- /dev/null +++ b/test/test_packages/TestThreads/test/runtests.jl @@ -0,0 +1,7 @@ +@assert haskey(ENV, "EXPECTED_NUM_THREADS_DEFAULT") +@assert haskey(ENV, "EXPECTED_NUM_THREADS_INTERACTIVE") +EXPECTED_NUM_THREADS_DEFAULT = parse(Int, ENV["EXPECTED_NUM_THREADS_DEFAULT"]) +EXPECTED_NUM_THREADS_INTERACTIVE = parse(Int, ENV["EXPECTED_NUM_THREADS_INTERACTIVE"]) +@assert Threads.nthreads() == EXPECTED_NUM_THREADS_DEFAULT +@assert Threads.nthreads(:default) == EXPECTED_NUM_THREADS_DEFAULT +@assert Threads.nthreads(:interactive) == EXPECTED_NUM_THREADS_INTERACTIVE From 2779d42f4991d09733bf52c9ca47478d1bef3a8d Mon Sep 17 00:00:00 2001 From: Heptazhou Date: Fri, 28 Feb 2025 21:51:24 +0000 Subject: [PATCH 031/154] Update toml-files.md (#4177) --- docs/src/toml-files.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/toml-files.md b/docs/src/toml-files.md index 262c1b5767..016d42450b 100644 --- a/docs/src/toml-files.md +++ b/docs/src/toml-files.md @@ -162,7 +162,7 @@ For the details, see [`Pkg.instantiate`](@ref). ### Different Manifests for Different Julia versions -Starting from Julia v1.11, there is an option to name manifest files in the format `Manifest-v{major}.{minor}.toml`. +Starting from Julia v1.10.8, there is an option to name manifest files in the format `Manifest-v{major}.{minor}.toml`. Julia will then preferentially use the version-specific manifest file if available. For example, if both `Manifest-v1.11.toml` and `Manifest.toml` exist, Julia 1.11 will prioritize using `Manifest-v1.11.toml`. However, Julia versions 1.10, 1.12, and all others will default to using `Manifest.toml`. From cf6191bfaae21c224f6314753b2a449f21cc8213 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 1 Mar 2025 12:33:40 -0500 Subject: [PATCH 032/154] Pidlock source and artifact installation. Registry pidlock fixes. (#4168) * protect against an interrupted mv * pidlock download_source * add tests * try a proper package * actually test ffmpeg works * windows fix * pidlock artifact downloads * don't claim to have done work * more tests * debug hang * debug: try a yield * increase stale_age for downloads * rm debugs * use (and rename) mv_temp_artifact_dir * disable modifying permissions in mv_temp_artifact_dir * rm goto's * rename as generic and move to utils * reduce to 1 test. lock ffmpeg version. * show info if test fails * debug: turn on verbose testsets for timing * tweaks * fix error printing * check which registries need installing after acquiring pidlock * rm ispath check outside of pidlock * add time print * color process prints * use early throw `Base.Experimental.@sync` * narrow try * reduce stale_age * fix! * remove print from happy path * also pidlock git installs --- src/Artifacts.jl | 213 +++++++++++++++++---------------------- src/Operations.jl | 86 ++++++++++------ src/Registry/Registry.jl | 192 ++++++++++++++++++----------------- src/utils.jl | 52 ++++++++++ test/new.jl | 95 ++++++++++++++++- test/runtests.jl | 5 +- 6 files changed, 390 insertions(+), 253 deletions(-) diff --git a/src/Artifacts.jl b/src/Artifacts.jl index a80eb63a28..0702e3670c 100644 --- a/src/Artifacts.jl +++ b/src/Artifacts.jl @@ -3,9 +3,10 @@ module Artifacts using Artifacts, Base.BinaryPlatforms, SHA using ..MiniProgressBars, ..PlatformEngines using Tar: can_symlink +using FileWatching: FileWatching import ..set_readonly, ..GitTools, ..TOML, ..pkg_server, ..can_fancyprint, - ..stderr_f, ..printpkgstyle + ..stderr_f, ..printpkgstyle, ..mv_temp_dir_retries import Base: get, SHA1 import Artifacts: artifact_names, ARTIFACTS_DIR_OVERRIDE, ARTIFACT_OVERRIDES, artifact_paths, @@ -49,7 +50,7 @@ function create_artifact(f::Function) # as something that was foolishly overridden. This should be virtually impossible # unless the user has been very unwise, but let's be cautious. new_path = artifact_path(artifact_hash; honor_overrides=false) - _mv_temp_artifact_dir(temp_dir, new_path) + mv_temp_dir_retries(temp_dir, new_path) # Give the people what they want return artifact_hash @@ -59,55 +60,6 @@ function create_artifact(f::Function) end end -""" - _mv_temp_artifact_dir(temp_dir::String, new_path::String)::Nothing -Either rename the directory at `temp_dir` to `new_path` and set it to read-only -or if `new_path` artifact already exists try to do nothing. -""" -function _mv_temp_artifact_dir(temp_dir::String, new_path::String)::Nothing - # Sometimes a rename can fail because the temp_dir is locked by - # anti-virus software scanning the new files. - # In this case we want to sleep and try again. - # I am using the list of error codes to retry from: - # https://github.com/isaacs/node-graceful-fs/blob/234379906b7d2f4c9cfeb412d2516f42b0fb4953/polyfills.js#L87 - # Retry for up to about 60 seconds by retrying 20 times with exponential backoff. - retry = 0 - max_num_retries = 20 # maybe this should be configurable? - sleep_amount = 0.01 # seconds - max_sleep_amount = 5.0 # seconds - while true - isdir(new_path) && return - # This next step is like - # `mv(temp_dir, new_path)`. - # However, `mv` defaults to `cp` if `rename` returns an error. - # `cp` is not atomic, so avoid the potential of calling it. - err = ccall(:jl_fs_rename, Int32, (Cstring, Cstring), temp_dir, new_path) - if err ≥ 0 - # rename worked - new_path_mode = filemode(dirname(new_path)) - if Sys.iswindows() - # If this is Windows, ensure the directory mode is executable, - # as `filemode()` is incomplete. Some day, that may not be the - # case, there exists a test that will fail if this is changes. - new_path_mode |= 0o111 - end - chmod(new_path, new_path_mode) - set_readonly(new_path) - return - else - # Ignore rename error if `new_path` exists. - isdir(new_path) && return - if retry < max_num_retries && err ∈ (Base.UV_EACCES, Base.UV_EPERM, Base.UV_EBUSY) - sleep(sleep_amount) - sleep_amount = min(sleep_amount*2.0, max_sleep_amount) - retry += 1 - else - Base.uv_error("rename of $(repr(temp_dir)) to $(repr(new_path))", err) - end - end - end -end - """ remove_artifact(hash::SHA1; honor_overrides::Bool=false) @@ -330,87 +282,102 @@ function download_artifact( io::IO=stderr_f(), progress::Union{Function, Nothing} = nothing, ) - if artifact_exists(tree_hash) - return true + _artifact_paths = Artifacts.artifact_paths(tree_hash) + pidfile = _artifact_paths[1] * ".pid" + mkpath(dirname(pidfile)) + t_wait_msg = Timer(2) do t + if progress === nothing + @info "downloading $tarball_url ($hex) in another process" + else + progress(0, 0; status="downloading in another process") + end end + ret = FileWatching.mkpidlock(pidfile, stale_age = 20) do + close(t_wait_msg) + if artifact_exists(tree_hash) + return true + end - # Ensure the `artifacts` directory exists in our default depot - artifacts_dir = first(artifacts_dirs()) - mkpath(artifacts_dir) - # expected artifact path - dst = joinpath(artifacts_dir, bytes2hex(tree_hash.bytes)) + # Ensure the `artifacts` directory exists in our default depot + artifacts_dir = first(artifacts_dirs()) + mkpath(artifacts_dir) + # expected artifact path + dst = joinpath(artifacts_dir, bytes2hex(tree_hash.bytes)) - # We download by using a temporary directory. We do this because the download may - # be corrupted or even malicious; we don't want to clobber someone else's artifact - # by trusting the tree hash that has been given to us; we will instead download it - # to a temporary directory, calculate the true tree hash, then move it to the proper - # location only after knowing what it is, and if something goes wrong in the process, - # everything should be cleaned up. + # We download by using a temporary directory. We do this because the download may + # be corrupted or even malicious; we don't want to clobber someone else's artifact + # by trusting the tree hash that has been given to us; we will instead download it + # to a temporary directory, calculate the true tree hash, then move it to the proper + # location only after knowing what it is, and if something goes wrong in the process, + # everything should be cleaned up. - # Temporary directory where we'll do our creation business - temp_dir = mktempdir(artifacts_dir) + # Temporary directory where we'll do our creation business + temp_dir = mktempdir(artifacts_dir) - try - download_verify_unpack(tarball_url, tarball_hash, temp_dir; - ignore_existence=true, verbose, quiet_download, io, progress) - isnothing(progress) || progress(10000, 10000; status="verifying") - calc_hash = SHA1(GitTools.tree_hash(temp_dir)) - - # Did we get what we expected? If not, freak out. - if calc_hash.bytes != tree_hash.bytes - msg = """ - Tree Hash Mismatch! - Expected git-tree-sha1: $(bytes2hex(tree_hash.bytes)) - Calculated git-tree-sha1: $(bytes2hex(calc_hash.bytes)) - """ - # Since tree hash calculation is rather fragile and file system dependent, - # we allow setting JULIA_PKG_IGNORE_HASHES=1 to ignore the error and move - # the artifact to the expected location and return true - ignore_hash_env_set = get(ENV, "JULIA_PKG_IGNORE_HASHES", "") != "" - if ignore_hash_env_set - ignore_hash = Base.get_bool_env("JULIA_PKG_IGNORE_HASHES", false) - ignore_hash === nothing && @error( - "Invalid ENV[\"JULIA_PKG_IGNORE_HASHES\"] value", - ENV["JULIA_PKG_IGNORE_HASHES"], - ) - ignore_hash = something(ignore_hash, false) - else - # default: false except Windows users who can't symlink - ignore_hash = Sys.iswindows() && - !mktempdir(can_symlink, artifacts_dir) + try + download_verify_unpack(tarball_url, tarball_hash, temp_dir; + ignore_existence=true, verbose, quiet_download, io, progress) + isnothing(progress) || progress(10000, 10000; status="verifying") + calc_hash = SHA1(GitTools.tree_hash(temp_dir)) + + # Did we get what we expected? If not, freak out. + if calc_hash.bytes != tree_hash.bytes + msg = """ + Tree Hash Mismatch! + Expected git-tree-sha1: $(bytes2hex(tree_hash.bytes)) + Calculated git-tree-sha1: $(bytes2hex(calc_hash.bytes)) + """ + # Since tree hash calculation is rather fragile and file system dependent, + # we allow setting JULIA_PKG_IGNORE_HASHES=1 to ignore the error and move + # the artifact to the expected location and return true + ignore_hash_env_set = get(ENV, "JULIA_PKG_IGNORE_HASHES", "") != "" + if ignore_hash_env_set + ignore_hash = Base.get_bool_env("JULIA_PKG_IGNORE_HASHES", false) + ignore_hash === nothing && @error( + "Invalid ENV[\"JULIA_PKG_IGNORE_HASHES\"] value", + ENV["JULIA_PKG_IGNORE_HASHES"], + ) + ignore_hash = something(ignore_hash, false) + else + # default: false except Windows users who can't symlink + ignore_hash = Sys.iswindows() && + !mktempdir(can_symlink, artifacts_dir) + end + if ignore_hash + desc = ignore_hash_env_set ? + "Environment variable \$JULIA_PKG_IGNORE_HASHES is true" : + "System is Windows and user cannot create symlinks" + msg *= "\n$desc: \ + ignoring hash mismatch and moving \ + artifact to the expected location" + @error(msg) + else + error(msg) + end end - if ignore_hash - desc = ignore_hash_env_set ? - "Environment variable \$JULIA_PKG_IGNORE_HASHES is true" : - "System is Windows and user cannot create symlinks" - msg *= "\n$desc: \ - ignoring hash mismatch and moving \ - artifact to the expected location" - @error(msg) - else - error(msg) + # Move it to the location we expected + isnothing(progress) || progress(10000, 10000; status="moving to artifact store") + mv_temp_dir_retries(temp_dir, dst) + catch err + @debug "download_artifact error" tree_hash tarball_url tarball_hash err + if isa(err, InterruptException) + rethrow(err) + end + # If something went wrong during download, return the error + return err + finally + # Always attempt to cleanup + try + rm(temp_dir; recursive=true, force=true) + catch e + e isa InterruptException && rethrow() + @warn("Failed to clean up temporary directory $(repr(temp_dir))", exception=e) end end - # Move it to the location we expected - isnothing(progress) || progress(10000, 10000; status="moving to artifact store") - _mv_temp_artifact_dir(temp_dir, dst) - catch err - @debug "download_artifact error" tree_hash tarball_url tarball_hash err - if isa(err, InterruptException) - rethrow(err) - end - # If something went wrong during download, return the error - return err - finally - # Always attempt to cleanup - try - rm(temp_dir; recursive=true, force=true) - catch e - e isa InterruptException && rethrow() - @warn("Failed to clean up temporary directory $(repr(temp_dir))", exception=e) - end + return true end - return true + + return ret end """ diff --git a/src/Operations.jl b/src/Operations.jl index d5e079a687..0fa871bef3 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -2,6 +2,7 @@ module Operations +using FileWatching: FileWatching using UUIDs using Random: randstring import LibGit2, Dates, TOML @@ -9,7 +10,7 @@ import LibGit2, Dates, TOML using ..Types, ..Resolve, ..PlatformEngines, ..GitTools, ..MiniProgressBars import ..depots, ..depots1, ..devdir, ..set_readonly, ..Types.PackageEntry import ..Artifacts: ensure_artifact_installed, artifact_names, extract_all_hashes, - artifact_exists, select_downloadable_artifacts + artifact_exists, select_downloadable_artifacts, mv_temp_dir_retries using Base.BinaryPlatforms import ...Pkg import ...Pkg: pkg_server, Registry, pathrepr, can_fancyprint, printpkgstyle, stderr_f, OFFLINE_MODE @@ -757,8 +758,9 @@ function install_archive( url_success || continue # Move content to version path - !isdir(version_path) && mkpath(version_path) - mv(unpacked, version_path; force=true) + !isdir(dirname(version_path)) && mkpath(dirname(version_path)) + mv_temp_dir_retries(unpacked, version_path; set_permissions = false) + break # successful install end # Clean up and exit @@ -914,7 +916,7 @@ function download_artifacts(ctx::Context; try dstate.state = :running ret() - if !fancyprint + if !fancyprint && dstate.bar.max > 1 # if another process downloaded, then max is never set greater than 1 @lock print_lock printpkgstyle(io, :Installed, "artifact $rname $(MiniProgressBars.pkg_format_bytes(dstate.bar.max; sigdigits=1))") end catch @@ -1041,12 +1043,14 @@ end download_source(ctx::Context; readonly=true) = download_source(ctx, values(ctx.env.manifest); readonly) function download_source(ctx::Context, pkgs; readonly=true) + pidfile_stale_age = 10 # recommended value is about 3-5x an estimated normal download time (i.e. 2-3s) pkgs_to_install = NamedTuple{(:pkg, :urls, :path), Tuple{eltype(pkgs), Set{String}, String}}[] for pkg in pkgs tracking_registered_version(pkg, ctx.julia_version) || continue path = source_path(ctx.env.manifest_file, pkg, ctx.julia_version) path === nothing && continue - ispath(path) && continue + mkpath(dirname(path)) # the `packages/Package` dir needs to exist for the pidfile to be created + FileWatching.mkpidlock(() -> ispath(path), path * ".pid", stale_age = pidfile_stale_age) && continue urls = find_urls(ctx.registries, pkg.uuid) push!(pkgs_to_install, (;pkg, urls, path)) end @@ -1069,7 +1073,8 @@ function download_source(ctx::Context, pkgs; readonly=true) nothing end - @sync begin + # use eager throw version + Base.Experimental.@sync begin jobs = Channel{eltype(pkgs_to_install)}(ctx.num_concurrent_downloads) results = Channel(ctx.num_concurrent_downloads) @@ -1079,14 +1084,19 @@ function download_source(ctx::Context, pkgs; readonly=true) end end - for i in 1:ctx.num_concurrent_downloads + for i in 1:ctx.num_concurrent_downloads # (default 8) @async begin for (pkg, urls, path) in jobs - if ctx.use_git_for_all_downloads - put!(results, (pkg, false, (urls, path))) - continue - end - try + mkpath(dirname(path)) # the `packages/Package` dir needs to exist for the pidfile to be created + FileWatching.mkpidlock(path * ".pid", stale_age = pidfile_stale_age) do + if ispath(path) + put!(results, (pkg, nothing, (urls, path))) + return + end + if ctx.use_git_for_all_downloads + put!(results, (pkg, false, (urls, path))) + return + end archive_urls = Pair{String,Bool}[] # Check if the current package is available in one of the registries being tracked by the pkg server # In that case, download from the package server @@ -1106,16 +1116,18 @@ function download_source(ctx::Context, pkgs; readonly=true) url = get_archive_url_for_version(repo_url, pkg.tree_hash) url !== nothing && push!(archive_urls, url => false) end - success = install_archive(archive_urls, pkg.tree_hash, path, io=ctx.io) - if success && readonly - set_readonly(path) # In add mode, files should be read-only - end - if ctx.use_only_tarballs_for_downloads && !success - pkgerror("failed to get tarball from $(urls)") + try + success = install_archive(archive_urls, pkg.tree_hash, path, io=ctx.io) + if success && readonly + set_readonly(path) # In add mode, files should be read-only + end + if ctx.use_only_tarballs_for_downloads && !success + pkgerror("failed to get tarball from $(urls)") + end + put!(results, (pkg, success, (urls, path))) + catch err + put!(results, (pkg, err, catch_backtrace())) end - put!(results, (pkg, success, (urls, path))) - catch err - put!(results, (pkg, err, catch_backtrace())) end end end @@ -1127,10 +1139,15 @@ function download_source(ctx::Context, pkgs; readonly=true) fancyprint = can_fancyprint(ctx.io) try for i in 1:length(pkgs_to_install) - pkg::eltype(pkgs), exc_or_success, bt_or_pathurls = take!(results) - exc_or_success isa Exception && pkgerror("Error when installing package $(pkg.name):\n", - sprint(Base.showerror, exc_or_success, bt_or_pathurls)) - success, (urls, path) = exc_or_success, bt_or_pathurls + pkg::eltype(pkgs), exc_or_success_or_nothing, bt_or_pathurls = take!(results) + if exc_or_success_or_nothing isa Exception + exc = exc_or_success_or_nothing + pkgerror("Error when installing package $(pkg.name):\n", sprint(Base.showerror, exc, bt_or_pathurls)) + end + if exc_or_success_or_nothing === nothing + continue # represents when another process did the install + end + success, (urls, path) = exc_or_success_or_nothing, bt_or_pathurls success || push!(missed_packages, (; pkg, urls, path)) bar.current = i str = sprint(; context=ctx.io) do io @@ -1158,15 +1175,18 @@ function download_source(ctx::Context, pkgs; readonly=true) # Use LibGit2 to download any remaining packages # ################################################## for (pkg, urls, path) in missed_packages - install_git(ctx.io, pkg.uuid, pkg.name, pkg.tree_hash, urls, path) - readonly && set_readonly(path) - vstr = if pkg.version !== nothing - "v$(pkg.version)" - else - short_treehash = string(pkg.tree_hash)[1:16] - "[$short_treehash]" + FileWatching.mkpidlock(path * ".pid", stale_age = pidfile_stale_age) do + ispath(path) && return + install_git(ctx.io, pkg.uuid, pkg.name, pkg.tree_hash, urls, path) + readonly && set_readonly(path) + vstr = if pkg.version !== nothing + "v$(pkg.version)" + else + short_treehash = string(pkg.tree_hash)[1:16] + "[$short_treehash]" + end + printpkgstyle(ctx.io, :Installed, string(rpad(pkg.name * " ", max_name + 2, "─"), " ", vstr)) end - printpkgstyle(ctx.io, :Installed, string(rpad(pkg.name * " ", max_name + 2, "─"), " ", vstr)) end return Set{UUID}(entry.pkg.uuid for entry in pkgs_to_install) diff --git a/src/Registry/Registry.jl b/src/Registry/Registry.jl index d5e938baa1..cf814999bb 100644 --- a/src/Registry/Registry.jl +++ b/src/Registry/Registry.jl @@ -174,108 +174,112 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depot::String=d isdir(regdir) || mkpath(regdir) # only allow one julia process to download and install registries at a time FileWatching.mkpidlock(joinpath(regdir, ".pid"), stale_age = 10) do - registry_urls = pkg_server_registry_urls() - for reg in regs - if reg.path !== nothing && reg.url !== nothing - Pkg.Types.pkgerror(""" - ambiguous registry specification; both `url` and `path` are set: - url=\"$(reg.url)\" - path=\"$(reg.path)\" - """ - ) - end - url = get(registry_urls, reg.uuid, nothing) - if url !== nothing && registry_read_from_tarball() - tmp = tempname() - try - download_verify(url, nothing, tmp) - catch err - Pkg.Types.pkgerror("could not download $url \nException: $(sprint(showerror, err))") - end - _hash = pkg_server_url_hash(url) - if !verify_archive_tree_hash(tmp, _hash) - Pkg.Types.pkgerror("unable to verify download from $url") - end - if reg.name === nothing - # Need to look up the registry name here - reg_unc = uncompress_registry(tmp) - reg.name = TOML.parse(reg_unc["Registry.toml"])["name"]::String - end - mv(tmp, joinpath(regdir, reg.name * ".tar.gz"); force=true) - reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(_hash), "path" => reg.name * ".tar.gz") - open(joinpath(regdir, reg.name * ".toml"), "w") do io - TOML.print(io, reg_info) + # once we're pidlocked check if another process has installed any of the registries + reachable_uuids = map(r -> r.uuid, reachable_registries()) + filter!(r -> !in(r.uuid, reachable_uuids), regs) + + registry_urls = pkg_server_registry_urls() + for reg in regs + if reg.path !== nothing && reg.url !== nothing + Pkg.Types.pkgerror(""" + ambiguous registry specification; both `url` and `path` are set: + url=\"$(reg.url)\" + path=\"$(reg.path)\" + """ + ) end - printpkgstyle(io, :Added, "`$(reg.name)` registry to $(Base.contractuser(regdir))") - else - mktempdir() do tmp - if reg.path !== nothing && reg.linked == true # symlink to local source - registry = Registry.RegistryInstance(reg.path) - regpath = joinpath(regdir, registry.name) - printpkgstyle(io, :Symlinking, "registry from `$(Base.contractuser(reg.path))`") - isdir(dirname(regpath)) || mkpath(dirname(regpath)) - symlink(reg.path, regpath) - isfile(joinpath(regpath, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in linked registry.") - registry = Registry.RegistryInstance(regpath) - printpkgstyle(io, :Symlinked, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`") - return - elseif reg.url !== nothing && reg.linked == true - Pkg.Types.pkgerror(""" - A symlinked registry was requested but `path` was not set and `url` was set to `$url`. - Set only `path` and `linked = true` to use registry symlinking. - """) - elseif url !== nothing && registry_use_pkg_server() - # download from Pkg server - try - download_verify_unpack(url, nothing, tmp, ignore_existence = true, io = io) - catch err - Pkg.Types.pkgerror("could not download $url \nException: $(sprint(showerror, err))") - end - tree_info_file = joinpath(tmp, ".tree_info.toml") - hash = pkg_server_url_hash(url) - write(tree_info_file, "git-tree-sha1 = " * repr(string(hash))) - elseif reg.path !== nothing # copy from local source - printpkgstyle(io, :Copying, "registry from `$(Base.contractuser(reg.path))`") - isfile(joinpath(reg.path, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in source directory.") - registry = Registry.RegistryInstance(reg.path) - regpath = joinpath(regdir, registry.name) - cp(reg.path, regpath; force=true) # has to be cp given we're copying - printpkgstyle(io, :Copied, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`") - return - elseif reg.url !== nothing # clone from url - # retry to help spurious connection issues, particularly on CI - repo = retry(GitTools.clone, delays = fill(1.0, 5), check=(s,e)->isa(e, LibGit2.GitError))(io, reg.url, tmp; header = "registry from $(repr(reg.url))") - LibGit2.close(repo) - else - Pkg.Types.pkgerror("no path or url specified for registry") + url = get(registry_urls, reg.uuid, nothing) + if url !== nothing && registry_read_from_tarball() + tmp = tempname() + try + download_verify(url, nothing, tmp) + catch err + Pkg.Types.pkgerror("could not download $url \nException: $(sprint(showerror, err))") + end + _hash = pkg_server_url_hash(url) + if !verify_archive_tree_hash(tmp, _hash) + Pkg.Types.pkgerror("unable to verify download from $url") + end + if reg.name === nothing + # Need to look up the registry name here + reg_unc = uncompress_registry(tmp) + reg.name = TOML.parse(reg_unc["Registry.toml"])["name"]::String end - # verify that the clone looks like a registry - if !isfile(joinpath(tmp, "Registry.toml")) - Pkg.Types.pkgerror("no `Registry.toml` file in cloned registry.") + mv(tmp, joinpath(regdir, reg.name * ".tar.gz"); force=true) + reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(_hash), "path" => reg.name * ".tar.gz") + open(joinpath(regdir, reg.name * ".toml"), "w") do io + TOML.print(io, reg_info) end - registry = Registry.RegistryInstance(tmp) - regpath = joinpath(regdir, registry.name) - # copy to `depot` - ispath(dirname(regpath)) || mkpath(dirname(regpath)) - if isfile(joinpath(regpath, "Registry.toml")) - existing_registry = Registry.RegistryInstance(regpath) - if registry.uuid == existing_registry.uuid - println(io, - "Registry `$(registry.name)` already exists in `$(Base.contractuser(regpath))`.") + printpkgstyle(io, :Added, "`$(reg.name)` registry to $(Base.contractuser(regdir))") + else + mktempdir() do tmp + if reg.path !== nothing && reg.linked == true # symlink to local source + registry = Registry.RegistryInstance(reg.path) + regpath = joinpath(regdir, registry.name) + printpkgstyle(io, :Symlinking, "registry from `$(Base.contractuser(reg.path))`") + isdir(dirname(regpath)) || mkpath(dirname(regpath)) + symlink(reg.path, regpath) + isfile(joinpath(regpath, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in linked registry.") + registry = Registry.RegistryInstance(regpath) + printpkgstyle(io, :Symlinked, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`") + return + elseif reg.url !== nothing && reg.linked == true + Pkg.Types.pkgerror(""" + A symlinked registry was requested but `path` was not set and `url` was set to `$url`. + Set only `path` and `linked = true` to use registry symlinking. + """) + elseif url !== nothing && registry_use_pkg_server() + # download from Pkg server + try + download_verify_unpack(url, nothing, tmp, ignore_existence = true, io = io) + catch err + Pkg.Types.pkgerror("could not download $url \nException: $(sprint(showerror, err))") + end + tree_info_file = joinpath(tmp, ".tree_info.toml") + hash = pkg_server_url_hash(url) + write(tree_info_file, "git-tree-sha1 = " * repr(string(hash))) + elseif reg.path !== nothing # copy from local source + printpkgstyle(io, :Copying, "registry from `$(Base.contractuser(reg.path))`") + isfile(joinpath(reg.path, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in source directory.") + registry = Registry.RegistryInstance(reg.path) + regpath = joinpath(regdir, registry.name) + cp(reg.path, regpath; force=true) # has to be cp given we're copying + printpkgstyle(io, :Copied, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`") + return + elseif reg.url !== nothing # clone from url + # retry to help spurious connection issues, particularly on CI + repo = retry(GitTools.clone, delays = fill(1.0, 5), check=(s,e)->isa(e, LibGit2.GitError))(io, reg.url, tmp; header = "registry from $(repr(reg.url))") + LibGit2.close(repo) else - throw(Pkg.Types.PkgError("registry `$(registry.name)=\"$(registry.uuid)\"` conflicts with " * - "existing registry `$(existing_registry.name)=\"$(existing_registry.uuid)\"`. " * - "To install it you can clone it manually into e.g. " * - "`$(Base.contractuser(joinpath(regdir, registry.name*"-2")))`.")) + Pkg.Types.pkgerror("no path or url specified for registry") + end + # verify that the clone looks like a registry + if !isfile(joinpath(tmp, "Registry.toml")) + Pkg.Types.pkgerror("no `Registry.toml` file in cloned registry.") + end + registry = Registry.RegistryInstance(tmp) + regpath = joinpath(regdir, registry.name) + # copy to `depot` + ispath(dirname(regpath)) || mkpath(dirname(regpath)) + if isfile(joinpath(regpath, "Registry.toml")) + existing_registry = Registry.RegistryInstance(regpath) + if registry.uuid == existing_registry.uuid + println(io, + "Registry `$(registry.name)` already exists in `$(Base.contractuser(regpath))`.") + else + throw(Pkg.Types.PkgError("registry `$(registry.name)=\"$(registry.uuid)\"` conflicts with " * + "existing registry `$(existing_registry.name)=\"$(existing_registry.uuid)\"`. " * + "To install it you can clone it manually into e.g. " * + "`$(Base.contractuser(joinpath(regdir, registry.name*"-2")))`.")) + end + elseif (url !== nothing && registry_use_pkg_server()) || reg.linked !== true + # if the dir doesn't exist, or exists but doesn't contain a Registry.toml + mv(tmp, regpath, force=true) + printpkgstyle(io, :Added, "registry `$(registry.name)` to `$(Base.contractuser(regpath))`") end - elseif (url !== nothing && registry_use_pkg_server()) || reg.linked !== true - # if the dir doesn't exist, or exists but doesn't contain a Registry.toml - mv(tmp, regpath, force=true) - printpkgstyle(io, :Added, "registry `$(registry.name)` to `$(Base.contractuser(regpath))`") end end end - end end # mkpidlock return nothing end diff --git a/src/utils.jl b/src/utils.jl index 12826de397..2daed7b5e1 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -61,6 +61,58 @@ function set_readonly(path) end set_readonly(::Nothing) = nothing +""" + mv_temp_dir_retries(temp_dir::String, new_path::String; set_permissions::Bool=true)::Nothing + +Either rename the directory at `temp_dir` to `new_path` and set it to read-only +or if `new_path` already exists try to do nothing. +""" +function mv_temp_dir_retries(temp_dir::String, new_path::String; set_permissions::Bool=true)::Nothing + # Sometimes a rename can fail because the temp_dir is locked by + # anti-virus software scanning the new files. + # In this case we want to sleep and try again. + # I am using the list of error codes to retry from: + # https://github.com/isaacs/node-graceful-fs/blob/234379906b7d2f4c9cfeb412d2516f42b0fb4953/polyfills.js#L87 + # Retry for up to about 60 seconds by retrying 20 times with exponential backoff. + retry = 0 + max_num_retries = 20 # maybe this should be configurable? + sleep_amount = 0.01 # seconds + max_sleep_amount = 5.0 # seconds + while true + isdir(new_path) && return + # This next step is like + # `mv(temp_dir, new_path)`. + # However, `mv` defaults to `cp` if `rename` returns an error. + # `cp` is not atomic, so avoid the potential of calling it. + err = ccall(:jl_fs_rename, Int32, (Cstring, Cstring), temp_dir, new_path) + if err ≥ 0 + if set_permissions + # rename worked + new_path_mode = filemode(dirname(new_path)) + if Sys.iswindows() + # If this is Windows, ensure the directory mode is executable, + # as `filemode()` is incomplete. Some day, that may not be the + # case, there exists a test that will fail if this is changes. + new_path_mode |= 0o111 + end + chmod(new_path, new_path_mode) + set_readonly(new_path) + end + return + else + # Ignore rename error if `new_path` exists. + isdir(new_path) && return + if retry < max_num_retries && err ∈ (Base.UV_EACCES, Base.UV_EPERM, Base.UV_EBUSY) + sleep(sleep_amount) + sleep_amount = min(sleep_amount*2.0, max_sleep_amount) + retry += 1 + else + Base.uv_error("rename of $(repr(temp_dir)) to $(repr(new_path))", err) + end + end + end +end + # try to call realpath on as much as possible function safe_realpath(path) isempty(path) && return path diff --git a/test/new.jl b/test/new.jl index 2540f99bc8..c92d9fa0fa 100644 --- a/test/new.jl +++ b/test/new.jl @@ -141,6 +141,99 @@ Pkg._auto_gc_enabled[] = false end end +function copy_this_pkg_cache(new_depot) + source = joinpath(Base.DEPOT_PATH[1], "compiled", "v$(VERSION.major).$(VERSION.minor)", "Pkg") + dest = joinpath(new_depot, "compiled", "v$(VERSION.major).$(VERSION.minor)", "Pkg") + mkpath(dirname(dest)) + cp(source, dest) +end + +function kill_with_info(p) + if Sys.islinux() + SIGINFO = 10 + elseif Sys.isbsd() + SIGINFO = 29 + end + if @isdefined(SIGINFO) + kill(p, SIGINFO) + timedwait(()->process_exited(p), 20; pollint = 1.0) # Allow time for profile to collect and print before killing + end + kill(p) + wait(p) + nothing +end + +# This test tests that multiple julia processes can install within same depot concurrently without +# corrupting the depot and being able to load the package. Only one process will do each of these, others will wait on +# the specific action for the specific thing: +# - Install the default registries +# - Install source of package and deps +# - Install artifacts +# - Precompile package and deps +# - Load & use package +@testset "Concurrent setup/installation/precompilation across processes" begin + @testset for test in 1:1 # increase for stress testing + mktempdir() do tmp + copy_this_pkg_cache(tmp) + pathsep = Sys.iswindows() ? ";" : ":" + Pkg_dir = dirname(@__DIR__) + withenv("JULIA_DEPOT_PATH" => string(tmp, pathsep)) do + script = """ + using Dates + t = Timer(t->println(Dates.now()), 0; interval = 30) + import Pkg + samefile(pkgdir(Pkg), $(repr(Pkg_dir))) || error("Using wrong Pkg") + Pkg.activate(temp=true) + Pkg.add(name="FFMPEG", version="0.4") # a package with a lot of deps but fast to load + using FFMPEG + @showtime FFMPEG.exe("-version") + @showtime FFMPEG.exe("-f", "lavfi", "-i", "testsrc=duration=1:size=128x128:rate=10", "-f", "null", "-") # more complete quick test (~10ms) + close(t) + """ + cmd = `$(Base.julia_cmd()) --project=$(dirname(@__DIR__)) --startup-file=no --color=no -e $script` + did_install_package = Threads.Atomic{Int}(0) + did_install_artifact = Threads.Atomic{Int}(0) + any_failed = Threads.Atomic{Bool}(false) + outputs = fill("", 3) + t = @elapsed @sync begin + # All but 1 process should be waiting, so should be ok to run many + for i in 1:3 + Threads.@spawn begin + iob = IOBuffer() + start = time() + p = run(pipeline(cmd, stdout=iob, stderr=iob), wait=false) + if timedwait(() -> process_exited(p), 5*60; pollint = 1.0) === :timed_out + kill_with_info(p) + end + if !success(p) + Threads.atomic_cas!(any_failed, false, true) + end + str = String(take!(iob)) + if occursin(r"Installed FFMPEG ─", str) + Threads.atomic_add!(did_install_package, 1) + end + if occursin(r"Installed artifact FFMPEG ", str) + Threads.atomic_add!(did_install_artifact, 1) + end + outputs[i] = string("=== test $test, process $i. Took $(time() - start) seconds.\n", str) + end + end + end + if any_failed[] + println("=== Concurrent Pkg.add test $test failed after $t seconds") + for i in 1:3 + printstyled(stdout, outputs[i]; color=(:blue, :green, :yellow)[i]) + end + end + # only 1 should have actually installed FFMPEG + @test !any_failed[] + @test did_install_package[] == 1 + @test did_install_artifact[] == 1 + end + end + end +end + # # ## Sandboxing # @@ -2069,7 +2162,7 @@ end end end end - end + end end # diff --git a/test/runtests.jl b/test/runtests.jl index 1d1d258174..4afbbdc9ef 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -75,10 +75,11 @@ Logging.with_logger((islogging || Pkg.DEFAULT_IO[] == devnull) ? Logging.Console push!(test_files, "aqua.jl") end - @testset "Pkg" begin + verbose = true + @testset "Pkg" verbose=verbose begin Pkg.activate(; temp=true) # make sure we're in an active project and that it's clean try - @testset "$f" for f in test_files + @testset "$f" verbose=verbose for f in test_files @info "==== Testing `test/$f`" flush(Pkg.DEFAULT_IO[]) include(f) From 6fa32d5c301f1b3165ddeb983b42f49236390c36 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 1 Mar 2025 16:12:37 -0500 Subject: [PATCH 033/154] use temp dir on same filesystem for source install (#4180) --- src/Operations.jl | 5 ++++- src/utils.jl | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Operations.jl b/src/Operations.jl index 0fa871bef3..a174f45cc7 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -717,10 +717,13 @@ function install_archive( version_path::String; io::IO=stderr_f() )::Bool + depot_temp = mkpath(joinpath(dirname(dirname(dirname(version_path))), "tmp")) tmp_objects = String[] url_success = false for (url, top) in urls - path = tempname() * randstring(6) + # the temp dir should be in the same depot because the `rename` operation in `mv_temp_dir_retries` + # is possible only if the source and destination are on the same filesystem + path = tempname(depot_temp) * randstring(6) push!(tmp_objects, path) # for cleanup url_success = true try diff --git a/src/utils.jl b/src/utils.jl index 2daed7b5e1..25600a327a 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -65,7 +65,8 @@ set_readonly(::Nothing) = nothing mv_temp_dir_retries(temp_dir::String, new_path::String; set_permissions::Bool=true)::Nothing Either rename the directory at `temp_dir` to `new_path` and set it to read-only -or if `new_path` already exists try to do nothing. +or if `new_path` already exists try to do nothing. Both `temp_dir` and `new_path` must +be on the same filesystem. """ function mv_temp_dir_retries(temp_dir::String, new_path::String; set_permissions::Bool=true)::Nothing # Sometimes a rename can fail because the temp_dir is locked by From 61115bee77676bc88a66d8694539a0d1fa35911f Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 1 Mar 2025 19:14:57 -0500 Subject: [PATCH 034/154] fix the right temp dir (#4181) --- src/Operations.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Operations.jl b/src/Operations.jl index a174f45cc7..4c64d73908 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -721,9 +721,7 @@ function install_archive( tmp_objects = String[] url_success = false for (url, top) in urls - # the temp dir should be in the same depot because the `rename` operation in `mv_temp_dir_retries` - # is possible only if the source and destination are on the same filesystem - path = tempname(depot_temp) * randstring(6) + path = tempname() * randstring(6) push!(tmp_objects, path) # for cleanup url_success = true try @@ -733,7 +731,9 @@ function install_archive( url_success = false end url_success || continue - dir = joinpath(tempdir(), randstring(12)) + # the temp dir should be in the same depot because the `rename` operation in `mv_temp_dir_retries` + # is possible only if the source and destination are on the same filesystem + dir = tempname(depot_temp) * randstring(6) push!(tmp_objects, dir) # for cleanup # Might fail to extract an archive (https://github.com/JuliaPackaging/PkgServer.jl/issues/126) try From 60e3185e38b1d49425fb3396b0fc9fe2ab1a3ab8 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 1 Mar 2025 23:29:09 -0500 Subject: [PATCH 035/154] Test fix: handle if tested Pkg is shipped version (#4182) --- test/new.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/new.jl b/test/new.jl index c92d9fa0fa..6b6a96abfd 100644 --- a/test/new.jl +++ b/test/new.jl @@ -143,6 +143,7 @@ end function copy_this_pkg_cache(new_depot) source = joinpath(Base.DEPOT_PATH[1], "compiled", "v$(VERSION.major).$(VERSION.minor)", "Pkg") + isdir(source) || return # doesn't exist if using shipped Pkg (e.g. Julia CI) dest = joinpath(new_depot, "compiled", "v$(VERSION.major).$(VERSION.minor)", "Pkg") mkpath(dirname(dest)) cp(source, dest) From c0ba6947aeec329a52d50d6a04085cee90df3663 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 7 Mar 2025 12:26:39 -0500 Subject: [PATCH 036/154] rename depot tmp dir to temp (#4183) --- src/Operations.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Operations.jl b/src/Operations.jl index 4c64d73908..e9f34f8753 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -717,7 +717,7 @@ function install_archive( version_path::String; io::IO=stderr_f() )::Bool - depot_temp = mkpath(joinpath(dirname(dirname(dirname(version_path))), "tmp")) + depot_temp = mkpath(joinpath(dirname(dirname(dirname(version_path))), "temp")) tmp_objects = String[] url_success = false for (url, top) in urls From c9d161f62d705905abd705c378548eed334ebf54 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 14 Mar 2025 07:53:07 -0400 Subject: [PATCH 037/154] store temp packages in packages/temp (#4188) --- src/Operations.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Operations.jl b/src/Operations.jl index e9f34f8753..812832a895 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -717,7 +717,11 @@ function install_archive( version_path::String; io::IO=stderr_f() )::Bool - depot_temp = mkpath(joinpath(dirname(dirname(dirname(version_path))), "temp")) + # Because we use `mv_temp_dir_retries` which uses `rename` not `mv` it can fail if the temp + # files are on a different fs. So use a temp dir in the same depot dir as some systems might + # be serving different parts of the depot on different filesystems via links i.e. pkgeval does this. + depot_temp = mkpath(joinpath(dirname(dirname(version_path)), "temp")) # .julia/packages/temp + tmp_objects = String[] url_success = false for (url, top) in urls From 26bdeeeb10f3ab0c6cf7478cb8454a800b4578be Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Sat, 15 Mar 2025 16:07:30 +0100 Subject: [PATCH 038/154] `Project` construction, `Manifest` construction: improve type stability (#4187) * `Project` construction: improve type stability * `Manifest` construction: improve type stability --- src/manifest.jl | 12 +++++++++++- src/project.jl | 12 +++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/manifest.jl b/src/manifest.jl index b8042ff312..87af60823b 100644 --- a/src/manifest.jl +++ b/src/manifest.jl @@ -95,6 +95,16 @@ function read_apps(apps::Dict) return appinfos end +read_exts(::Nothing) = Dict{String, Union{String, Vector{String}}}() +function read_exts(raw::Dict{String, Any}) + exts = Dict{String, Union{String, Vector{String}}}() + for (key, val) in raw + val isa Union{String, Vector{String}} || pkgerror("Expected `ext` entry to be a `Union{String, Vector{String}}`.") + exts[key] = val + end + return exts +end + struct Stage1 uuid::UUID entry::PackageEntry @@ -197,7 +207,7 @@ function Manifest(raw::Dict{String, Any}, f_or_io::Union{String, IO})::Manifest deps = read_deps(get(info::Dict, "deps", nothing)::Union{Nothing, Dict{String, Any}, Vector{String}}) weakdeps = read_deps(get(info::Dict, "weakdeps", nothing)::Union{Nothing, Dict{String, Any}, Vector{String}}) entry.apps = read_apps(get(info::Dict, "apps", nothing)::Union{Nothing, Dict{String, Any}}) - entry.exts = get(Dict{String, String}, info, "extensions") + entry.exts = read_exts(get(info, "extensions", nothing)) catch # TODO: Should probably not unconditionally log something # @debug "Could not parse manifest entry for `$name`" f_or_io diff --git a/src/project.jl b/src/project.jl index a856ddafe0..9006936697 100644 --- a/src/project.jl +++ b/src/project.jl @@ -62,19 +62,21 @@ function read_project_deps(raw, section_name::String) pkgerror("Expected `$(section_name)` section to be a key-value list") end -read_project_targets(::Nothing, project::Project) = Dict{String,Any}() +read_project_targets(::Nothing, project::Project) = Dict{String,Vector{String}}() function read_project_targets(raw::Dict{String,Any}, project::Project) + targets = Dict{String,Vector{String}}() for (target, deps) in raw deps isa Vector{String} || pkgerror(""" Expected value for target `$target` to be a list of dependency names. """) + targets[target] = deps end - return raw + return targets end read_project_targets(raw, project::Project) = pkgerror("Expected `targets` section to be a key-value list") -read_project_apps(::Nothing, project::Project) = Dict{String,Any}() +read_project_apps(::Nothing, project::Project) = Dict{String,AppInfo}() function read_project_apps(raw::Dict{String,Any}, project::Project) other = raw appinfos = Dict{String,AppInfo}() @@ -103,10 +105,10 @@ end read_project_compat(raw, project::Project) = pkgerror("Expected `compat` section to be a key-value list") -read_project_sources(::Nothing, project::Project) = Dict{String,Any}() +read_project_sources(::Nothing, project::Project) = Dict{String,Dict{String,String}}() function read_project_sources(raw::Dict{String,Any}, project::Project) valid_keys = ("path", "url", "rev", "subdir") - sources = Dict{String,Any}() + sources = Dict{String,Dict{String,String}}() for (name, source) in raw if !(source isa AbstractDict) pkgerror("Expected `source` section to be a table") From 6ac0df4d8bb5e7cc85b344b0b55ad2ef767812f2 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Sat, 15 Mar 2025 22:09:13 -0700 Subject: [PATCH 039/154] allow authors to be a TOML array (#3710) * allow authors to be a TOML array * use CFF schema, mention Array of Tables --- docs/src/toml-files.md | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/docs/src/toml-files.md b/docs/src/toml-files.md index 016d42450b..74e606250b 100644 --- a/docs/src/toml-files.md +++ b/docs/src/toml-files.md @@ -22,13 +22,38 @@ are described below. ### The `authors` field -For a package, the optional `authors` field is a list of strings describing the -package authors, in the form `NAME `. For example: +For a package, the optional `authors` field is a TOML array describing the package authors. +Entries in the array can either be a string in the form `"NAME"` or `"NAME "`, or a table keys following the [Citation File Format schema](https://github.com/citation-file-format/citation-file-format/blob/main/schema-guide.md) for either a +[`person`](https://github.com/citation-file-format/citation-file-format/blob/main/schema-guide.md#definitionsperson) or an[`entity`](https://github.com/citation-file-format/citation-file-format/blob/main/schema-guide.md#definitionsentity). + +For example: ```toml -authors = ["Some One ", - "Foo Bar "] +authors = [ + "Some One ", + "Foo Bar ", + {given-names = "Baz", family-names = "Qux", email = "bazqux@example.com", orcid = "https://orcid.org/0000-0000-0000-0000", website = "https://github.com/bazqux"}, +] ``` +If all authors are specified by tables, it is possible to use [the TOML Array of Tables syntax](https://toml.io/en/v1.0.0#array-of-tables) +```toml +[[authors]] +given-names = "Some" +family-names = "One" +email = "someone@email.com" + +[[authors]] +given-names = "Foo" +family-names = "Bar" +email = "foo@bar.com" + +[[authors]] +given-names = "Baz" +family-names = "Qux" +email = "bazqux@example.com" +orcid = "https://orcid.org/0000-0000-0000-0000" +website = "https://github.com/bazqux" +``` ### The `name` field From ab5370466deca77cc1f377e8543cbd100ec01ef6 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 19 Mar 2025 00:20:33 -0400 Subject: [PATCH 040/154] Create dependabot.yml (#4190) --- .github/dependabot.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..d7a3ed5357 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" # Location of package manifests + schedule: + interval: "weekly" + groups: + all-actions: + patterns: + - "*" From dfd0b8327a7d3b2fdac2adc2d499a42708aed018 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 19 Mar 2025 00:21:05 -0400 Subject: [PATCH 041/154] Update codecov-action to v5 (#4191) --- .github/workflows/test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 290596a0eb..1199c21398 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -84,9 +84,10 @@ jobs: - uses: julia-actions/julia-processcoverage@v1 env: JULIA_PKG_SERVER: ${{ matrix.pkg-server }} - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v5 with: - file: lcov.info + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} docs: runs-on: ubuntu-latest timeout-minutes: 60 From b808cb1e9f21ee3965bd60b086e1c0131284d472 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Wed, 19 Mar 2025 22:49:31 -0400 Subject: [PATCH 042/154] Add a helper script for making missing Pkg tags (#4194) * add a helper script to make Pkg.jl tags * rethrow --- contrib/list_missing_pkg_tags.jl | 89 ++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 contrib/list_missing_pkg_tags.jl diff --git a/contrib/list_missing_pkg_tags.jl b/contrib/list_missing_pkg_tags.jl new file mode 100644 index 0000000000..ce95b9e517 --- /dev/null +++ b/contrib/list_missing_pkg_tags.jl @@ -0,0 +1,89 @@ +using LibGit2 + +const JULIA_REPO_URL = "https://github.com/JuliaLang/julia.git" +const JULIA_REPO_DIR = "julia" +const PKG_VERSION_PATH = "stdlib/Pkg.version" +const PKG_REPO_URL = "https://github.com/JuliaLang/Pkg.jl.git" +const PKG_REPO_DIR = "Pkg.jl" + +function checkout_or_update_repo(url, dir) + if isdir(dir) + println("Updating existing repository: $dir") + repo = LibGit2.GitRepo(dir) + LibGit2.fetch(repo) + else + println("Cloning repository: $url") + LibGit2.clone(url, dir) + end +end + +function get_tags(repo) + refs = LibGit2.ref_list(repo) + tags = filter(ref -> startswith(ref, "refs/tags/"), refs) + return sort!(replace.(tags, "refs/tags/" => "")) +end + +function is_stable_v1_release(tag) + return occursin(r"^v\d+\.\d+\.\d+$", tag) && VersionNumber(tag) >= v"1.0.0" +end + +function extract_pkg_sha1(text::AbstractString) + m = match(r"PKG_SHA1\s*=\s*([a-f0-9]{40})", text) + return m !== nothing ? m[1] : nothing +end + +function get_commit_hash_for_pkg_version(repo, tag) + try + tag_ref = LibGit2.GitReference(repo, "refs/tags/" * tag) + LibGit2.checkout!(repo, string(LibGit2.GitHash(LibGit2.peel(tag_ref)))) + version_file = joinpath(JULIA_REPO_DIR, PKG_VERSION_PATH) + if isfile(version_file) + return extract_pkg_sha1(readchomp(version_file)) + else + println("Warning: Pkg.version file missing for tag $tag") + return nothing + end + catch + println("Error processing tag $tag") + rethrow() + end +end + +tempdir = mktempdir() +cd(tempdir) do + # Update Julia repo + checkout_or_update_repo(JULIA_REPO_URL, JULIA_REPO_DIR) + julia_repo = LibGit2.GitRepo(JULIA_REPO_DIR) + + # Get Julia tags, filtering only stable releases + julia_tags = filter(is_stable_v1_release, get_tags(julia_repo)) + version_commit_map = Dict{String, String}() + + for tag in julia_tags + println("Processing Julia tag: $tag") + commit_hash = get_commit_hash_for_pkg_version(julia_repo, tag) + if commit_hash !== nothing + version_commit_map[tag] = commit_hash + end + end + + # Update Pkg.jl repo + checkout_or_update_repo(PKG_REPO_URL, PKG_REPO_DIR) + pkg_repo = LibGit2.GitRepo(PKG_REPO_DIR) + + # Get existing tags in Pkg.jl + pkg_tags = Set(get_tags(pkg_repo)) + + # Filter out versions that already exist + missing_versions = filter(v -> v ∉ pkg_tags, collect(keys(version_commit_map))) + + # Sort versions numerically + sort!(missing_versions, by=VersionNumber) + + # Generate `git tag` commands + println("\nGit tag commands for missing Pkg.jl versions:") + for version in missing_versions + commit = version_commit_map[version] + println("git tag $version $commit") + end +end From b323a38299903e613540455c044aedfab382f94f Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Tue, 25 Mar 2025 13:51:39 +0800 Subject: [PATCH 043/154] Artifacts: Add download size to `Artifacts.toml` (#4171) This shows adds an optional download size value to an artifact binding. While this value is not used anywhere yet, it could be used to create warnings when downloading large artifacts, or to feed progress meters when the server does not provide content size headers. --- src/Artifacts.jl | 66 ++++++++++++++++++++++++++++++++++++++++------- test/artifacts.jl | 11 +++++--- 2 files changed, 63 insertions(+), 14 deletions(-) diff --git a/src/Artifacts.jl b/src/Artifacts.jl index 0702e3670c..b3dfcde867 100644 --- a/src/Artifacts.jl +++ b/src/Artifacts.jl @@ -18,7 +18,7 @@ import ..Types: write_env_usage, parse_toml export create_artifact, artifact_exists, artifact_path, remove_artifact, verify_artifact, artifact_meta, artifact_hash, bind_artifact!, unbind_artifact!, download_artifact, find_artifacts_toml, ensure_artifact_installed, @artifact_str, archive_artifact, - select_downloadable_artifacts + select_downloadable_artifacts, ArtifactDownloadInfo """ create_artifact(f::Function) @@ -138,6 +138,56 @@ function archive_artifact(hash::SHA1, tarball_path::String; honor_overrides::Boo end end +""" + ArtifactDownloadInfo + +Auxilliary information about an artifact to be used with `bind_artifact!()` to give +a download location for that artifact, as well as the hash and size of that artifact. +""" +struct ArtifactDownloadInfo + # URL the artifact is available at as a gzip-compressed tarball + url::String + + # SHA256 hash of the tarball + hash::Vector{UInt8} + + # Size in bytes of the tarball. `size <= 0` means unknown. + size::Int64 + + function ArtifactDownloadInfo(url, hash::AbstractVector, size = 0) + valid_hash_len = SHA.digestlen(SHA256_CTX) + hash_len = length(hash) + if hash_len != valid_hash_len + throw(ArgumentError("Invalid hash length '$(hash_len)', must be $(valid_hash_len)")) + end + return new( + String(url), + Vector{UInt8}(hash), + Int64(size), + ) + end +end + +# Convenience constructor for string hashes +ArtifactDownloadInfo(url, hash::AbstractString, args...) = ArtifactDownloadInfo(url, hex2bytes(hash), args...) + +# Convenience constructor for legacy Tuple representation +ArtifactDownloadInfo(args::Tuple) = ArtifactDownloadInfo(args...) + +ArtifactDownloadInfo(adi::ArtifactDownloadInfo) = adi + +# Make the dict that will be embedded in the TOML +function make_dict(adi::ArtifactDownloadInfo) + ret = Dict{String,Any}( + "url" => adi.url, + "sha256" => bytes2hex(adi.hash), + ) + if adi.size > 0 + ret["size"] = adi.size + end + return ret +end + """ bind_artifact!(artifacts_toml::String, name::String, hash::SHA1; platform::Union{AbstractPlatform,Nothing} = nothing, @@ -159,7 +209,7 @@ downloaded until it is accessed via the `artifact"name"` syntax, or """ function bind_artifact!(artifacts_toml::String, name::String, hash::SHA1; platform::Union{AbstractPlatform,Nothing} = nothing, - download_info::Union{Vector{<:Tuple},Nothing} = nothing, + download_info::Union{Vector{<:Tuple},Vector{<:ArtifactDownloadInfo},Nothing} = nothing, lazy::Bool = false, force::Bool = false) # First, check to see if this artifact is already bound: @@ -188,15 +238,11 @@ function bind_artifact!(artifacts_toml::String, name::String, hash::SHA1; meta["lazy"] = true end - # Integrate download info, if it is given. We represent the download info as a - # vector of dicts, each with its own `url` and `sha256`, since different tarballs can - # expand to the same tree hash. + # Integrate download info, if it is given. Note that there can be multiple + # download locations, each with its own tarball with its own hash, but which + # expands to the same content/treehash. if download_info !== nothing - meta["download"] = [ - Dict("url" => dl[1], - "sha256" => dl[2], - ) for dl in download_info - ] + meta["download"] = make_dict.(ArtifactDownloadInfo.(download_info)) end if platform === nothing diff --git a/test/artifacts.jl b/test/artifacts.jl index fb9c6e1a66..4f180e5f51 100644 --- a/test/artifacts.jl +++ b/test/artifacts.jl @@ -238,8 +238,8 @@ end # Test platform-specific binding and providing download_info download_info = [ - ("http://google.com/hello_world", "0"^64), - ("http://microsoft.com/hello_world", "a"^64), + ArtifactDownloadInfo("http://google.com/hello_world", "0"^64), + ArtifactDownloadInfo("http://microsoft.com/hello_world", "a"^64, 1), ] # First, test the binding of things with various platforms and overwriting and such works properly @@ -249,8 +249,8 @@ end @test artifact_hash("foo_txt", artifacts_toml; platform=linux64) == hash @test artifact_hash("foo_txt", artifacts_toml; platform=Platform("x86_64", "macos")) == nothing @test_throws ErrorException bind_artifact!(artifacts_toml, "foo_txt", hash2; download_info=download_info, platform=linux64) - bind_artifact!(artifacts_toml, "foo_txt", hash2; download_info=download_info, platform=linux64, force=true) bind_artifact!(artifacts_toml, "foo_txt", hash; download_info=download_info, platform=win32) + bind_artifact!(artifacts_toml, "foo_txt", hash2; download_info=download_info, platform=linux64, force=true) @test artifact_hash("foo_txt", artifacts_toml; platform=linux64) == hash2 @test artifact_hash("foo_txt", artifacts_toml; platform=win32) == hash @test ensure_artifact_installed("foo_txt", artifacts_toml; platform=linux64) == artifact_path(hash2) @@ -259,7 +259,9 @@ end # Next, check that we can get the download_info properly: meta = artifact_meta("foo_txt", artifacts_toml; platform=win32) @test meta["download"][1]["url"] == "http://google.com/hello_world" + @test !haskey(meta["download"][1], "size") @test meta["download"][2]["sha256"] == "a"^64 + @test meta["download"][2]["size"] == 1 rm(artifacts_toml) @@ -419,11 +421,12 @@ end ) disengaged_platform = HostPlatform() disengaged_platform["flooblecrank"] = "disengaged" + disengaged_adi = ArtifactDownloadInfo(disengaged_url, disengaged_sha256) Pkg.Artifacts.bind_artifact!( artifacts_toml, "gooblebox", disengaged_hash; - download_info = [(disengaged_url, disengaged_sha256)], + download_info = [disengaged_adi], platform = disengaged_platform, ) end From 93fb66db1834ddf432141a4809b2b6395b5d3260 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 28 Mar 2025 23:46:55 -0400 Subject: [PATCH 044/154] help debug freebsd test failure (#4199) --- test/new.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/new.jl b/test/new.jl index 6b6a96abfd..9fd9466279 100644 --- a/test/new.jl +++ b/test/new.jl @@ -220,7 +220,7 @@ end end end end - if any_failed[] + if any_failed[] || did_install_package[] != 1 || did_install_artifact[] != 1 println("=== Concurrent Pkg.add test $test failed after $t seconds") for i in 1:3 printstyled(stdout, outputs[i]; color=(:blue, :green, :yellow)[i]) From 895e9d35ef14ec93f54f16e485d6d6cec757b313 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sun, 30 Mar 2025 15:46:17 -0400 Subject: [PATCH 045/154] Lock io within printpkgstyle to avoid print tearing (#4202) --- src/utils.jl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 25600a327a..b71b802617 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -1,9 +1,12 @@ +# "Precompiling" is the longest operation +const pkgstyle_indent = textwidth(string(:Precompiling)) function printpkgstyle(io::IO, cmd::Symbol, text::String, ignore_indent::Bool=false; color=:green) - indent = textwidth(string(:Precompiling)) # "Precompiling" is the longest operation - ignore_indent && (indent = 0) - printstyled(io, lpad(string(cmd), indent), color=color, bold=true) - println(io, " ", text) + indent = ignore_indent ? 0 : pkgstyle_indent + @lock io begin + printstyled(io, lpad(string(cmd), indent), color=color, bold=true) + println(io, " ", text) + end end function linewrap(str::String; io = stdout_f(), padding = 0, width = Base.displaysize(io)[2]) From 6f309f17f493a9e4fee4b7c1fcb1b6abfdb3bb17 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sun, 30 Mar 2025 15:47:26 -0400 Subject: [PATCH 046/154] fix eagerly stopping hint completions (#4201) --- ext/REPLExt/completions.jl | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/ext/REPLExt/completions.jl b/ext/REPLExt/completions.jl index 2a5e625116..3a18512834 100644 --- a/ext/REPLExt/completions.jl +++ b/ext/REPLExt/completions.jl @@ -54,9 +54,9 @@ end const JULIA_UUID = UUID("1222c4b2-2114-5bfd-aeef-88e4692bbb3e") -function complete_remote_package(partial; hint::Bool) - found_match = false - isempty(partial) && return String[] +function complete_remote_package!(comps, partial; hint::Bool) + isempty(partial) && return true # true means returned early + found_match = !isempty(comps) cmp = Set{String}() for reg in Registry.reachable_registries() for (uuid, regpkg) in reg @@ -80,9 +80,9 @@ function complete_remote_package(partial; hint::Bool) if is_julia_compat === nothing || is_julia_compat push!(cmp, name) # In hint mode the result is only used if there is a single matching entry - # so we abort the search + # so we can return no matches in case of more than one match if hint && found_match - return sort!(collect(cmp)) + return true # true means returned early end found_match = true break @@ -91,7 +91,8 @@ function complete_remote_package(partial; hint::Bool) end end end - return sort!(collect(cmp)) + append!(comps, sort!(collect(cmp))) + return false # false means performed full search end function complete_help(options, partial; hint::Bool) @@ -149,8 +150,9 @@ function complete_add_dev(options, partial, i1, i2; hint::Bool) if occursin(Base.Filesystem.path_separator_re, partial) return comps, idx, !isempty(comps) end - comps = vcat(comps, sort(complete_remote_package(partial; hint))) - if !isempty(partial) + returned_early = complete_remote_package!(comps, partial; hint) + # returning early means that no further search should be done here + if !returned_early append!(comps, filter!(startswith(partial), [info.name for info in values(Types.stdlib_infos())])) end return comps, idx, !isempty(comps) From 5210c0dc23e2f42fd451d36eb5af2fbf9e8baa90 Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Tue, 1 Apr 2025 10:52:55 +0200 Subject: [PATCH 047/154] Use `git -C` instead of `cd` (#4206) It is simpler to not let the current Julia process change its working directory and just tell `git` where to work. (Noticed this when launching a new tmux tab which opened up with a `.julia/clones/` working diretory, for example.) --- src/GitTools.jl | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/GitTools.jl b/src/GitTools.jl index 03cc08adff..a3b2731d9d 100644 --- a/src/GitTools.jl +++ b/src/GitTools.jl @@ -169,13 +169,11 @@ function fetch(io::IO, repo::LibGit2.GitRepo, remoteurl=nothing; header=nothing, try if use_cli_git() let remoteurl=remoteurl - cd(LibGit2.path(repo)) do - cmd = `git fetch -q $remoteurl $(only(refspecs))` - try - run(pipeline(cmd; stdout=devnull)) - catch err - Pkg.Types.pkgerror("The command $(cmd) failed, error: $err") - end + cmd = `git -C $(LibGit2.path(repo)) fetch -q $remoteurl $(only(refspecs))` + try + run(pipeline(cmd; stdout=devnull)) + catch err + Pkg.Types.pkgerror("The command $(cmd) failed, error: $err") end end else From a99f05c8edb7716e9fbdb2d5ab2580932f307dc0 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 5 Apr 2025 23:22:47 -0400 Subject: [PATCH 048/154] fix flaky debug prints (#4210) --- test/new.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/new.jl b/test/new.jl index 9fd9466279..4f68bca11d 100644 --- a/test/new.jl +++ b/test/new.jl @@ -181,7 +181,7 @@ end withenv("JULIA_DEPOT_PATH" => string(tmp, pathsep)) do script = """ using Dates - t = Timer(t->println(Dates.now()), 0; interval = 30) + t = Timer(t->println(stderr, Dates.now()), 4*60; interval = 10) import Pkg samefile(pkgdir(Pkg), $(repr(Pkg_dir))) || error("Using wrong Pkg") Pkg.activate(temp=true) From 5432fdd30f06707ee227237224b0b72168c2c2f5 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sun, 6 Apr 2025 08:03:35 -0400 Subject: [PATCH 049/154] Don't warn about git General if people are opting out of the pkgserver explicitly (#4207) --- src/Registry/Registry.jl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Registry/Registry.jl b/src/Registry/Registry.jl index cf814999bb..32e9364050 100644 --- a/src/Registry/Registry.jl +++ b/src/Registry/Registry.jl @@ -451,7 +451,10 @@ function update(regs::Vector{RegistrySpec}; io::IO=stderr_f(), force::Bool=true, registry_update_log[string(reg.uuid)] = now() @label done_tarball_read else - if reg.name == "General" && Base.get_bool_env("JULIA_PKG_GEN_REG_FMT_CHECK", true) + if reg.name == "General" && + Base.get_bool_env("JULIA_PKG_GEN_REG_FMT_CHECK", true) && + get(ENV, "JULIA_PKG_SERVER", nothing) != "" + # warn if JULIA_PKG_SERVER is set to a non-empty string or not set @info """ The General registry is installed via unpacked tarball. Consider reinstalling it via the newer faster direct from @@ -478,7 +481,10 @@ function update(regs::Vector{RegistrySpec}; io::IO=stderr_f(), force::Bool=true, end elseif isdir(joinpath(reg.path, ".git")) printpkgstyle(io, :Updating, "registry at " * regpath) - if reg.name == "General" && Base.get_bool_env("JULIA_PKG_GEN_REG_FMT_CHECK", true) + if reg.name == "General" && + Base.get_bool_env("JULIA_PKG_GEN_REG_FMT_CHECK", true) && + get(ENV, "JULIA_PKG_SERVER", nothing) != "" + # warn if JULIA_PKG_SERVER is set to a non-empty string or not set @info """ The General registry is installed via git. Consider reinstalling it via the newer faster direct from tarball format by running: From bde7ce0392712e4cc16464fe1ee50cedbc43a6d4 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 9 Apr 2025 10:04:49 +0200 Subject: [PATCH 050/154] also collect packages tracking a repo when "traversing" developed packages (#4198) --- src/Operations.jl | 22 +++++++++++-------- test/sources.jl | 6 +++++ .../WithSources/TestMonorepo/Project.toml | 4 ++++ .../TestMonorepo/src/TestMonorepo.jl | 1 + .../WithSources/TestMonorepo/test/runtests.jl | 1 + .../URLSourceInDevvedPackage/Project.toml | 10 +++++++++ .../src/URLSourceInDevvedPackage.jl | 5 +++++ .../URLSourceInDevvedPackage/test/runtests.jl | 2 ++ 8 files changed, 42 insertions(+), 9 deletions(-) create mode 100644 test/test_packages/WithSources/URLSourceInDevvedPackage/Project.toml create mode 100644 test/test_packages/WithSources/URLSourceInDevvedPackage/src/URLSourceInDevvedPackage.jl create mode 100644 test/test_packages/WithSources/URLSourceInDevvedPackage/test/runtests.jl diff --git a/src/Operations.jl b/src/Operations.jl index 812832a895..3fb44792ed 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -359,18 +359,22 @@ function collect_developed!(env::EnvCache, pkg::PackageSpec, developed::Vector{P source = project_rel_path(env, source_path(env.manifest_file, pkg)) source_env = EnvCache(projectfile_path(source)) pkgs = load_project_deps(source_env.project, source_env.project_file, source_env.manifest, source_env.manifest_file) - for pkg in filter(is_tracking_path, pkgs) + for pkg in pkgs if any(x -> x.uuid == pkg.uuid, developed) continue end - # normalize path - # TODO: If path is collected from project, it is relative to the project file - # otherwise relative to manifest file.... - pkg.path = Types.relative_project_path(env.manifest_file, - project_rel_path(source_env, - source_path(source_env.manifest_file, pkg))) - push!(developed, pkg) - collect_developed!(env, pkg, developed) + if is_tracking_path(pkg) + # normalize path + # TODO: If path is collected from project, it is relative to the project file + # otherwise relative to manifest file.... + pkg.path = Types.relative_project_path(env.manifest_file, + project_rel_path(source_env, + source_path(source_env.manifest_file, pkg))) + push!(developed, pkg) + collect_developed!(env, pkg, developed) + elseif is_tracking_repo(pkg) + push!(developed, pkg) + end end end diff --git a/test/sources.jl b/test/sources.jl index e17c40d6c3..35425aea21 100644 --- a/test/sources.jl +++ b/test/sources.jl @@ -43,6 +43,12 @@ temp_pkg_dir() do project_path Pkg.test() end end + + cd(joinpath(dir, "WithSources", "URLSourceInDevvedPackage")) do + with_current_env() do + Pkg.test() + end + end end end end diff --git a/test/test_packages/WithSources/TestMonorepo/Project.toml b/test/test_packages/WithSources/TestMonorepo/Project.toml index b6a31a7b51..7c726b9389 100644 --- a/test/test_packages/WithSources/TestMonorepo/Project.toml +++ b/test/test_packages/WithSources/TestMonorepo/Project.toml @@ -3,11 +3,15 @@ uuid = "864d8eef-2526-4817-933e-34008eadd182" authors = ["KristofferC "] version = "0.1.0" +[deps] +Unregistered = "dcb67f36-efa0-11e8-0cef-2fc465ed98ae" + [extras] Example = "d359f271-ef68-451f-b4fc-6b43e571086c" [sources] Example = {url = "https://github.com/JuliaLang/Pkg.jl", subdir = "test/test_packages/Example"} +Unregistered = {url = "https://github.com/00vareladavid/Unregistered.jl", rev = "1b7a462"} [targets] test = ["Example"] diff --git a/test/test_packages/WithSources/TestMonorepo/src/TestMonorepo.jl b/test/test_packages/WithSources/TestMonorepo/src/TestMonorepo.jl index e4d52f12ff..9a4aa4f8f7 100644 --- a/test/test_packages/WithSources/TestMonorepo/src/TestMonorepo.jl +++ b/test/test_packages/WithSources/TestMonorepo/src/TestMonorepo.jl @@ -1,4 +1,5 @@ module TestMonorepo +using Unregistered greet() = print("Hello World!") diff --git a/test/test_packages/WithSources/TestMonorepo/test/runtests.jl b/test/test_packages/WithSources/TestMonorepo/test/runtests.jl index 3e04fee8cc..81a7bcd223 100644 --- a/test/test_packages/WithSources/TestMonorepo/test/runtests.jl +++ b/test/test_packages/WithSources/TestMonorepo/test/runtests.jl @@ -1 +1,2 @@ using Example +using Unregistered diff --git a/test/test_packages/WithSources/URLSourceInDevvedPackage/Project.toml b/test/test_packages/WithSources/URLSourceInDevvedPackage/Project.toml new file mode 100644 index 0000000000..a73c636c7d --- /dev/null +++ b/test/test_packages/WithSources/URLSourceInDevvedPackage/Project.toml @@ -0,0 +1,10 @@ +name = "URLSourceInDevvedPackage" +uuid = "78d3b172-12ec-4a7f-9187-8bf78594552a" +version = "0.1.0" +authors = ["Kristoffer "] + +[deps] +TestMonorepo = "864d8eef-2526-4817-933e-34008eadd182" + +[sources] +TestMonorepo = {path = "../TestMonorepo"} diff --git a/test/test_packages/WithSources/URLSourceInDevvedPackage/src/URLSourceInDevvedPackage.jl b/test/test_packages/WithSources/URLSourceInDevvedPackage/src/URLSourceInDevvedPackage.jl new file mode 100644 index 0000000000..e1de92b8a8 --- /dev/null +++ b/test/test_packages/WithSources/URLSourceInDevvedPackage/src/URLSourceInDevvedPackage.jl @@ -0,0 +1,5 @@ +module URLSourceInDevvedPackage + +greet() = print("Hello World!") + +end # module URLSourceInDevvedPackage diff --git a/test/test_packages/WithSources/URLSourceInDevvedPackage/test/runtests.jl b/test/test_packages/WithSources/URLSourceInDevvedPackage/test/runtests.jl new file mode 100644 index 0000000000..7279d9d735 --- /dev/null +++ b/test/test_packages/WithSources/URLSourceInDevvedPackage/test/runtests.jl @@ -0,0 +1,2 @@ +using URLSourceInDevvedPackage +using TestMonorepo From 4c9b08431bb4d0cf875c517e65b9193906211746 Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Fri, 11 Apr 2025 12:07:29 -0500 Subject: [PATCH 051/154] Stop using the `module Mod; using Mod; end` pattern [NFC] (#3750) * Stop using the `module Mod; using Mod; end` pattern * quality Pkg.Artifacts in tests --- src/Artifacts.jl | 6 ++++-- src/{BinaryPlatforms_compat.jl => BinaryPlatformsCompat.jl} | 6 ++++-- src/Pkg.jl | 2 +- test/artifacts.jl | 2 +- test/binaryplatforms.jl | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) rename src/{BinaryPlatforms_compat.jl => BinaryPlatformsCompat.jl} (97%) diff --git a/src/Artifacts.jl b/src/Artifacts.jl index b3dfcde867..8a9d568891 100644 --- a/src/Artifacts.jl +++ b/src/Artifacts.jl @@ -1,4 +1,4 @@ -module Artifacts +module PkgArtifacts using Artifacts, Base.BinaryPlatforms, SHA using ..MiniProgressBars, ..PlatformEngines @@ -664,4 +664,6 @@ ensure_all_artifacts_installed(artifacts_toml::AbstractString; kwargs...) = extract_all_hashes(artifacts_toml::AbstractString; kwargs...) = extract_all_hashes(string(artifacts_toml)::String; kwargs...) -end # module Artifacts +end # module PkgArtifacts + +const Artifacts = PkgArtifacts diff --git a/src/BinaryPlatforms_compat.jl b/src/BinaryPlatformsCompat.jl similarity index 97% rename from src/BinaryPlatforms_compat.jl rename to src/BinaryPlatformsCompat.jl index 879dcc0c83..05b1a6ba2e 100644 --- a/src/BinaryPlatforms_compat.jl +++ b/src/BinaryPlatformsCompat.jl @@ -1,4 +1,4 @@ -module BinaryPlatforms +module BinaryPlatformsCompat export platform_key_abi, platform_dlext, valid_dl_path, arch, libc, libgfortran_version, libstdcxx_version, cxxstring_abi, parse_dl_name_version, @@ -145,4 +145,6 @@ function valid_dl_path(path::AbstractString, platform::AbstractPlatform) end end -end # module BinaryPlatforms +end # module BinaryPlatformsCompat + +const BinaryPlatforms = BinaryPlatformsCompat diff --git a/src/Pkg.jl b/src/Pkg.jl index 8c7837d10d..53ff1e1b52 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -68,7 +68,7 @@ include("Versions.jl") include("Registry/Registry.jl") include("Resolve/Resolve.jl") include("Types.jl") -include("BinaryPlatforms_compat.jl") +include("BinaryPlatformsCompat.jl") include("Artifacts.jl") include("Operations.jl") include("API.jl") diff --git a/test/artifacts.jl b/test/artifacts.jl index 4f180e5f51..feb1fec4db 100644 --- a/test/artifacts.jl +++ b/test/artifacts.jl @@ -809,7 +809,7 @@ end cp(joinpath(@__DIR__, "test_packages", "ArtifactInstallation", "Artifacts.toml"), artifacts_toml) Pkg.activate(tmpdir) cts_real_hash = create_artifact() do dir - local meta = Artifacts.artifact_meta("collapse_the_symlink", artifacts_toml) + local meta = Pkg.Artifacts.artifact_meta("collapse_the_symlink", artifacts_toml) local collapse_url = meta["download"][1]["url"] local collapse_hash = meta["download"][1]["sha256"] # Because "BINARYPROVIDER_COPYDEREF"=>"true", this will copy symlinks. diff --git a/test/binaryplatforms.jl b/test/binaryplatforms.jl index 3400f7ff2f..38a40587d0 100644 --- a/test/binaryplatforms.jl +++ b/test/binaryplatforms.jl @@ -9,7 +9,7 @@ const platform = @inferred Platform platform_key_abi() # This is a compatibility test; once we've fully migrated away from Pkg.BinaryPlatforms # to the new Base.BinaryPlatforms module, we can throw away the shim definitions in -# `BinaryPlatforms_compat.jl` and drop these tests. +# `BinaryPlatformsCompat.jl` and drop these tests. @testset "Compat - PlatformNames" begin # Ensure the platform type constructors are well behaved @testset "Platform constructors" begin From cda9881b28653409ced37b59448f4b7ada81bdf5 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 15 Apr 2025 09:56:10 -0500 Subject: [PATCH 052/154] Fix compat example for `~0.0.3` (#4178) Co-authored-by: Viral B. Shah --- docs/src/compatibility.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/compatibility.md b/docs/src/compatibility.md index bc1c58e3e9..3aab8b75d6 100644 --- a/docs/src/compatibility.md +++ b/docs/src/compatibility.md @@ -97,7 +97,7 @@ PkgA = "~1.2.3" # [1.2.3, 1.3.0) PkgB = "~1.2" # [1.2.0, 1.3.0) PkgC = "~1" # [1.0.0, 2.0.0) PkgD = "~0.2.3" # [0.2.3, 0.3.0) -PkgE = "~0.0.3" # [0.0.3, 0.0.4) +PkgE = "~0.0.3" # [0.0.3, 0.1.0) PkgF = "~0.0" # [0.0.0, 0.1.0) PkgG = "~0" # [0.0.0, 1.0.0) ``` From 8f013c7652fd4fbbe044176a585ed0326a703338 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 15 Apr 2025 17:49:45 +0200 Subject: [PATCH 053/154] some small `Box` removals from `download_artifacts` code (#4219) --- src/Operations.jl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Operations.jl b/src/Operations.jl index 3fb44792ed..f72de21453 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -886,7 +886,7 @@ function download_artifacts(ctx::Context; download_states = Dict{SHA1, DownloadState}() errors = Channel{Any}(Inf) - is_done = false + is_done = Ref{Bool}(false) ansi_moveup(n::Int) = string("\e[", n, "A") ansi_movecol1 = "\e[1G" ansi_cleartoend = "\e[0J" @@ -903,8 +903,8 @@ function download_artifacts(ctx::Context; # For each Artifacts.toml, install each artifact we've collected from it for name in keys(artifacts) local rname = rpad(name, longest_name_length) - local hash = SHA1(artifacts[name]["git-tree-sha1"]) - local bar = MiniProgressBar(;header=rname, main=false, indent=2, color = Base.info_color(), mode=:data, always_reprint=true) + local hash = SHA1(artifacts[name]["git-tree-sha1"]::String) + local bar = MiniProgressBar(;header=rname, main=false, indent=2, color = Base.info_color()::Symbol, mode=:data, always_reprint=true) local dstate = DownloadState(:ready, "", time_ns(), Base.ReentrantLock(), bar) function progress(total, current; status="") local t = time_ns() @@ -951,9 +951,9 @@ function download_artifacts(ctx::Context; # TODO: Implement as a new MiniMultiProgressBar main_bar = MiniProgressBar(; indent=2, header = "Installing artifacts", color = :green, mode = :int, always_reprint=true) main_bar.max = length(download_states) - while !is_done + while !is_done[] main_bar.current = count(x -> x.state == :done, values(download_states)) - str = sprint(context=io) do iostr + local str = sprint(context=io) do iostr first || print(iostr, ansi_cleartoend) n_printed = 1 show_progress(iostr, main_bar; carriagereturn=false) @@ -970,7 +970,7 @@ function download_artifacts(ctx::Context; println(iostr) n_printed += 1 end - is_done || print(iostr, ansi_moveup(n_printed), ansi_movecol1) + is_done[] || print(iostr, ansi_moveup(n_printed), ansi_movecol1) first = false end print(io, str) @@ -991,26 +991,26 @@ function download_artifacts(ctx::Context; printpkgstyle(io, :Installing, "$(length(download_jobs)) artifacts") end sema = Base.Semaphore(ctx.num_concurrent_downloads) - interrupted = false + interrupted = Ref{Bool}(false) @sync for f in values(download_jobs) - interrupted && break + interrupted[] && break Base.acquire(sema) Threads.@spawn try f() catch e - e isa InterruptException && (interrupted = true) + e isa InterruptException && (interrupted[] = true) put!(errors, e) finally Base.release(sema) end end - is_done = true + is_done[] = true fancyprint && wait(t_print) close(errors) if !isempty(errors) all_errors = collect(errors) - str = sprint(context=io) do iostr + local str = sprint(context=io) do iostr for e in all_errors Base.showerror(iostr, e) length(all_errors) > 1 && println(iostr) From 7b293e2f5f435658685f8190c89766cda052f9a1 Mon Sep 17 00:00:00 2001 From: adienes <51664769+adienes@users.noreply.github.com> Date: Tue, 29 Apr 2025 14:24:39 -0400 Subject: [PATCH 054/154] Update extensions.jl (#4224) --- test/extensions.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/extensions.jl b/test/extensions.jl index 05a2a6bac6..f7bcb137d4 100644 --- a/test/extensions.jl +++ b/test/extensions.jl @@ -32,7 +32,7 @@ using UUIDs Pkg.status(; extensions=true, mode=Pkg.PKGMODE_MANIFEST, io) # TODO: Test output when ext deps are loaded etc. str = String(take!(io)) - @test contains(str, "└─ OffsetArraysExt [OffsetArrays]" ) + @test contains(str, "└─ OffsetArraysExt [OffsetArrays]") || contains(str, "├─ OffsetArraysExt [OffsetArrays]") @test !any(endswith(".cov"), readdir(joinpath(hdwe_root, "src"))) @test !any(endswith(".cov"), readdir(joinpath(he_root, "src"))) @test !any(endswith(".cov"), readdir(joinpath(he_root, "ext"))) From 495126824792e23f766132bf075dc85b9bda2a0f Mon Sep 17 00:00:00 2001 From: abhro <5664668+abhro@users.noreply.github.com> Date: Mon, 5 May 2025 08:12:18 -0400 Subject: [PATCH 055/154] Add backticks and syntax highlight tags in docstrings (#4226) --- src/Pkg.jl | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Pkg.jl b/src/Pkg.jl index 53ff1e1b52..d93269d225 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -292,7 +292,7 @@ Otherwise a feasible set of packages is resolved and installed. During the tests, test-specific dependencies are active, which are given in the project file as e.g. -``` +```toml [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" @@ -306,20 +306,20 @@ Inlining of functions during testing can be disabled (for better coverage accura by starting julia with `--inline=no`. The tests can be run as if different command line arguments were passed to julia by passing the arguments instead to the `julia_args` keyword argument, e.g. -``` +```julia Pkg.test("foo"; julia_args=["--inline"]) ``` To pass some command line arguments to be used in the tests themselves, pass the arguments to the `test_args` keyword argument. These could be used to control the code being tested, or to control the tests in some way. For example, the tests could have optional additional tests: -``` +```julia if "--extended" in ARGS @test some_function() end ``` which could be enabled by testing with -``` +```julia Pkg.test("foo"; test_args=["--extended"]) ``` """ @@ -474,14 +474,14 @@ Request a `ProjectInfo` struct which contains information about the active proje # `ProjectInfo` fields -| Field | Description | -|:-------------|:--------------------------------------------------------------------------------------------| -| name | The project's name | -| uuid | The project's UUID | -| version | The project's version | -| ispackage | Whether the project is a package (has a name and uuid) | -| dependencies | The project's direct dependencies as a `Dict` which maps dependency name to dependency UUID | -| path | The location of the project file which defines the active project | +| Field | Description | +|:---------------|:--------------------------------------------------------------------------------------------| +| `name` | The project's name | +| `uuid` | The project's UUID | +| `version` | The project's version | +| `ispackage` | Whether the project is a package (has a name and uuid) | +| `dependencies` | The project's direct dependencies as a `Dict` which maps dependency name to dependency UUID | +| `path` | The location of the project file which defines the active project | """ const project = API.project @@ -605,7 +605,7 @@ If no argument is given to `activate`, then use the first project found in `LOAD `@v#.#` environment. # Examples -``` +```julia Pkg.activate() Pkg.activate("local/path") Pkg.activate("MyDependency") From 54b46b9188924ce239f2c217dc0baeebdfcffc32 Mon Sep 17 00:00:00 2001 From: Lasse Peters Date: Mon, 5 May 2025 14:12:49 +0200 Subject: [PATCH 056/154] Re-land: Add entries to [sources] automatically if package is added by URL or devved (#4225) Co-authored-by: Kristoffer Co-authored-by: Florian Atteneder --- CHANGELOG.md | 4 +++- src/API.jl | 30 +++++++++++++++++++++--------- src/Types.jl | 8 ++++++++ test/new.jl | 22 ++++++++++++---------- test/sources.jl | 2 +- 5 files changed, 45 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f39463873..1aaac21ee3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ Pkg v1.12 Release Notes The functions `Pkg.status`, `Pkg.why`, `Pkg.instantiate`, `Pkg.precompile` (and their REPL variants) have been updated to take a `workspace` option. Read more about this feature in the manual about the TOML-files. - `status` now shows when different versions/sources of dependencies are loaded than that which is expected by the manifest ([#4109]) +- It is now possible to specify "sources" for packages in a `[sources]` section in Project.toml. + This can be used to add non-registered normal or test dependencies. Packages are also automatically added to `[sources]` when they are added by url or devved. Pkg v1.11 Release Notes ======================= @@ -22,7 +24,7 @@ Pkg v1.10 Release Notes ======================= Pkg v1.9 Release Notes -======================= +====================== - New functionality: `Pkg.why` and `pkg> why` to show why a package is inside the environment (shows all "paths" to a package starting at the direct dependencies). - When code coverage tracking is enabled for `Pkg.test` the new path-specific code-coverage option is used to limit coverage diff --git a/src/API.jl b/src/API.jl index 834fbdcef5..70a59334e3 100644 --- a/src/API.jl +++ b/src/API.jl @@ -190,15 +190,26 @@ end function update_source_if_set(project, pkg) source = get(project.sources, pkg.name, nothing) source === nothing && return - # This should probably not modify the dicts directly... - if pkg.repo.source !== nothing - source["url"] = pkg.repo.source - end - if pkg.repo.rev !== nothing - source["rev"] = pkg.repo.rev - end - if pkg.path !== nothing - source["path"] = pkg.path + if pkg.repo == GitRepo() + delete!(project.sources, pkg.name) + else + # This should probably not modify the dicts directly... + if pkg.repo.source !== nothing + source["url"] = pkg.repo.source + delete!(source, "path") + end + if pkg.repo.rev !== nothing + source["rev"] = pkg.repo.rev + delete!(source, "path") + end + if pkg.repo.subdir !== nothing + source["subdir"] = pkg.repo.subdir + end + if pkg.path !== nothing + source["path"] = pkg.path + delete!(source, "url") + delete!(source, "rev") + end end if pkg.subdir !== nothing source["subdir"] = pkg.subdir @@ -427,6 +438,7 @@ function pin(ctx::Context, pkgs::Vector{PackageSpec}; all_pkgs::Bool=false, kwar pkgerror("pinning a package requires a single version, not a versionrange") end end + update_source_if_set(ctx.env.project, pkg) end project_deps_resolve!(ctx.env, pkgs) diff --git a/src/Types.jl b/src/Types.jl index 5f72f49c96..96f951434f 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -1230,6 +1230,14 @@ function write_env(env::EnvCache; update_undo=true, @assert entry.repo.subdir == repo.subdir end end + if entry.path !== nothing + env.project.sources[pkg] = Dict("path" => entry.path) + elseif entry.repo != GitRepo() + d = Dict("url" => entry.repo.source) + entry.repo.rev !== nothing && (d["rev"] = entry.repo.rev) + entry.repo.subdir !== nothing && (d["subdir"] = entry.repo.subdir) + env.project.sources[pkg] = d + end end if (env.project != env.original_project) && (!skip_writing_project) diff --git a/test/new.jl b/test/new.jl index 4f68bca11d..8b4f4c3bc1 100644 --- a/test/new.jl +++ b/test/new.jl @@ -3248,16 +3248,18 @@ temp_pkg_dir() do project_path end end @testset "test resolve with tree hash" begin - mktempdir() do dir - path = copy_test_package(dir, "ResolveWithRev") - cd(path) do - with_current_env() do - @test !isfile("Manifest.toml") - @test !isdir(joinpath(DEPOT_PATH[1], "packages", "Example")) - Pkg.resolve() - @test isdir(joinpath(DEPOT_PATH[1], "packages", "Example")) - rm(joinpath(DEPOT_PATH[1], "packages", "Example"); recursive = true) - Pkg.resolve() + isolate() do + mktempdir() do dir + path = copy_test_package(dir, "ResolveWithRev") + cd(path) do + with_current_env() do + @test !isfile("Manifest.toml") + @test !isdir(joinpath(DEPOT_PATH[1], "packages", "Example")) + Pkg.resolve() + @test isdir(joinpath(DEPOT_PATH[1], "packages", "Example")) + rm(joinpath(DEPOT_PATH[1], "packages", "Example"); recursive=true) + Pkg.resolve() + end end end end diff --git a/test/sources.jl b/test/sources.jl index 35425aea21..aeb42313e0 100644 --- a/test/sources.jl +++ b/test/sources.jl @@ -21,7 +21,7 @@ temp_pkg_dir() do project_path cp("Project.toml.bak", "Project.toml"; force=true) cp("BadManifest.toml", "Manifest.toml"; force=true) Pkg.resolve() - @test Pkg.project().sources["Example"] == Dict("url" => "https://github.com/JuliaLang/Example.jl") + @test Pkg.project().sources["Example"] == Dict("rev" => "master", "url" => "https://github.com/JuliaLang/Example.jl") @test Pkg.project().sources["LocalPkg"] == Dict("path" => "LocalPkg") end end From 2afba2c74a087fa5832ddd462c6d37a6ff4e658a Mon Sep 17 00:00:00 2001 From: Wylie Fowler Date: Mon, 5 May 2025 08:40:31 -0400 Subject: [PATCH 057/154] Add note about character minimums (#4166) Co-authored-by: SundaraRaman R Co-authored-by: Viral B. Shah --- docs/src/creating-packages.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/src/creating-packages.md b/docs/src/creating-packages.md index 7bb72c2e91..0d7a23a286 100644 --- a/docs/src/creating-packages.md +++ b/docs/src/creating-packages.md @@ -593,6 +593,7 @@ may fit your package better. 4. Err on the side of clarity, even if clarity seems long-winded to you. * `RandomMatrices` is a less ambiguous name than `RndMat` or `RMT`, even though the latter are shorter. + * Generally package names should be at least 5 characters long not including the `.jl` extension 5. A less systematic name may suit a package that implements one of several possible approaches to its domain. @@ -621,9 +622,12 @@ may fit your package better. there's no copyright or trademark infringement etc.) 9. Packages should follow the [Stylistic Conventions](https://docs.julialang.org/en/v1/manual/variables/#Stylistic-Conventions). - * The package name begin with a capital letter and word separation is shown with upper camel case + * The package name should begin with a capital letter and word separation is shown with upper camel case * Packages that provide the functionality of a project from another language should use the Julia convention - * Packages that [provide pre-built libraries and executables](https://docs.binarybuilder.org/stable/jll/) can keep orignal name, but should get `_jll`as a suffix. For example `pandoc_jll` wraps pandoc. However, note that the generation and release of most JLL packages is handled by the [Yggdrasil](https://github.com/JuliaPackaging/Yggdrasil) system. + * Packages that [provide pre-built libraries and executables](https://docs.binarybuilder.org/stable/jll/) can keep their original name, but should get `_jll`as a suffix. For example `pandoc_jll` wraps pandoc. However, note that the generation and release of most JLL packages is handled by the [Yggdrasil](https://github.com/JuliaPackaging/Yggdrasil) system. + +10. For the complete list of rules for automatic merging into the General registry, see [these guidelines](https://juliaregistries.github.io/RegistryCI.jl/stable/guidelines/). + ## Registering packages From 244b421498a6f6e3fe3d01a83f5051bbcde342a7 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 5 May 2025 14:41:04 +0200 Subject: [PATCH 058/154] set version to 1.13.0 (#4147) --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index fe6c8ccc9d..cdc10d1205 100644 --- a/Project.toml +++ b/Project.toml @@ -3,7 +3,7 @@ uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" keywords = ["package management"] license = "MIT" desc = "The next-generation Julia package manager." -version = "1.12.0" +version = "1.13.0" [workspace] projects = ["test", "docs"] From 80d2e7505970b9f1a33b2a4e1c4c5bcea7a709b0 Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Mon, 5 May 2025 07:54:14 -0500 Subject: [PATCH 059/154] Export PkgArtifacts from itself under the name Artifacts for compat (#4230) --- src/Artifacts.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Artifacts.jl b/src/Artifacts.jl index 8a9d568891..3509887bd4 100644 --- a/src/Artifacts.jl +++ b/src/Artifacts.jl @@ -14,8 +14,8 @@ import Artifacts: artifact_names, ARTIFACTS_DIR_OVERRIDE, ARTIFACT_OVERRIDES, ar query_override, with_artifacts_directory, load_overrides import ..Types: write_env_usage, parse_toml - -export create_artifact, artifact_exists, artifact_path, remove_artifact, verify_artifact, +const Artifacts = PkgArtifacts # This is to preserve compatability for folks who depend on the internals of this module +export Artifacts, create_artifact, artifact_exists, artifact_path, remove_artifact, verify_artifact, artifact_meta, artifact_hash, bind_artifact!, unbind_artifact!, download_artifact, find_artifacts_toml, ensure_artifact_installed, @artifact_str, archive_artifact, select_downloadable_artifacts, ArtifactDownloadInfo @@ -328,7 +328,7 @@ function download_artifact( io::IO=stderr_f(), progress::Union{Function, Nothing} = nothing, ) - _artifact_paths = Artifacts.artifact_paths(tree_hash) + _artifact_paths = artifact_paths(tree_hash) pidfile = _artifact_paths[1] * ".pid" mkpath(dirname(pidfile)) t_wait_msg = Timer(2) do t From cb8875039de3200183a1c630cb54dd898faf5a54 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 12 May 2025 08:50:05 +0200 Subject: [PATCH 060/154] add some missing hash definitions (#4228) --- src/Types.jl | 7 ++++++- test/new.jl | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Types.jl b/src/Types.jl index 96f951434f..2b475d4b10 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -85,7 +85,8 @@ end Base.:(==)(r1::GitRepo, r2::GitRepo) = r1.source == r2.source && r1.rev == r2.rev && r1.subdir == r2.subdir - +Base.hash(r::GitRepo, h::UInt) = + foldr(hash, [r.source, r.rev, r.subdir], init=h) mutable struct PackageSpec name::Union{Nothing,String} @@ -119,11 +120,15 @@ PackageSpec(name::AbstractString, uuid::UUID) = PackageSpec(;name=name, uuid=uui PackageSpec(name::AbstractString, version::VersionTypes) = PackageSpec(;name=name, version=version)::PackageSpec PackageSpec(n::AbstractString, u::UUID, v::VersionTypes) = PackageSpec(;name=n, uuid=u, version=v)::PackageSpec +# XXX: These definitions are a bit fishy. It seems to be used in an `==` call in status printing function Base.:(==)(a::PackageSpec, b::PackageSpec) return a.name == b.name && a.uuid == b.uuid && a.version == b.version && a.tree_hash == b.tree_hash && a.repo == b.repo && a.path == b.path && a.pinned == b.pinned end +function Base.hash(a::PackageSpec, h::UInt) + return foldr(hash, [a.name, a.uuid, a.version, a.tree_hash, a.repo, a.path, a.pinned], init=h) +end function err_rep(pkg::PackageSpec) x = pkg.name !== nothing && pkg.uuid !== nothing ? x = "$(pkg.name) [$(string(pkg.uuid)[1:8])]" : diff --git a/test/new.jl b/test/new.jl index 8b4f4c3bc1..1554711267 100644 --- a/test/new.jl +++ b/test/new.jl @@ -3280,4 +3280,6 @@ end @test occursin("[loaded: v0.5.4]", out) end +@test allunique(unique([Pkg.PackageSpec(path="foo"), Pkg.PackageSpec(path="foo")])) + end #module From afac85ba3cfb17a55399207dd077d8e2197e5de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= <765740+giordano@users.noreply.github.com> Date: Wed, 21 May 2025 18:01:14 +0100 Subject: [PATCH 061/154] [test] Use permalinks for `ArtifactInstallation` artifacts (#4240) --- test/test_packages/ArtifactInstallation/Artifacts.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_packages/ArtifactInstallation/Artifacts.toml b/test/test_packages/ArtifactInstallation/Artifacts.toml index 798e65c7bd..e32e4c7d56 100644 --- a/test/test_packages/ArtifactInstallation/Artifacts.toml +++ b/test/test_packages/ArtifactInstallation/Artifacts.toml @@ -147,16 +147,16 @@ git-tree-sha1 = "43563e7631a7eafae1f9f8d9d332e3de44ad7239" lazy = true [[socrates.download]] -url = "https://github.com/staticfloat/small_bin/raw/master/socrates.tar.gz" +url = "https://github.com/staticfloat/small_bin/raw/91f3ecf327d1de943fe076657833252791ba9f60/socrates.tar.gz" sha256 = "e65d2f13f2085f2c279830e863292312a72930fee5ba3c792b14c33ce5c5cc58" [[socrates.download]] -url = "https://github.com/staticfloat/small_bin/raw/master/socrates.tar.bz2" +url = "https://github.com/staticfloat/small_bin/raw/91f3ecf327d1de943fe076657833252791ba9f60/socrates.tar.bz2" sha256 = "13fc17b97be41763b02cbb80e9d048302cec3bd3d446c2ed6e8210bddcd3ac76" [collapse_the_symlink] git-tree-sha1 = "69a468bd51751f4ed7eda31c240e775df06d6ee6" [[collapse_the_symlink.download]] -url = "https://github.com/staticfloat/small_bin/raw/master/collapse_the_symlink/collapse_the_symlink.tar.gz" +url = "https://github.com/staticfloat/small_bin/raw/91f3ecf327d1de943fe076657833252791ba9f60/collapse_the_symlink/collapse_the_symlink.tar.gz" sha256 = "956c1201405f64d3465cc28cb0dec9d63c11a08cad28c381e13bb22e1fc469d3" From cf2d47a70e97f66296f750a5d9daf80193b43afd Mon Sep 17 00:00:00 2001 From: Lasse Peters Date: Wed, 21 May 2025 19:02:29 +0200 Subject: [PATCH 062/154] Clean up CHANGELOG.md (#4231) --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1aaac21ee3..f5b9688414 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,7 @@ Pkg v1.12 Release Notes The functions `Pkg.status`, `Pkg.why`, `Pkg.instantiate`, `Pkg.precompile` (and their REPL variants) have been updated to take a `workspace` option. Read more about this feature in the manual about the TOML-files. - `status` now shows when different versions/sources of dependencies are loaded than that which is expected by the manifest ([#4109]) -- It is now possible to specify "sources" for packages in a `[sources]` section in Project.toml. - This can be used to add non-registered normal or test dependencies. Packages are also automatically added to `[sources]` when they are added by url or devved. +- Packages are also automatically added to `[sources]` when they are added by url or devved. Pkg v1.11 Release Notes ======================= From 1b26c92210dbaa7263cdfc03b031b6c7f97ca02e Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 21 May 2025 19:03:36 +0200 Subject: [PATCH 063/154] improve package syntax parsing a bit (#4227) --- src/REPLMode/argument_parsers.jl | 41 ++++++++++++++++++++++---------- test/new.jl | 6 +++++ 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/REPLMode/argument_parsers.jl b/src/REPLMode/argument_parsers.jl index 5d909e91ba..0901b58a1c 100644 --- a/src/REPLMode/argument_parsers.jl +++ b/src/REPLMode/argument_parsers.jl @@ -57,7 +57,7 @@ let url = raw"((git|ssh|http(s)?)|(git@[\w\-\.]+))(:(//)?)([\w\.@\:/\-~]+)(\.git name_uuid = raw"[^@\#\s:]+\s*=\s*[^@\#\s:]+", # Match a `#BRANCH` branch or tag specifier. - branch = raw"\#\s*[^@\#\s]*", + branch = raw"\#\s*[^@^:\s]+", # Match an `@VERSION` version specifier. version = raw"@\s*[^@\#\s]*", @@ -94,19 +94,34 @@ function parse_package_args(args::Vector{PackageToken}; add_or_dev=false)::Vecto # check for and apply PackageSpec modifier (e.g. `#foo` or `@v1.0.2`) function apply_modifier!(pkg::PackageSpec, args::Vector{PackageToken}) (isempty(args) || args[1] isa PackageIdentifier) && return - modifier = popfirst!(args) - if modifier isa Subdir - pkg.subdir = modifier.dir - (isempty(args) || args[1] isa PackageIdentifier) && return + parsed_subdir = false + parsed_version = false + parsed_rev = false + while !isempty(args) modifier = popfirst!(args) - end - - if modifier isa VersionToken - pkg.version = modifier.version - elseif modifier isa Rev - pkg.rev = modifier.rev - else - pkgerror("Package name/uuid must precede subdir specifier `$args`.") + if modifier isa Subdir + if parsed_subdir + pkgerror("Multiple subdir specifiers `$args` found.") + end + pkg.subdir = modifier.dir + (isempty(args) || args[1] isa PackageIdentifier) && return + modifier = popfirst!(args) + parsed_subdir = true + elseif modifier isa VersionToken + if parsed_version + pkgerror("Multiple version specifiers `$args` found.") + end + pkg.version = modifier.version + parsed_version = true + elseif modifier isa Rev + if parsed_rev + pkgerror("Multiple revision specifiers `$args` found.") + end + pkg.rev = modifier.rev + parsed_rev = true + else + pkgerror("Package name/uuid must precede subdir specifier `$args`.") + end end end diff --git a/test/new.jl b/test/new.jl index 1554711267..a3295a06ef 100644 --- a/test/new.jl +++ b/test/new.jl @@ -439,6 +439,12 @@ end arg = args[1] @test arg.url == "https://github.com/JuliaLang/Pkg.jl" @test arg.rev == "aa/gitlab" + + api, args, opts = first(Pkg.pkg"add https://github.com/TimG1964/XLSX.jl#Bug-fixing-post-#289:subdir") + arg = args[1] + @test arg.url == "https://github.com/TimG1964/XLSX.jl" + @test arg.rev == "Bug-fixing-post-#289" + @test arg.subdir == "subdir" end end From 8a887091263deeea883a22c6d490328294719946 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Wed, 21 May 2025 15:01:47 -0300 Subject: [PATCH 064/154] Make threads test more robust + fix bug parsing JULIA_NUM_THREADS (#4241) Co-authored-by: Ian Butterworth --- src/Operations.jl | 3 +++ test/new.jl | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/Operations.jl b/src/Operations.jl index f72de21453..f3ddd02e6e 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -2014,6 +2014,9 @@ end function get_threads_spec() if haskey(ENV, "JULIA_NUM_THREADS") + if isempty(ENV["JULIA_NUM_THREADS"]) + throw(ArgumentError("JULIA_NUM_THREADS is set to an empty string. It is not clear what Pkg.test should set for `-t` on the test worker.")) + end # if set, prefer JULIA_NUM_THREADS because this is passed to the test worker via --threads # which takes precedence in the worker ENV["JULIA_NUM_THREADS"] diff --git a/test/new.jl b/test/new.jl index a3295a06ef..fc3a7a75fa 100644 --- a/test/new.jl +++ b/test/new.jl @@ -2128,6 +2128,7 @@ end withenv( "EXPECTED_NUM_THREADS_DEFAULT" => "$default_nthreads_default", "EXPECTED_NUM_THREADS_INTERACTIVE" => "$default_nthreads_interactive", + "JULIA_NUM_THREADS" => nothing, ) do Pkg.test("TestThreads") end @@ -2154,6 +2155,7 @@ end withenv( "EXPECTED_NUM_THREADS_DEFAULT" => "$other_nthreads_default", "EXPECTED_NUM_THREADS_INTERACTIVE" => "$default_nthreads_interactive", + "JULIA_NUM_THREADS" => nothing, ) do Pkg.test("TestThreads"; julia_args=`--threads=$other_nthreads_default`) end @@ -2162,6 +2164,7 @@ end withenv( "EXPECTED_NUM_THREADS_DEFAULT" => "$other_nthreads_default", "EXPECTED_NUM_THREADS_INTERACTIVE" => "$other_nthreads_interactive", + "JULIA_NUM_THREADS" => nothing, ) do Pkg.test("TestThreads"; julia_args=`--threads=$other_nthreads_default,$other_nthreads_interactive`) end From 88629b552621cf8b0d5dbf3bbd5b00ecaa10d0e0 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Wed, 21 May 2025 18:13:06 -0500 Subject: [PATCH 065/154] Disallow precompiling Pkg in Pkg tests (#4176) --- ext/REPLExt/REPLExt.jl | 4 ++++ src/Pkg.jl | 5 +++++ test/new.jl | 8 -------- test/pkg.jl | 4 ++-- test/repl.jl | 18 ++++++++++-------- test/runtests.jl | 1 + test/utils.jl | 13 ++++++++++++- 7 files changed, 34 insertions(+), 19 deletions(-) diff --git a/ext/REPLExt/REPLExt.jl b/ext/REPLExt/REPLExt.jl index b7942b4a95..980f8d7a24 100644 --- a/ext/REPLExt/REPLExt.jl +++ b/ext/REPLExt/REPLExt.jl @@ -1,5 +1,9 @@ module REPLExt +if Base.get_bool_env("JULIA_PKG_DISALLOW_PKG_PRECOMPILATION", false) == true + error("Precompililing Pkg extension REPLExt is disallowed. JULIA_PKG_DISALLOW_PKG_PRECOMPILATION=$(ENV["JULIA_PKG_DISALLOW_PKG_PRECOMPILATION"])") +end + using Markdown, UUIDs, Dates import REPL diff --git a/src/Pkg.jl b/src/Pkg.jl index d93269d225..6241b394eb 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -2,6 +2,11 @@ module Pkg +# In Pkg tests we want to avoid Pkg being re-precompiled by subprocesses, so this is enabled in the test suite +if Base.get_bool_env("JULIA_PKG_DISALLOW_PKG_PRECOMPILATION", false) == true + error("Precompililing Pkg is disallowed. JULIA_PKG_DISALLOW_PKG_PRECOMPILATION=$(ENV["JULIA_PKG_DISALLOW_PKG_PRECOMPILATION"])") +end + if isdefined(Base, :Experimental) && isdefined(Base.Experimental, Symbol("@max_methods")) @eval Base.Experimental.@max_methods 1 end diff --git a/test/new.jl b/test/new.jl index fc3a7a75fa..f575c7223d 100644 --- a/test/new.jl +++ b/test/new.jl @@ -141,14 +141,6 @@ Pkg._auto_gc_enabled[] = false end end -function copy_this_pkg_cache(new_depot) - source = joinpath(Base.DEPOT_PATH[1], "compiled", "v$(VERSION.major).$(VERSION.minor)", "Pkg") - isdir(source) || return # doesn't exist if using shipped Pkg (e.g. Julia CI) - dest = joinpath(new_depot, "compiled", "v$(VERSION.major).$(VERSION.minor)", "Pkg") - mkpath(dirname(dest)) - cp(source, dest) -end - function kill_with_info(p) if Sys.islinux() SIGINFO = 10 diff --git a/test/pkg.jl b/test/pkg.jl index 2793c246e2..c5f4371efe 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -401,12 +401,12 @@ temp_pkg_dir() do project_path Sys.CPU_THREADS == 1 && error("Cannot test for atomic usage log file interaction effectively with only Sys.CPU_THREADS=1") # Precompile Pkg given we're in a different depot # and make sure the General registry is installed - Utils.show_output_if_command_errors(`$(Base.julia_cmd()[1]) --project="$(pkgdir(Pkg))" -e "import Pkg; isempty(Pkg.Registry.reachable_registries()) && Pkg.Registry.add()"`) + Utils.show_output_if_command_errors(`$(Base.julia_cmd()) --project="$(pkgdir(Pkg))" -e "import Pkg; isempty(Pkg.Registry.reachable_registries()) && Pkg.Registry.add()"`) flag_start_dir = tempdir() # once n=Sys.CPU_THREADS files are in here, the processes can proceed to the concurrent test flag_end_file = tempname() # use creating this file as a way to stop the processes early if an error happens for i in 1:Sys.CPU_THREADS iob = IOBuffer() - t = @async run(pipeline(`$(Base.julia_cmd()[1]) --project="$(pkgdir(Pkg))" + t = @async run(pipeline(`$(Base.julia_cmd()) --project="$(pkgdir(Pkg))" -e "import Pkg; Pkg.UPDATED_REGISTRY_THIS_SESSION[] = true; Pkg.activate(temp = true); diff --git a/test/repl.jl b/test/repl.jl index ce817eb20c..ed3c635a98 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -735,14 +735,16 @@ end end @testset "JuliaLang/julia #55850" begin - tmp_55850 = mktempdir() - tmp_sym_link = joinpath(tmp_55850, "sym") - symlink(tmp_55850, tmp_sym_link; dir_target=true) - depot_path = join([tmp_sym_link, Base.DEPOT_PATH...], Sys.iswindows() ? ";" : ":") - # include the symlink in the depot path and include the regular default depot so we don't precompile this Pkg again - withenv("JULIA_DEPOT_PATH" => depot_path, "JULIA_LOAD_PATH" => nothing) do - prompt = readchomp(`$(Base.julia_cmd()[1]) --project=$(dirname(@__DIR__)) --startup-file=no -e "using Pkg, REPL; Pkg.activate(io=devnull); REPLExt = Base.get_extension(Pkg, :REPLExt); print(REPLExt.promptf())"`) - @test prompt == "(@v$(VERSION.major).$(VERSION.minor)) pkg> " + mktempdir() do tmp + copy_this_pkg_cache(tmp) + tmp_sym_link = joinpath(tmp, "sym") + symlink(tmp, tmp_sym_link; dir_target=true) + depot_path = tmp_sym_link * (Sys.iswindows() ? ";" : ":") + # include the symlink in the depot path and include the regular default depot so we don't precompile this Pkg again + withenv("JULIA_DEPOT_PATH" => depot_path, "JULIA_LOAD_PATH" => nothing) do + prompt = readchomp(`$(Base.julia_cmd()) --project=$(dirname(@__DIR__)) --startup-file=no -e "using Pkg, REPL; Pkg.activate(io=devnull); REPLExt = Base.get_extension(Pkg, :REPLExt); print(REPLExt.promptf())"`) + @test prompt == "(@v$(VERSION.major).$(VERSION.minor)) pkg> " + end end end diff --git a/test/runtests.jl b/test/runtests.jl index 4afbbdc9ef..8cc8d886be 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -20,6 +20,7 @@ if realpath(dirname(dirname(Base.pathof(Pkg)))) != realpath(dirname(@__DIR__)) end ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0 +ENV["JULIA_PKG_DISALLOW_PKG_PRECOMPILATION"]=1 logdir = get(ENV, "JULIA_TEST_VERBOSE_LOGS_DIR", nothing) ### Send all Pkg output to a file called Pkg.log diff --git a/test/utils.jl b/test/utils.jl index 7bbc606ade..df1e82c7fa 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -11,7 +11,7 @@ using UUIDs export temp_pkg_dir, cd_tempdir, isinstalled, write_build, with_current_env, with_temp_env, with_pkg_env, git_init_and_commit, copy_test_package, git_init_package, add_this_pkg, TEST_SIG, TEST_PKG, isolate, LOADED_DEPOT, - list_tarball_files, recursive_rm_cov_files + list_tarball_files, recursive_rm_cov_files, copy_this_pkg_cache const CACHE_DIRECTORY = realpath(mktempdir(; cleanup = true)) @@ -22,6 +22,17 @@ const REGISTRY_DIR = joinpath(REGISTRY_DEPOT, "registries", "General") const GENERAL_UUID = UUID("23338594-aafe-5451-b93e-139f81909106") +function copy_this_pkg_cache(new_depot) + for p in ("Pkg", "REPLExt") + subdir = joinpath("compiled", "v$(VERSION.major).$(VERSION.minor)") + source = joinpath(Base.DEPOT_PATH[1], subdir, p) + isdir(source) || continue # doesn't exist if using shipped Pkg (e.g. Julia CI) + dest = joinpath(new_depot, subdir, p) + mkpath(dirname(dest)) + cp(source, dest) + end +end + function check_init_reg() isfile(joinpath(REGISTRY_DIR, "Registry.toml")) && return mkpath(REGISTRY_DIR) From 5577f68d612139693282c037d070f515bf160d1b Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 3 Jun 2025 11:52:12 -0400 Subject: [PATCH 066/154] Isolate threads test from parent state (#4243) Co-authored-by: gbaraldi --- test/new.jl | 61 +++++++++++-------- .../TestThreads/test/runtests.jl | 7 ++- test/utils.jl | 2 +- 3 files changed, 42 insertions(+), 28 deletions(-) diff --git a/test/new.jl b/test/new.jl index f575c7223d..0e4b5169cb 100644 --- a/test/new.jl +++ b/test/new.jl @@ -2111,57 +2111,66 @@ end mktempdir() do dir path = copy_test_package(dir, "TestThreads") cd(path) do - with_current_env() do - default_nthreads_default = Threads.nthreads(:default) - default_nthreads_interactive = Threads.nthreads(:interactive) - other_nthreads_default = default_nthreads_default == 1 ? 2 : 1 - other_nthreads_interactive = default_nthreads_interactive == 0 ? 1 : 0 - @testset "default" begin + # Do this all in a subprocess to protect against the parent having non-default threadpool sizes. + script = """ + using Pkg, Test + @testset "JULIA_NUM_THREADS=1" begin withenv( - "EXPECTED_NUM_THREADS_DEFAULT" => "$default_nthreads_default", - "EXPECTED_NUM_THREADS_INTERACTIVE" => "$default_nthreads_interactive", - "JULIA_NUM_THREADS" => nothing, + "EXPECTED_NUM_THREADS_DEFAULT" => "1", + "EXPECTED_NUM_THREADS_INTERACTIVE" => "0", # https://github.com/JuliaLang/julia/pull/57454 + "JULIA_NUM_THREADS" => "1", ) do Pkg.test("TestThreads") end end - @testset "JULIA_NUM_THREADS=other_nthreads_default" begin + @testset "JULIA_NUM_THREADS=2" begin withenv( - "EXPECTED_NUM_THREADS_DEFAULT" => "$other_nthreads_default", - "EXPECTED_NUM_THREADS_INTERACTIVE" => "$default_nthreads_interactive", - "JULIA_NUM_THREADS" => "$other_nthreads_default", + "EXPECTED_NUM_THREADS_DEFAULT" => "2", + "EXPECTED_NUM_THREADS_INTERACTIVE" => "1", + "JULIA_NUM_THREADS" => "2", ) do Pkg.test("TestThreads") end end - @testset "JULIA_NUM_THREADS=other_nthreads_default,other_nthreads_interactive" begin + @testset "JULIA_NUM_THREADS=2,0" begin withenv( - "EXPECTED_NUM_THREADS_DEFAULT" => "$other_nthreads_default", - "EXPECTED_NUM_THREADS_INTERACTIVE" => "$other_nthreads_interactive", - "JULIA_NUM_THREADS" => "$other_nthreads_default,$other_nthreads_interactive", + "EXPECTED_NUM_THREADS_DEFAULT" => "2", + "EXPECTED_NUM_THREADS_INTERACTIVE" => "0", + "JULIA_NUM_THREADS" => "2,0", ) do Pkg.test("TestThreads") end end - @testset "--threads=other_nthreads_default" begin + + @testset "--threads=1" begin withenv( - "EXPECTED_NUM_THREADS_DEFAULT" => "$other_nthreads_default", - "EXPECTED_NUM_THREADS_INTERACTIVE" => "$default_nthreads_interactive", + "EXPECTED_NUM_THREADS_DEFAULT" => "1", + "EXPECTED_NUM_THREADS_INTERACTIVE" => "0", # https://github.com/JuliaLang/julia/pull/57454 "JULIA_NUM_THREADS" => nothing, ) do - Pkg.test("TestThreads"; julia_args=`--threads=$other_nthreads_default`) + Pkg.test("TestThreads"; julia_args=`--threads=1`) end end - @testset "--threads=other_nthreads_default,other_nthreads_interactive" begin + @testset "--threads=2" begin withenv( - "EXPECTED_NUM_THREADS_DEFAULT" => "$other_nthreads_default", - "EXPECTED_NUM_THREADS_INTERACTIVE" => "$other_nthreads_interactive", + "EXPECTED_NUM_THREADS_DEFAULT" => "2", + "EXPECTED_NUM_THREADS_INTERACTIVE" => "1", "JULIA_NUM_THREADS" => nothing, ) do - Pkg.test("TestThreads"; julia_args=`--threads=$other_nthreads_default,$other_nthreads_interactive`) + Pkg.test("TestThreads"; julia_args=`--threads=2`) end end - end + @testset "--threads=2,0" begin + withenv( + "EXPECTED_NUM_THREADS_DEFAULT" => "2", + "EXPECTED_NUM_THREADS_INTERACTIVE" => "0", + "JULIA_NUM_THREADS" => nothing, + ) do + Pkg.test("TestThreads"; julia_args=`--threads=2,0`) + end + end + """ + @test Utils.show_output_if_command_errors(`$(Base.julia_cmd()) --project=$(path) --startup-file=no -e "$script"`) end end end diff --git a/test/test_packages/TestThreads/test/runtests.jl b/test/test_packages/TestThreads/test/runtests.jl index 43b8df8628..8d060b77f5 100644 --- a/test/test_packages/TestThreads/test/runtests.jl +++ b/test/test_packages/TestThreads/test/runtests.jl @@ -4,4 +4,9 @@ EXPECTED_NUM_THREADS_DEFAULT = parse(Int, ENV["EXPECTED_NUM_THREADS_DEFAULT"]) EXPECTED_NUM_THREADS_INTERACTIVE = parse(Int, ENV["EXPECTED_NUM_THREADS_INTERACTIVE"]) @assert Threads.nthreads() == EXPECTED_NUM_THREADS_DEFAULT @assert Threads.nthreads(:default) == EXPECTED_NUM_THREADS_DEFAULT -@assert Threads.nthreads(:interactive) == EXPECTED_NUM_THREADS_INTERACTIVE +if Threads.nthreads() == 1 + @info "Convert me back to an assert once https://github.com/JuliaLang/julia/pull/57454 has landed" Threads.nthreads(:interactive) EXPECTED_NUM_THREADS_INTERACTIVE +else + @assert Threads.nthreads(:interactive) == EXPECTED_NUM_THREADS_INTERACTIVE +end + diff --git a/test/utils.jl b/test/utils.jl index df1e82c7fa..a8da71d364 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -346,7 +346,7 @@ function show_output_if_command_errors(cmd::Cmd) println(read(out, String)) Base.pipeline_error(proc) end - return nothing + return true end function recursive_rm_cov_files(rootdir::String) From 313fddccb20044517ac8029c8530cdd3f3eb1b5f Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Mon, 9 Jun 2025 09:31:50 -0400 Subject: [PATCH 067/154] Internals: Add fallback `Base.show(::IO, ::RegistryInstance)` method (#4251) --- src/Registry/registry_instance.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Registry/registry_instance.jl b/src/Registry/registry_instance.jl index c5743fed4f..93990d2c8c 100644 --- a/src/Registry/registry_instance.jl +++ b/src/Registry/registry_instance.jl @@ -368,6 +368,7 @@ function Base.show(io::IO, ::MIME"text/plain", r::RegistryInstance) end println(io, " packages: ", length(r.pkgs)) end +Base.show(io::IO, r::RegistryInstance) = Base.show(io, MIME"text/plain"(), r) function uuids_from_name(r::RegistryInstance, name::String) create_name_uuid_mapping!(r) From a94a6bcaec7646a3e43ae14b2aa9a93935c8fb8b Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 13 Jun 2025 15:10:35 +0200 Subject: [PATCH 068/154] fix dev taking when the app is already installed (#4259) Co-authored-by: KristofferC --- src/Apps/Apps.jl | 9 ++------- test/apps.jl | 8 ++++++++ test/test_packages/Rot13.jl/src/Rot13_edited.jl | 7 +++++++ 3 files changed, 17 insertions(+), 7 deletions(-) create mode 100644 test/test_packages/Rot13.jl/src/Rot13_edited.jl diff --git a/src/Apps/Apps.jl b/src/Apps/Apps.jl index f8f9b49140..61150bdaf5 100644 --- a/src/Apps/Apps.jl +++ b/src/Apps/Apps.jl @@ -78,7 +78,6 @@ function _resolve(manifest::Manifest, pkgname=nothing) end projectfile = joinpath(app_env_folder(), pkg.name, "Project.toml") - sourcepath = source_path(app_manifest_file(), pkg) # TODO: Add support for existing manifest # Create a manifest with the manifest entry @@ -87,12 +86,8 @@ function _resolve(manifest::Manifest, pkgname=nothing) if isempty(ctx.env.project.deps) ctx.env.project.deps[pkg.name] = uuid end - if isempty(ctx.env.manifest) - ctx.env.manifest.deps[uuid] = pkg - Pkg.resolve(ctx) - else - Pkg.instantiate(ctx) - end + ctx.env.manifest.deps[uuid] = pkg + Pkg.resolve(ctx) end # TODO: Julia path diff --git a/test/apps.jl b/test/apps.jl index 64e2b264a0..a7a21e4263 100644 --- a/test/apps.jl +++ b/test/apps.jl @@ -33,6 +33,14 @@ isolate(loaded_depot=true) do Pkg.Apps.rm("Rot13") @test Sys.which(exename) == nothing end + + # https://github.com/JuliaLang/Pkg.jl/issues/4258 + Pkg.Apps.add(path=path) + Pkg.Apps.develop(path=path) + mv(joinpath(path, "src", "Rot13_edited.jl"), joinpath(path, "src", "Rot13.jl"); force=true) + withenv("PATH" => string(joinpath(first(DEPOT_PATH), "bin"), sep, current_path)) do + @test read(`$exename test`, String) == "Updated!\n" + end end end diff --git a/test/test_packages/Rot13.jl/src/Rot13_edited.jl b/test/test_packages/Rot13.jl/src/Rot13_edited.jl new file mode 100644 index 0000000000..69c626ff30 --- /dev/null +++ b/test/test_packages/Rot13.jl/src/Rot13_edited.jl @@ -0,0 +1,7 @@ +module Rot13 + +function (@main)(ARGS) + println("Updated!") +end + +end # module Rot13 From a42046240803e86bdb711571db8e40a7a38c4c82 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 19 Jun 2025 12:08:56 +0200 Subject: [PATCH 069/154] don't use tree hash from manifest if the path is set from sources (#4260) Co-authored-by: KristofferC --- src/API.jl | 4 +++- src/Operations.jl | 4 +++- src/manifest.jl | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/API.jl b/src/API.jl index 70a59334e3..e8bfc50d1e 100644 --- a/src/API.jl +++ b/src/API.jl @@ -400,7 +400,9 @@ function up(ctx::Context, pkgs::Vector{PackageSpec}; manifest_resolve!(ctx.env.manifest, pkgs) ensure_resolved(ctx, ctx.env.manifest, pkgs) end - + for pkg in pkgs + update_source_if_set(ctx.env.project, pkg) + end Operations.up(ctx, pkgs, level; skip_writing_project, preserve) return end diff --git a/src/Operations.jl b/src/Operations.jl index f3ddd02e6e..b175199636 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -1769,7 +1769,9 @@ function up_load_versions!(ctx::Context, pkg::PackageSpec, entry::PackageEntry, entry.version !== nothing || return false # no version to set if entry.pinned || level == UPLEVEL_FIXED pkg.version = entry.version - pkg.tree_hash = entry.tree_hash + if pkg.path === nothing + pkg.tree_hash = entry.tree_hash + end elseif entry.repo.source !== nothing || source_repo.source !== nothing # repo packages have a version but are treated specially if source_repo.source !== nothing pkg.repo = source_repo diff --git a/src/manifest.jl b/src/manifest.jl index 87af60823b..25b43004b8 100644 --- a/src/manifest.jl +++ b/src/manifest.jl @@ -298,6 +298,9 @@ function destructure(manifest::Manifest)::Dict end for (uuid, entry) in manifest + # https://github.com/JuliaLang/Pkg.jl/issues/4086 + @assert !(entry.tree_hash !== nothing && entry.path !== nothing) + new_entry = something(entry.other, Dict{String,Any}()) new_entry["uuid"] = string(uuid) entry!(new_entry, "version", entry.version) From cae9ce02a4f044eccaad36287b2b3e2c10fe96ff Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 20 Jun 2025 01:40:09 -0400 Subject: [PATCH 070/154] Fix historical stdlib fixup if `Pkg` is in the Manifest (#4264) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The code here appeared to be untested and broke whenever something actually used it. Example backtrace: ``` julia> JLLPrefixes.collect_artifact_paths(["ripgrep_jll"]) ERROR: MethodError: no method matching haskey(::Vector{Base.UUID}, ::String) The function `haskey` exists, but no method is defined for this combination of argument types. Closest candidates are: haskey(::Pkg.Types.Manifest, ::Any) @ Pkg ~/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.13/Pkg/src/Types.jl:323 haskey(::REPL.Terminals.TTYTerminal, ::Any) @ REPL ~/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.13/REPL/src/Terminals.jl:155 haskey(::LibGit2.CachedCredentials, ::Any) @ LibGit2 ~/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.13/LibGit2/src/types.jl:1357 ... Stacktrace: [1] fixups_from_projectfile!(ctx::Pkg.Types.Context) @ Pkg.Operations ~/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.13/Pkg/src/Operations.jl:238 [2] add(ctx::Pkg.Types.Context, pkgs::Vector{…}, new_git::Set{…}; allow_autoprecomp::Bool, preserve::Pkg.Types.PreserveLevel, platform::Base.BinaryPlatforms.Platform, target::Symbol) @ Pkg.Operations ~/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.13/Pkg/src/Operations.jl:1710 [3] add @ ~/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.13/Pkg/src/Operations.jl:1680 [inlined] [4] add(ctx::Pkg.Types.Context, pkgs::Vector{…}; preserve::Pkg.Types.PreserveLevel, platform::Base.BinaryPlatforms.Platform, target::Symbol, allow_autoprecomp::Bool, kwargs::@Kwargs{…}) @ Pkg.API ~/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.13/Pkg/src/API.jl:328 ``` --- src/Operations.jl | 2 +- test/historical_stdlib_version.jl | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Operations.jl b/src/Operations.jl index b175199636..d6bdb597b6 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -235,7 +235,7 @@ function fixups_from_projectfile!(ctx::Context) # pkg.exts = p.exts # TODO: STDLIBS_BY_VERSION doesn't record this # pkg.entryfile = p.entryfile # TODO: STDLIBS_BY_VERSION doesn't record this for (name, _) in pkg.weakdeps - if !haskey(p.deps, name) + if !(name in p.deps) delete!(pkg.deps, name) end end diff --git a/test/historical_stdlib_version.jl b/test/historical_stdlib_version.jl index e19574155d..7720e2a91f 100644 --- a/test/historical_stdlib_version.jl +++ b/test/historical_stdlib_version.jl @@ -255,6 +255,11 @@ isolate(loaded_depot=true) do end end + @testset "Old Pkg add regression" begin + Pkg.activate(temp=true) + Pkg.add(; name="Pkg", julia_version=v"1.11") + end + @testset "Stdlib add" begin Pkg.activate(temp=true) # Stdlib add (current julia version) From e3d4561272fc029e9a5f940fe101ba4570fa875d Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 20 Jun 2025 07:40:22 +0200 Subject: [PATCH 071/154] add update function to apps and fix a bug when adding an already installed app (#4263) Co-authored-by: KristofferC --- src/Apps/Apps.jl | 54 +++++++++++++++++++++++++--- src/REPLMode/command_declarations.jl | 15 ++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/src/Apps/Apps.jl b/src/Apps/Apps.jl index 61150bdaf5..a3060778e5 100644 --- a/src/Apps/Apps.jl +++ b/src/Apps/Apps.jl @@ -93,7 +93,6 @@ function _resolve(manifest::Manifest, pkgname=nothing) # TODO: Julia path generate_shims_for_apps(pkg.name, pkg.apps, dirname(projectfile), joinpath(Sys.BINDIR, "julia")) end - write_manifest(manifest, app_manifest_file()) end @@ -127,10 +126,12 @@ function add(pkg::PackageSpec) new = Pkg.Operations.download_source(ctx, pkgs) end + # Run Pkg.build()? + + Base.rm(joinpath(app_env_folder(), pkg.name); force=true, recursive=true) sourcepath = source_path(ctx.env.manifest_file, pkg) project = get_project(sourcepath) # TODO: Wrong if package itself has a sourcepath? - entry = PackageEntry(;apps = project.apps, name = pkg.name, version = project.version, tree_hash = pkg.tree_hash, path = pkg.path, repo = pkg.repo, uuid=pkg.uuid) manifest.deps[pkg.uuid] = entry @@ -174,6 +175,50 @@ function develop(pkg::PackageSpec) @info "For package: $(pkg.name) installed apps: $(join(keys(project.apps), ","))" end + +update(pkgs_or_apps::String) = update([pkgs_or_apps]) +function update(pkgs_or_apps::Vector) + for pkg_or_app in pkgs_or_apps + if pkg_or_app isa String + pkg_or_app = PackageSpec(pkg_or_app) + end + update(pkg_or_app) + end +end + +function update(pkg::Union{PackageSpec, Nothing}=nothing) + ctx = app_context() + manifest = ctx.env.manifest + deps = Pkg.Operations.load_manifest_deps(manifest) + for dep in deps + info = manifest.deps[dep.uuid] + if pkg === nothing || info.name !== pkg.name + continue + end + Pkg.activate(joinpath(app_env_folder(), info.name)) do + # precompile only after updating all apps? + if pkg !== nothing + Pkg.update(pkg) + else + Pkg.update() + end + end + sourcepath = abspath(source_path(ctx.env.manifest_file, info)) + project = get_project(sourcepath) + # Get the tree hash from the project file + manifest_file = manifestfile_path(joinpath(app_env_folder(), info.name)) + manifest_app = Pkg.Types.read_manifest(manifest_file) + manifest_entry = manifest_app.deps[info.uuid] + + entry = PackageEntry(;apps = project.apps, name = manifest_entry.name, version = manifest_entry.version, tree_hash = manifest_entry.tree_hash, + path = manifest_entry.path, repo = manifest_entry.repo, uuid = manifest_entry.uuid) + + manifest.deps[dep.uuid] = entry + Pkg.Types.write_manifest(manifest, app_manifest_file()) + end + return +end + function status(pkgs_or_apps::Vector) if isempty(pkgs_or_apps) status() @@ -238,8 +283,7 @@ end function require_not_empty(pkgs, f::Symbol) - - if pkgs == nothing || isempty(pkgs) + if pkgs === nothing || isempty(pkgs) pkgerror("app $f requires at least one package") end end @@ -287,7 +331,7 @@ function rm(pkg_or_app::Union{PackageSpec, Nothing}=nothing) end end end - + # XXX: What happens if something fails above and we do not write out the updated manifest? Pkg.Types.write_manifest(manifest, app_manifest_file()) return end diff --git a/src/REPLMode/command_declarations.jl b/src/REPLMode/command_declarations.jl index a9d9cc3304..98153e5b48 100644 --- a/src/REPLMode/command_declarations.jl +++ b/src/REPLMode/command_declarations.jl @@ -647,6 +647,21 @@ pkg> app develop ~/mypackages/Example pkg> app develop --local Example ``` """ +], +PSA[:name => "update", + :short_name => "up", + :api => Apps.update, + :completions => :complete_installed_apps, + :should_splat => false, + :arg_count => 0 => Inf, + :arg_parser => parse_package, + :description => "update app", + :help => md""" + app update pkg + +Updates the apps for packages `pkg...` or apps `app...`. +``` +""", ], # app ] ] #command_declarations From 4d1c6b0a393da7397dc2c1f0c94b394bfa666798 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 20 Jun 2025 07:06:42 -0400 Subject: [PATCH 072/154] explain no reg installed when no reg installed (#4261) --- src/Operations.jl | 6 ++++++ test/pkg.jl | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/src/Operations.jl b/src/Operations.jl index d6bdb597b6..245a910d63 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -1586,6 +1586,12 @@ function is_all_registered(registries::Vector{Registry.RegistryInstance}, pkgs:: end function check_registered(registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}) + if isempty(registries) && !isempty(pkgs) + registry_pkgs = filter(tracking_registered_version, pkgs) + if !isempty(registry_pkgs) + pkgerror("no registries have been installed. Cannot resolve the following packages:\n$(join(map(pkg -> " " * err_rep(pkg), registry_pkgs), "\n"))") + end + end pkg = is_all_registered(registries, pkgs) if pkg isa PackageSpec pkgerror("expected package $(err_rep(pkg)) to be registered") diff --git a/test/pkg.jl b/test/pkg.jl index c5f4371efe..8271697110 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -1055,4 +1055,39 @@ end Pkg.activate(prev_project) end +@testset "check_registered error paths" begin + # Test the "no registries have been installed" error path + isolate(loaded_depot=false, linked_reg=false) do + with_temp_env() do + # Ensure we have no registries available + @test isempty(Pkg.Registry.reachable_registries()) + + # Should install General registry automatically + Pkg.add("Example") + + Pkg.Registry.rm("General") + @test isempty(Pkg.Registry.reachable_registries()) + + @test_throws r"no registries have been installed\. Cannot resolve the following packages:" begin + Pkg.resolve() + end + end + end + + # Test the "expected package to be registered" error path with a custom unregistered package + isolate(loaded_depot=true) do; mktempdir() do tempdir + with_temp_env() do + # Create a fake package with a manifest that references an unregistered UUID + fake_pkg_path = copy_test_package(tempdir, "UnregisteredUUID") + Pkg.activate(fake_pkg_path) + + # This should fail with "expected package to be registered" error + @test_throws r"expected package.*to be registered" begin + Pkg.add("JSON") # This will fail because Example UUID in manifest is unregistered + end + end + end + end +end + end # module From eefbef6491666b88400c94081258e2c41c9845c8 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 27 Jun 2025 09:40:47 +0200 Subject: [PATCH 073/154] feat(errors): Improve error message for incorrect package UUID (#4270) --- src/Operations.jl | 19 ++++++++++++++++++- test/new.jl | 6 +++--- test/pkg.jl | 12 ++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/Operations.jl b/src/Operations.jl index 245a910d63..e434f59d56 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -1594,7 +1594,24 @@ function check_registered(registries::Vector{Registry.RegistryInstance}, pkgs::V end pkg = is_all_registered(registries, pkgs) if pkg isa PackageSpec - pkgerror("expected package $(err_rep(pkg)) to be registered") + msg = "expected package $(err_rep(pkg)) to be registered" + # check if the name exists in the registry with a different uuid + if pkg.name !== nothing + reg_uuid = Pair{String, Vector{UUID}}[] + for reg in registries + uuids = Registry.uuids_from_name(reg, pkg.name) + if !isempty(uuids) + push!(reg_uuid, reg.name => uuids) + end + end + if !isempty(reg_uuid) + msg *= "\n You may have provided the wrong UUID for package $(pkg.name).\n Found the following UUIDs for that name:" + for (reg, uuids) in reg_uuid + msg *= "\n - $(join(uuids, ", ")) from registry: $reg" + end + end + end + pkgerror(msg) end return nothing end diff --git a/test/new.jl b/test/new.jl index 0e4b5169cb..acf4ba34cc 100644 --- a/test/new.jl +++ b/test/new.jl @@ -538,7 +538,7 @@ end isolate(loaded_depot=true) do; mktempdir() do tempdir package_path = copy_test_package(tempdir, "UnregisteredUUID") Pkg.activate(package_path) - @test_throws PkgError("expected package `Example [142fd7e7]` to be registered") Pkg.add("JSON") + @test_throws PkgError Pkg.add("JSON") end end # empty git repo (no commits) isolate(loaded_depot=true) do; mktempdir() do tempdir @@ -1536,7 +1536,7 @@ end isolate(loaded_depot=true) do; mktempdir() do tempdir package_path = copy_test_package(tempdir, "UnregisteredUUID") Pkg.activate(package_path) - @test_throws PkgError("expected package `Example [142fd7e7]` to be registered") Pkg.update() + @test_throws PkgError Pkg.update() end end end @@ -1714,7 +1714,7 @@ end isolate(loaded_depot=true) do; mktempdir() do tempdir package_path = copy_test_package(tempdir, "UnregisteredUUID") Pkg.activate(package_path) - @test_throws PkgError("expected package `Example [142fd7e7]` to be registered") Pkg.update() + @test_throws PkgError Pkg.update() end end # package does not exist in the manifest isolate(loaded_depot=true) do diff --git a/test/pkg.jl b/test/pkg.jl index 8271697110..f0e9950ce1 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -207,6 +207,18 @@ temp_pkg_dir() do project_path @testset "package with wrong UUID" begin @test_throws PkgError Pkg.add(PackageSpec(TEST_PKG.name, UUID(UInt128(1)))) + @testset "package with wrong UUID but correct name" begin + try + Pkg.add(PackageSpec(name="Example", uuid=UUID(UInt128(2)))) + catch e + @test e isa PkgError + errstr = sprint(showerror, e) + @test occursin("expected package `Example [00000000]` to be registered", errstr) + @test occursin("You may have provided the wrong UUID for package Example.", errstr) + @test occursin("Found the following UUIDs for that name:", errstr) + @test occursin("- 7876af07-990d-54b4-ab0e-23690620f79a from registry: General", errstr) + end + end # Missing uuid @test_throws PkgError Pkg.add(PackageSpec(uuid = uuid4())) end From 8b1f0b9ffe559aae285a951d81ae05ed543ea130 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 27 Jun 2025 12:33:09 -0400 Subject: [PATCH 074/154] prompt for confirmation before removing compat entry (#4254) --- src/API.jl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/API.jl b/src/API.jl index e8bfc50d1e..103e7c99f3 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1410,12 +1410,22 @@ function compat(ctx::Context, pkg::String, compat_str::Union{Nothing,String}; io io = something(io, ctx.io) pkg = pkg == "Julia" ? "julia" : pkg isnothing(compat_str) || (compat_str = string(strip(compat_str, '"'))) + existing_compat = Operations.get_compat_str(ctx.env.project, pkg) + # Double check before deleting a compat entry issue/3567 + if isinteractive() && (isnothing(compat_str) || isempty(compat_str)) + if !isnothing(existing_compat) + ans = Base.prompt(stdin, ctx.io, "No compat string was given. Delete existing compat entry `$pkg = $(repr(existing_compat))`? [y]/n", default = "y") + if lowercase(ans) !== "y" + return + end + end + end if haskey(ctx.env.project.deps, pkg) || pkg == "julia" success = Operations.set_compat(ctx.env.project, pkg, isnothing(compat_str) ? nothing : isempty(compat_str) ? nothing : compat_str) success === false && pkgerror("invalid compat version specifier \"$(compat_str)\"") write_env(ctx.env) if isnothing(compat_str) || isempty(compat_str) - printpkgstyle(io, :Compat, "entry removed for $(pkg)") + printpkgstyle(io, :Compat, "entry removed:\n $pkg = $(repr(existing_compat))") else printpkgstyle(io, :Compat, "entry set:\n $(pkg) = $(repr(compat_str))") end From e9a05524087cbbbeb151e9b1c355008ef639ed06 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 30 Jun 2025 11:19:18 +0200 Subject: [PATCH 075/154] fix what project file to look at when package without path but with a subdir is devved by name (#4271) --- src/Types.jl | 2 +- test/subdir.jl | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Types.jl b/src/Types.jl index 2b475d4b10..2f2f84d35b 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -804,7 +804,7 @@ function handle_repo_develop!(ctx::Context, pkg::PackageSpec, shared::Bool) new = true end if !has_uuid(pkg) - resolve_projectfile!(pkg, dev_path) + resolve_projectfile!(pkg, joinpath(dev_path, pkg.repo.subdir === nothing ? "" : pkg.repo.subdir)) end error_if_in_sysimage(pkg) pkg.path = shared ? dev_path : relative_project_path(ctx.env.manifest_file, dev_path) diff --git a/test/subdir.jl b/test/subdir.jl index cddf27992f..a9833d2829 100644 --- a/test/subdir.jl +++ b/test/subdir.jl @@ -241,6 +241,8 @@ end pkgstr("add $(packages_dir):dependencies/Dep") @test !isinstalled("Package") @test isinstalled("Dep") + pkg"dev Dep" # 4269 + @test isinstalled("Dep") pkg"rm Dep" # Add from path at branch. From e02bcabd7443f6efec8d1add070cd8d85ef1347a Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Mon, 30 Jun 2025 05:25:39 -0400 Subject: [PATCH 076/154] Registry: Properly pass down `depot` (#4268) --- src/Pkg.jl | 12 ++++++++---- src/Registry/Registry.jl | 27 ++++++++++++++++----------- test/registry.jl | 2 +- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/src/Pkg.jl b/src/Pkg.jl index 6241b394eb..a43fa91de9 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -26,10 +26,14 @@ public activate, add, build, compat, develop, free, gc, generate, instantiate, pin, precompile, redo, rm, resolve, status, test, undo, update, why depots() = Base.DEPOT_PATH -function depots1() - d = depots() - isempty(d) && Pkg.Types.pkgerror("no depots found in DEPOT_PATH") - return d[1] +function depots1(depot_list::Union{String, Vector{String}}=depots()) + # Get the first depot from a list, with proper error handling + if depot_list isa String + return depot_list + else + isempty(depot_list) && Pkg.Types.pkgerror("no depots provided") + return depot_list[1] + end end function pkg_server() diff --git a/src/Registry/Registry.jl b/src/Registry/Registry.jl index 32e9364050..af5cab05e1 100644 --- a/src/Registry/Registry.jl +++ b/src/Registry/Registry.jl @@ -1,7 +1,7 @@ module Registry import ..Pkg -using ..Pkg: depots1, printpkgstyle, stderr_f, isdir_nothrow, pathrepr, pkg_server, +using ..Pkg: depots, depots1, printpkgstyle, stderr_f, isdir_nothrow, pathrepr, pkg_server, GitTools using ..Pkg.PlatformEngines: download_verify_unpack, download, download_verify, exe7z, verify_archive_tree_hash using UUIDs, LibGit2, TOML, Dates @@ -48,11 +48,11 @@ function add(; name=nothing, uuid=nothing, url=nothing, path=nothing, linked=not add([RegistrySpec(; name, uuid, url, path, linked)]; kwargs...) end end -function add(regs::Vector{RegistrySpec}; io::IO=stderr_f(), depot=depots1()) +function add(regs::Vector{RegistrySpec}; io::IO=stderr_f(), depots::Union{String, Vector{String}}=depots()) if isempty(regs) - download_default_registries(io, only_if_empty = false; depot) + download_default_registries(io, only_if_empty = false; depots=depots) else - download_registries(io, regs, depot) + download_registries(io, regs, depots) end end @@ -103,12 +103,15 @@ end pkg_server_url_hash(url::String) = Base.SHA1(split(url, '/')[end]) -function download_default_registries(io::IO; only_if_empty::Bool = true, depot=depots1()) - installed_registries = reachable_registries() +function download_default_registries(io::IO; only_if_empty::Bool = true, depots::Union{String, Vector{String}}=depots()) + # Check the specified depots for installed registries + installed_registries = reachable_registries(; depots) # Only clone if there are no installed registries, unless called # with false keyword argument. if isempty(installed_registries) || !only_if_empty - printpkgstyle(io, :Installing, "known registries into $(pathrepr(depot))") + # Install to the first depot in the list + target_depot = depots1(depots) + printpkgstyle(io, :Installing, "known registries into $(pathrepr(target_depot))") registries = copy(DEFAULT_REGISTRIES) for uuid in keys(pkg_server_registry_urls()) if !(uuid in (reg.uuid for reg in registries)) @@ -116,7 +119,7 @@ function download_default_registries(io::IO; only_if_empty::Bool = true, depot=d end end filter!(reg -> !(reg.uuid in installed_registries), registries) - download_registries(io, registries, depot) + download_registries(io, registries, depots) return true end return false @@ -168,14 +171,16 @@ function check_registry_state(reg) return nothing end -function download_registries(io::IO, regs::Vector{RegistrySpec}, depot::String=depots1()) +function download_registries(io::IO, regs::Vector{RegistrySpec}, depots::Union{String, Vector{String}}=depots()) + # Use the first depot as the target + target_depot = depots1(depots) populate_known_registries_with_urls!(regs) - regdir = joinpath(depot, "registries") + regdir = joinpath(target_depot, "registries") isdir(regdir) || mkpath(regdir) # only allow one julia process to download and install registries at a time FileWatching.mkpidlock(joinpath(regdir, ".pid"), stale_age = 10) do # once we're pidlocked check if another process has installed any of the registries - reachable_uuids = map(r -> r.uuid, reachable_registries()) + reachable_uuids = map(r -> r.uuid, reachable_registries(; depots)) filter!(r -> !in(r.uuid, reachable_uuids), regs) registry_urls = pkg_server_registry_urls() diff --git a/test/registry.jl b/test/registry.jl index 20d70ea038..96c4d1144a 100644 --- a/test/registry.jl +++ b/test/registry.jl @@ -257,7 +257,7 @@ end @test isempty(Registry.reachable_registries(; depots=[depot_off_path])) # After this, we have depots only in the depot that's off the path - Registry.add("General"; depot=depot_off_path) + Registry.add("General"; depots=depot_off_path) @test isempty(Registry.reachable_registries()) @test length(Registry.reachable_registries(; depots=[depot_off_path])) == 1 From d2e61025b775a785f4b6943ef408614ba460dcd0 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 30 Jun 2025 13:21:30 +0200 Subject: [PATCH 077/154] Fix leading whitespace in REPL commands with comma-separated packages (#4274) --- src/REPLMode/REPLMode.jl | 4 ++-- test/repl.jl | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/REPLMode/REPLMode.jl b/src/REPLMode/REPLMode.jl index 2abd52e5cc..b0c70745b6 100644 --- a/src/REPLMode/REPLMode.jl +++ b/src/REPLMode/REPLMode.jl @@ -223,7 +223,7 @@ function lex(cmd::String)::Vector{QString} return filter(x->!isempty(x.raw), qstrings) end -function tokenize(cmd::String) +function tokenize(cmd::AbstractString) cmd = replace(replace(cmd, "\r\n" => "; "), "\n" => "; ") # for multiline commands qstrings = lex(cmd) statements = foldl(qstrings; init=[QString[]]) do collection, next @@ -282,7 +282,7 @@ function core_parse(words::Vector{QString}; only_cmd=false) end parse(input::String) = - map(Base.Iterators.filter(!isempty, tokenize(input))) do words + map(Base.Iterators.filter(!isempty, tokenize(strip(input)))) do words statement, input_word = core_parse(words) statement.spec === nothing && pkgerror("`$input_word` is not a recognized command. Type ? for help with available commands") statement.options = map(parse_option, statement.options) diff --git a/test/repl.jl b/test/repl.jl index ed3c635a98..0d2f0e94ad 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -66,6 +66,9 @@ temp_pkg_dir(;rm=false) do project_path; cd(project_path) do; pkg"rm Example Random" pkg"add Example,Random" pkg"rm Example,Random" + # Test leading whitespace handling (issue #4239) + pkg" add Example, Random" + pkg"rm Example Random" pkg"add Example#master" pkg"rm Example" pkg"add https://github.com/JuliaLang/Example.jl#master" From c78b40b3514a560a40bffada39743687fba38115 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 30 Jun 2025 13:21:59 +0200 Subject: [PATCH 078/154] copy the app project instead of wrapping it (#4276) Co-authored-by: KristofferC --- src/Apps/Apps.jl | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/Apps/Apps.jl b/src/Apps/Apps.jl index a3060778e5..c3abf81455 100644 --- a/src/Apps/Apps.jl +++ b/src/Apps/Apps.jl @@ -77,15 +77,33 @@ function _resolve(manifest::Manifest, pkgname=nothing) continue end + # TODO: Add support for existing manifest + projectfile = joinpath(app_env_folder(), pkg.name, "Project.toml") - # TODO: Add support for existing manifest + sourcepath = source_path(app_manifest_file(), pkg) + original_project_file = projectfile_path(sourcepath) + + mkpath(dirname(projectfile)) + + if isfile(original_project_file) + cp(original_project_file, projectfile; force=true) + chmod(projectfile, 0o644) # Make the copied project file writable + + # Add entryfile stanza pointing to the package entry file + # TODO: What if project file has its own entryfile? + project_data = TOML.parsefile(projectfile) + project_data["entryfile"] = joinpath(sourcepath, "src", "$(pkg.name).jl") + open(projectfile, "w") do io + TOML.print(io, project_data) + end + else + error("could not find project file for package $pkg") + end + # Create a manifest with the manifest entry Pkg.activate(joinpath(app_env_folder(), pkg.name)) do ctx = Context() - if isempty(ctx.env.project.deps) - ctx.env.project.deps[pkg.name] = uuid - end ctx.env.manifest.deps[uuid] = pkg Pkg.resolve(ctx) end @@ -154,6 +172,7 @@ function develop(pkg::PackageSpec) handle_package_input!(pkg) ctx = app_context() handle_repo_develop!(ctx, pkg, #=shared =# true) + Base.rm(joinpath(app_env_folder(), pkg.name); force=true, recursive=true) sourcepath = abspath(source_path(ctx.env.manifest_file, pkg)) project = get_project(sourcepath) From 25c2390edc0b1ea142361dcb2666e3dd56bedae0 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 30 Jun 2025 15:52:00 +0200 Subject: [PATCH 079/154] feat(apps): Add support for multiple apps per package via submodules (#4277) Co-authored-by: Claude --- docs/src/apps.md | 45 ++++++++++++++++++++++-- src/Apps/Apps.jl | 14 ++++---- src/Types.jl | 1 + src/manifest.jl | 8 ++++- src/project.jl | 3 +- test/apps.jl | 16 +++++++++ test/test_packages/Rot13.jl/Project.toml | 1 + test/test_packages/Rot13.jl/src/CLI.jl | 18 ++++++++++ test/test_packages/Rot13.jl/src/Rot13.jl | 2 ++ 9 files changed, 98 insertions(+), 10 deletions(-) create mode 100644 test/test_packages/Rot13.jl/src/CLI.jl diff --git a/docs/src/apps.md b/docs/src/apps.md index 1fccbb8fbd..083d68f0e1 100644 --- a/docs/src/apps.md +++ b/docs/src/apps.md @@ -7,7 +7,6 @@ - You need to manually make `~/.julia/bin` available on the PATH environment. - The path to the julia executable used is the same as the one used to install the app. If this julia installation gets removed, you might need to reinstall the app. - - You can only have one app installed per package. Apps are Julia packages that are intended to be run as a "standalone programs" (by e.g. typing the name of the app in the terminal possibly together with some arguments or flags/options). This is in contrast to most Julia packages that are used as "libraries" and are loaded by other files or in the Julia REPL. @@ -17,7 +16,7 @@ This is in contrast to most Julia packages that are used as "libraries" and are A Julia app is structured similar to a standard Julia library with the following additions: - A `@main` entry point in the package module (see the [Julia help on `@main`](https://docs.julialang.org/en/v1/manual/command-line-interface/#The-Main.main-entry-point) for details) -- An `[app]` section in the `Project.toml` file listing the executable names that the package provides. +- An `[apps]` section in the `Project.toml` file listing the executable names that the package provides. A very simple example of an app that prints the reversed input arguments would be: @@ -45,6 +44,48 @@ reverse = {} ``` The empty table `{}` is to allow for giving metadata about the app but it is currently unused. +## Multiple Apps per Package + +A single package can define multiple apps by using submodules. Each app can have its own entry point in a different submodule of the package. + +```julia +# src/MyMultiApp.jl +module MyMultiApp + +function (@main)(ARGS) + println("Main app: ", join(ARGS, " ")) +end + +include("CLI.jl") + +end # module +``` + +```julia +# src/CLI.jl +module CLI + +function (@main)(ARGS) + println("CLI submodule: ", join(ARGS, " ")) +end + +end # module CLI +``` + +```toml +# Project.toml + +# standard fields here + +[apps] +main-app = {} +cli-app = { submodule = "CLI" } +``` + +This will create two executables: +- `main-app` that runs `julia -m MyMultiApp` +- `cli-app` that runs `julia -m MyMultiApp.CLI` + After installing this app one could run: ``` diff --git a/src/Apps/Apps.jl b/src/Apps/Apps.jl index c3abf81455..2cb20ab61b 100644 --- a/src/Apps/Apps.jl +++ b/src/Apps/Apps.jl @@ -397,9 +397,9 @@ function generate_shim(pkgname, app::AppInfo, env, julia) julia_bin_filename = joinpath(julia_bin_path(), filename) mkpath(dirname(filename)) content = if Sys.iswindows() - windows_shim(pkgname, julia, env) + windows_shim(pkgname, app, julia, env) else - bash_shim(pkgname, julia, env) + bash_shim(pkgname, app, julia, env) end overwrite_file_if_different(julia_bin_filename, content) if Sys.isunix() @@ -408,7 +408,8 @@ function generate_shim(pkgname, app::AppInfo, env, julia) end -function bash_shim(pkgname, julia::String, env) +function bash_shim(pkgname, app::AppInfo, julia::String, env) + module_spec = app.submodule === nothing ? pkgname : "$(pkgname).$(app.submodule)" return """ #!/usr/bin/env bash @@ -418,12 +419,13 @@ function bash_shim(pkgname, julia::String, env) export JULIA_DEPOT_PATH=$(repr(join(DEPOT_PATH, ':'))) exec $julia \\ --startup-file=no \\ - -m $(pkgname) \\ + -m $(module_spec) \\ "\$@" """ end -function windows_shim(pkgname, julia::String, env) +function windows_shim(pkgname, app::AppInfo, julia::String, env) + module_spec = app.submodule === nothing ? pkgname : "$(pkgname).$(app.submodule)" return """ @echo off @@ -435,7 +437,7 @@ function windows_shim(pkgname, julia::String, env) $julia ^ --startup-file=no ^ - -m $(pkgname) ^ + -m $(module_spec) ^ %* """ end diff --git a/src/Types.jl b/src/Types.jl index 2f2f84d35b..327a54ab0b 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -242,6 +242,7 @@ struct AppInfo name::String julia_command::Union{String, Nothing} julia_version::Union{VersionNumber, Nothing} + submodule::Union{String, Nothing} other::Dict{String,Any} end Base.@kwdef mutable struct Project diff --git a/src/manifest.jl b/src/manifest.jl index 25b43004b8..57eb3db2c4 100644 --- a/src/manifest.jl +++ b/src/manifest.jl @@ -86,9 +86,11 @@ read_apps(::Any) = pkgerror("Expected `apps` field to be a Dict") function read_apps(apps::Dict) appinfos = Dict{String, AppInfo}() for (appname, app) in apps + submodule = get(app, "submodule", nothing) appinfo = AppInfo(appname::String, app["julia_command"]::String, VersionNumber(app["julia_version"]::String), + submodule, app) appinfos[appinfo.name] = appinfo end @@ -344,7 +346,11 @@ function destructure(manifest::Manifest)::Dict for (appname, appinfo) in entry.apps julia_command = @something appinfo.julia_command joinpath(Sys.BINDIR, "julia" * (Sys.iswindows() ? ".exe" : "")) julia_version = @something appinfo.julia_version VERSION - new_entry["apps"][appname] = Dict{String,Any}("julia_command" => julia_command, "julia_version" => julia_version) + app_dict = Dict{String,Any}("julia_command" => julia_command, "julia_version" => julia_version) + if appinfo.submodule !== nothing + app_dict["submodule"] = appinfo.submodule + end + new_entry["apps"][appname] = app_dict end end if manifest.manifest_format.major == 1 diff --git a/src/project.jl b/src/project.jl index 9006936697..d4a2a0c47f 100644 --- a/src/project.jl +++ b/src/project.jl @@ -84,7 +84,8 @@ function read_project_apps(raw::Dict{String,Any}, project::Project) info isa Dict{String,Any} || pkgerror(""" Expected value for app `$name` to be a dictionary. """) - appinfos[name] = AppInfo(name, nothing, nothing, other) + submodule = get(info, "submodule", nothing) + appinfos[name] = AppInfo(name, nothing, nothing, submodule, other) end return appinfos end diff --git a/test/apps.jl b/test/apps.jl index a7a21e4263..2704c5a4bb 100644 --- a/test/apps.jl +++ b/test/apps.jl @@ -12,11 +12,19 @@ isolate(loaded_depot=true) do Pkg.Apps.develop(path=joinpath(@__DIR__, "test_packages", "Rot13.jl")) current_path = ENV["PATH"] exename = Sys.iswindows() ? "juliarot13.bat" : "juliarot13" + cliexename = Sys.iswindows() ? "juliarot13cli.bat" : "juliarot13cli" withenv("PATH" => string(joinpath(first(DEPOT_PATH), "bin"), sep, current_path)) do + # Test original app @test contains(Sys.which("$exename"), first(DEPOT_PATH)) @test read(`$exename test`, String) == "grfg\n" + + # Test submodule app + @test contains(Sys.which("$cliexename"), first(DEPOT_PATH)) + @test read(`$cliexename test`, String) == "CLI: grfg\n" + Pkg.Apps.rm("Rot13") @test Sys.which(exename) == nothing + @test Sys.which(cliexename) == nothing end end @@ -26,12 +34,20 @@ isolate(loaded_depot=true) do path = git_init_package(tmpdir, joinpath(@__DIR__, "test_packages", "Rot13.jl")) Pkg.Apps.add(path=path) exename = Sys.iswindows() ? "juliarot13.bat" : "juliarot13" + cliexename = Sys.iswindows() ? "juliarot13cli.bat" : "juliarot13cli" current_path = ENV["PATH"] withenv("PATH" => string(joinpath(first(DEPOT_PATH), "bin"), sep, current_path)) do + # Test original app @test contains(Sys.which(exename), first(DEPOT_PATH)) @test read(`$exename test`, String) == "grfg\n" + + # Test submodule app + @test contains(Sys.which(cliexename), first(DEPOT_PATH)) + @test read(`$cliexename test`, String) == "CLI: grfg\n" + Pkg.Apps.rm("Rot13") @test Sys.which(exename) == nothing + @test Sys.which(cliexename) == nothing end # https://github.com/JuliaLang/Pkg.jl/issues/4258 diff --git a/test/test_packages/Rot13.jl/Project.toml b/test/test_packages/Rot13.jl/Project.toml index fe3ae6389c..a1933ed8a2 100644 --- a/test/test_packages/Rot13.jl/Project.toml +++ b/test/test_packages/Rot13.jl/Project.toml @@ -4,3 +4,4 @@ version = "0.1.0" [apps] juliarot13 = {} +juliarot13cli = { submodule = "CLI" } diff --git a/test/test_packages/Rot13.jl/src/CLI.jl b/test/test_packages/Rot13.jl/src/CLI.jl new file mode 100644 index 0000000000..342756b393 --- /dev/null +++ b/test/test_packages/Rot13.jl/src/CLI.jl @@ -0,0 +1,18 @@ +module CLI + +using ..Rot13: rot13 + +function (@main)(ARGS) + if length(ARGS) == 0 + println("Usage: rot13cli ") + return 1 + end + + for arg in ARGS + # Add a prefix to distinguish from main module output + println("CLI: $(rot13(arg))") + end + return 0 +end + +end # module CLI \ No newline at end of file diff --git a/test/test_packages/Rot13.jl/src/Rot13.jl b/test/test_packages/Rot13.jl/src/Rot13.jl index facbb92527..a97779dd77 100644 --- a/test/test_packages/Rot13.jl/src/Rot13.jl +++ b/test/test_packages/Rot13.jl/src/Rot13.jl @@ -14,4 +14,6 @@ function (@main)(ARGS) return 0 end +include("CLI.jl") + end # module Rot13 From 109eaea66a0adb0ad8fa497e64913eadc2248ad1 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 30 Jun 2025 23:21:11 +0200 Subject: [PATCH 080/154] Various app improvements (#4278) Co-authored-by: KristofferC --- src/Apps/Apps.jl | 91 +++++++++++++++++++++++++++++++++++++++++------- src/Types.jl | 1 - src/manifest.jl | 4 +-- src/project.jl | 2 +- 4 files changed, 81 insertions(+), 17 deletions(-) diff --git a/src/Apps/Apps.jl b/src/Apps/Apps.jl index 2cb20ab61b..770c872448 100644 --- a/src/Apps/Apps.jl +++ b/src/Apps/Apps.jl @@ -15,8 +15,41 @@ julia_bin_path() = joinpath(first(DEPOT_PATH), "bin") app_context() = Context(env=EnvCache(joinpath(app_env_folder(), "Project.toml"))) +function validate_app_name(name::AbstractString) + if isempty(name) + error("App name cannot be empty") + end + if !occursin(r"^[a-zA-Z][a-zA-Z0-9_-]*$", name) + error("App name must start with a letter and contain only letters, numbers, underscores, and hyphens") + end + if occursin(r"\.\.", name) || occursin(r"[/\\]", name) + error("App name cannot contain path traversal sequences or path separators") + end +end + +function validate_package_name(name::AbstractString) + if isempty(name) + error("Package name cannot be empty") + end + if !occursin(r"^[a-zA-Z][a-zA-Z0-9_]*$", name) + error("Package name must start with a letter and contain only letters, numbers, and underscores") + end +end + +function validate_submodule_name(name::Union{AbstractString,Nothing}) + if name !== nothing + if isempty(name) + error("Submodule name cannot be empty") + end + if !occursin(r"^[a-zA-Z][a-zA-Z0-9_]*$", name) + error("Submodule name must start with a letter and contain only letters, numbers, and underscores") + end + end +end + function rm_shim(name; kwargs...) + validate_app_name(name) Base.rm(joinpath(julia_bin_path(), name * (Sys.iswindows() ? ".bat" : "")); kwargs...) end @@ -38,6 +71,30 @@ function overwrite_file_if_different(file, content) end end +function check_apps_in_path(apps) + for app_name in keys(apps) + which_result = Sys.which(app_name) + if which_result === nothing + @warn """ + App '$app_name' was installed but is not available in PATH. + Consider adding '$(julia_bin_path())' to your PATH environment variable. + """ maxlog=1 + break # Only show warning once per installation + else + # Check for collisions + expected_path = joinpath(julia_bin_path(), app_name * (Sys.iswindows() ? ".bat" : "")) + if which_result != expected_path + @warn """ + App '$app_name' collision detected: + Expected: $expected_path + Found: $which_result + Another application with the same name exists in PATH. + """ + end + end + end +end + function get_max_version_register(pkg::PackageSpec, regs) max_v = nothing tree_hash = nothing @@ -157,6 +214,7 @@ function add(pkg::PackageSpec) precompile(pkg.name) @info "For package: $(pkg.name) installed apps $(join(keys(project.apps), ","))" + check_apps_in_path(project.apps) end function develop(pkg::Vector{PackageSpec}) @@ -192,6 +250,7 @@ function develop(pkg::PackageSpec) _resolve(manifest, pkg.name) precompile(pkg.name) @info "For package: $(pkg.name) installed apps: $(join(keys(project.apps), ","))" + check_apps_in_path(project.apps) end @@ -205,6 +264,7 @@ function update(pkgs_or_apps::Vector) end end +# XXX: Is updating an app ever different from rm-ing and adding it from scratch? function update(pkg::Union{PackageSpec, Nothing}=nothing) ctx = app_context() manifest = ctx.env.manifest @@ -385,7 +445,6 @@ const SHIM_VERSION = 1.0 const SHIM_HEADER = """$SHIM_COMMENT This file is generated by the Julia package manager. $SHIM_COMMENT Shim version: $SHIM_VERSION""" - function generate_shims_for_apps(pkgname, apps, env, julia) for (_, app) in apps generate_shim(pkgname, app, env, julia) @@ -393,13 +452,23 @@ function generate_shims_for_apps(pkgname, apps, env, julia) end function generate_shim(pkgname, app::AppInfo, env, julia) + validate_package_name(pkgname) + validate_app_name(app.name) + validate_submodule_name(app.submodule) + + module_spec = app.submodule === nothing ? pkgname : "$(pkgname).$(app.submodule)" + filename = app.name * (Sys.iswindows() ? ".bat" : "") julia_bin_filename = joinpath(julia_bin_path(), filename) - mkpath(dirname(filename)) + mkpath(dirname(julia_bin_filename)) content = if Sys.iswindows() - windows_shim(pkgname, app, julia, env) + julia_escaped = "\"$(Base.shell_escape_wincmd(julia))\"" + module_spec_escaped = "\"$(Base.shell_escape_wincmd(module_spec))\"" + windows_shim(julia_escaped, module_spec_escaped, env) else - bash_shim(pkgname, app, julia, env) + julia_escaped = Base.shell_escape(julia) + module_spec_escaped = Base.shell_escape(module_spec) + bash_shim(julia_escaped, module_spec_escaped, env) end overwrite_file_if_different(julia_bin_filename, content) if Sys.isunix() @@ -408,8 +477,7 @@ function generate_shim(pkgname, app::AppInfo, env, julia) end -function bash_shim(pkgname, app::AppInfo, julia::String, env) - module_spec = app.submodule === nothing ? pkgname : "$(pkgname).$(app.submodule)" +function bash_shim(julia_escaped::String, module_spec_escaped::String, env) return """ #!/usr/bin/env bash @@ -417,15 +485,14 @@ function bash_shim(pkgname, app::AppInfo, julia::String, env) export JULIA_LOAD_PATH=$(repr(env)) export JULIA_DEPOT_PATH=$(repr(join(DEPOT_PATH, ':'))) - exec $julia \\ + exec $julia_escaped \\ --startup-file=no \\ - -m $(module_spec) \\ + -m $module_spec_escaped \\ "\$@" """ end -function windows_shim(pkgname, app::AppInfo, julia::String, env) - module_spec = app.submodule === nothing ? pkgname : "$(pkgname).$(app.submodule)" +function windows_shim(julia_escaped::String, module_spec_escaped::String, env) return """ @echo off @@ -435,9 +502,9 @@ function windows_shim(pkgname, app::AppInfo, julia::String, env) set JULIA_LOAD_PATH=$env set JULIA_DEPOT_PATH=$(join(DEPOT_PATH, ';')) - $julia ^ + $julia_escaped ^ --startup-file=no ^ - -m $(module_spec) ^ + -m $module_spec_escaped ^ %* """ end diff --git a/src/Types.jl b/src/Types.jl index 327a54ab0b..18224c02c9 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -241,7 +241,6 @@ Base.hash(t::Compat, h::UInt) = hash(t.val, h) struct AppInfo name::String julia_command::Union{String, Nothing} - julia_version::Union{VersionNumber, Nothing} submodule::Union{String, Nothing} other::Dict{String,Any} end diff --git a/src/manifest.jl b/src/manifest.jl index 57eb3db2c4..1101883ac3 100644 --- a/src/manifest.jl +++ b/src/manifest.jl @@ -89,7 +89,6 @@ function read_apps(apps::Dict) submodule = get(app, "submodule", nothing) appinfo = AppInfo(appname::String, app["julia_command"]::String, - VersionNumber(app["julia_version"]::String), submodule, app) appinfos[appinfo.name] = appinfo @@ -345,8 +344,7 @@ function destructure(manifest::Manifest)::Dict new_entry["apps"] = Dict{String,Any}() for (appname, appinfo) in entry.apps julia_command = @something appinfo.julia_command joinpath(Sys.BINDIR, "julia" * (Sys.iswindows() ? ".exe" : "")) - julia_version = @something appinfo.julia_version VERSION - app_dict = Dict{String,Any}("julia_command" => julia_command, "julia_version" => julia_version) + app_dict = Dict{String,Any}("julia_command" => julia_command) if appinfo.submodule !== nothing app_dict["submodule"] = appinfo.submodule end diff --git a/src/project.jl b/src/project.jl index d4a2a0c47f..bb5aeb9b9b 100644 --- a/src/project.jl +++ b/src/project.jl @@ -85,7 +85,7 @@ function read_project_apps(raw::Dict{String,Any}, project::Project) Expected value for app `$name` to be a dictionary. """) submodule = get(info, "submodule", nothing) - appinfos[name] = AppInfo(name, nothing, nothing, submodule, other) + appinfos[name] = AppInfo(name, nothing, submodule, other) end return appinfos end From 0640da5ed97997b7e09db7030d2b6342f8779a80 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 1 Jul 2025 15:59:16 +0200 Subject: [PATCH 081/154] Document [sources] section keys (#4281) Co-authored-by: KristofferC Co-authored-by: Claude --- docs/src/toml-files.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/src/toml-files.md b/docs/src/toml-files.md index 74e606250b..b731c1c2ba 100644 --- a/docs/src/toml-files.md +++ b/docs/src/toml-files.md @@ -22,7 +22,7 @@ are described below. ### The `authors` field -For a package, the optional `authors` field is a TOML array describing the package authors. +For a package, the optional `authors` field is a TOML array describing the package authors. Entries in the array can either be a string in the form `"NAME"` or `"NAME "`, or a table keys following the [Citation File Format schema](https://github.com/citation-file-format/citation-file-format/blob/main/schema-guide.md) for either a [`person`](https://github.com/citation-file-format/citation-file-format/blob/main/schema-guide.md#definitionsperson) or an[`entity`](https://github.com/citation-file-format/citation-file-format/blob/main/schema-guide.md#definitionsentity). @@ -122,6 +122,15 @@ Specifiying a path or repo (+ branch) for a dependency is done in the `[sources] These are especially useful for controlling unregistered dependencies without having to bundle a corresponding manifest file. +Each entry in the `[sources]` section supports the following keys: + +- **`url`**: The URL of the Git repository. Cannot be used with `path`. +- **`rev`**: The Git revision (branch name, tag, or commit hash) to use. Only valid with `url`. +- **`subdir`**: A subdirectory within the repository containing the package. +- **`path`**: A local filesystem path to the package. Cannot be used with `url` or `rev`. + +This might in practice look something like: + ```toml [sources] Example = {url = "https://github.com/JuliaLang/Example.jl", rev = "custom_branch"} From aaff8f2f98b64d1d33a2706aa0ee1ec5a5047a8f Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 1 Jul 2025 15:59:36 +0200 Subject: [PATCH 082/154] Various improvements to docs and docstrings (#4279) --- docs/make.jl | 1 + docs/src/api.md | 2 +- docs/src/apps.md | 24 ++++---- docs/src/artifacts.md | 2 +- docs/src/compatibility.md | 4 +- docs/src/creating-packages.md | 11 +++- docs/src/environments.md | 20 +++---- docs/src/getting-started.md | 36 ++++++------ docs/src/glossary.md | 4 +- docs/src/managing-packages.md | 82 ++++++++++++++-------------- docs/src/registries.md | 2 +- docs/src/toml-files.md | 2 +- src/Pkg.jl | 28 +++++++++- src/REPLMode/command_declarations.jl | 39 +++++++++---- 14 files changed, 154 insertions(+), 103 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index be6905de5a..976c1e4a21 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -35,6 +35,7 @@ makedocs( "managing-packages.md", "environments.md", "creating-packages.md", + "apps.md", "compatibility.md", "registries.md", "artifacts.md", diff --git a/docs/src/api.md b/docs/src/api.md index 61979453b9..dc9e9e1794 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -1,4 +1,4 @@ -# [**12.** API Reference](@id API-Reference) +# [**13.** API Reference](@id API-Reference) This section describes the functional API for interacting with Pkg.jl. It is recommended to use the functional API, rather than the Pkg REPL mode, diff --git a/docs/src/apps.md b/docs/src/apps.md index 083d68f0e1..00b12cada9 100644 --- a/docs/src/apps.md +++ b/docs/src/apps.md @@ -1,4 +1,4 @@ -# [**?.** Apps](@id Apps) +# [**6.** Apps](@id Apps) !!! note The app support in Pkg is currently considered experimental and some functionality and API may change. @@ -8,7 +8,7 @@ - The path to the julia executable used is the same as the one used to install the app. If this julia installation gets removed, you might need to reinstall the app. -Apps are Julia packages that are intended to be run as a "standalone programs" (by e.g. typing the name of the app in the terminal possibly together with some arguments or flags/options). +Apps are Julia packages that are intended to be run as "standalone programs" (by e.g. typing the name of the app in the terminal possibly together with some arguments or flags/options). This is in contrast to most Julia packages that are used as "libraries" and are loaded by other files or in the Julia REPL. ## Creating a Julia app @@ -44,6 +44,15 @@ reverse = {} ``` The empty table `{}` is to allow for giving metadata about the app but it is currently unused. +After installing this app one could run: + +``` +$ reverse some input string + emos tupni gnirts +``` + +directly in the terminal. + ## Multiple Apps per Package A single package can define multiple apps by using submodules. Each app can have its own entry point in a different submodule of the package. @@ -86,15 +95,6 @@ This will create two executables: - `main-app` that runs `julia -m MyMultiApp` - `cli-app` that runs `julia -m MyMultiApp.CLI` -After installing this app one could run: - -``` -$ reverse some input string -emos tupni gnirts -``` - -directly in the terminal. - ## Installing Julia apps -The installation of Julia apps are similar to installing julia libraries but instead of using e.g. `Pkg.add` or `pkg> add` one uses `Pkg.Apps.add` or `pkg> app add` (`develop` is also available). +The installation of Julia apps is similar to [installing Julia libraries](@ref Managing-Packages) but instead of using e.g. `Pkg.add` or `pkg> add` one uses `Pkg.Apps.add` or `pkg> app add` (`develop` is also available). diff --git a/docs/src/artifacts.md b/docs/src/artifacts.md index 66a55f99f5..d804dfeb10 100644 --- a/docs/src/artifacts.md +++ b/docs/src/artifacts.md @@ -1,4 +1,4 @@ -# [**8.** Artifacts](@id Artifacts) +# [**9.** Artifacts](@id Artifacts) `Pkg` can install and manage containers of data that are not Julia packages. These containers can contain platform-specific binaries, datasets, text, or any other kind of data that would be convenient to place within an immutable, life-cycled datastore. These containers, (called "Artifacts") can be created locally, hosted anywhere, and automatically downloaded and unpacked upon installation of your Julia package. diff --git a/docs/src/compatibility.md b/docs/src/compatibility.md index 3aab8b75d6..8115eea833 100644 --- a/docs/src/compatibility.md +++ b/docs/src/compatibility.md @@ -1,4 +1,4 @@ -# [**6.** Compatibility](@id Compatibility) +# [**7.** Compatibility](@id Compatibility) Compatibility refers to the ability to restrict the versions of the dependencies that your project is compatible with. If the compatibility for a dependency is not given, the project is assumed to be compatible with all versions of that dependency. @@ -164,7 +164,7 @@ PkgA = "0.2 - 0" # 0.2.0 - 0.*.* = [0.2.0, 1.0.0) ``` -## Fixing conflicts +## [Fixing conflicts](@id Fixing-conflicts) Version conflicts were introduced previously with an [example](@ref conflicts) of a conflict arising in a package `D` used by two other packages, `B` and `C`. diff --git a/docs/src/creating-packages.md b/docs/src/creating-packages.md index 0d7a23a286..ef3a8fc734 100644 --- a/docs/src/creating-packages.md +++ b/docs/src/creating-packages.md @@ -11,7 +11,7 @@ To generate the bare minimum files for a new package, use `pkg> generate`. ```julia-repl -(@v1.8) pkg> generate HelloWorld +(@v1.10) pkg> generate HelloWorld ``` This creates a new project `HelloWorld` in a subdirectory by the same name, with the following files (visualized with the external [`tree` command](https://linux.die.net/man/1/tree)): @@ -118,7 +118,7 @@ describe about public symbols. A public symbol is a symbol that is exported from package with the `export` keyword or marked as public with the `public` keyword. When you change the behavior of something that was previously public so that the new version no longer conforms to the specifications provided in the old version, you should -adjust your package version number according to [Julia's variant on SemVer](#Version-specifier-format). +adjust your package version number according to [Julia's variant on SemVer](@ref Version-specifier-format). If you would like to include a symbol in your public API without exporting it into the global namespace of folks who call `using YourPackage`, you should mark that symbol as public with `public that_symbol`. Symbols marked as public with the `public` keyword are @@ -653,3 +653,10 @@ To support the various use cases in the Julia package ecosystem, the Pkg develop * [`Preferences.jl`](https://github.com/JuliaPackaging/Preferences.jl) allows packages to read and write preferences to the top-level `Project.toml`. These preferences can be read at runtime or compile-time, to enable or disable different aspects of package behavior. Packages previously would write out files to their own package directories to record options set by the user or environment, but this is highly discouraged now that `Preferences` is available. + +## See Also + +- [Managing Packages](@ref Managing-Packages) - Learn how to add, update, and manage package dependencies +- [Working with Environments](@ref Working-with-Environments) - Understand environments and reproducible development +- [Compatibility](@ref Compatibility) - Specify version constraints for dependencies +- [API Reference](@ref) - Functional API for non-interactive package management diff --git a/docs/src/environments.md b/docs/src/environments.md index fd16427e10..d22ef8e58b 100644 --- a/docs/src/environments.md +++ b/docs/src/environments.md @@ -10,7 +10,7 @@ It should be pointed out that when two projects use the same package at the same In order to create a new project, create a directory for it and then activate that directory to make it the "active project", which package operations manipulate: ```julia-repl -(@v1.9) pkg> activate MyProject +(@v1.10) pkg> activate MyProject Activating new environment at `~/MyProject/Project.toml` (MyProject) pkg> st @@ -28,7 +28,7 @@ false Installed Example ─ v0.5.3 Updating `~/MyProject/Project.toml` [7876af07] + Example v0.5.3 - Updating `~~/MyProject/Manifest.toml` + Updating `~/MyProject/Manifest.toml` [7876af07] + Example v0.5.3 Precompiling environment... 1 dependency successfully precompiled in 2 seconds @@ -45,7 +45,7 @@ Example = "7876af07-990d-54b4-ab0e-23690620f79a" julia> print(read(joinpath("MyProject", "Manifest.toml"), String)) # This file is machine-generated - editing it directly is not advised -julia_version = "1.9.4" +julia_version = "1.10.0" manifest_format = "2.0" project_hash = "2ca1c6c58cb30e79e021fb54e5626c96d05d5fdc" @@ -66,7 +66,7 @@ shell> git clone https://github.com/JuliaLang/Example.jl.git Cloning into 'Example.jl'... ... -(@v1.12) pkg> activate Example.jl +(@v1.10) pkg> activate Example.jl Activating project at `~/Example.jl` (Example) pkg> instantiate @@ -82,7 +82,7 @@ If you only have a `Project.toml`, a `Manifest.toml` must be generated by "resol If you already have a resolved `Manifest.toml`, then you will still need to ensure that the packages are installed and with the correct versions. Again `instantiate` does this for you. -In short, `instantiate` is your friend to make sure an environment is ready to use. If there's nothing to do, `instantiate` does nothing. +In short, [`instantiate`](@ref Pkg.instantiate) is your friend to make sure an environment is ready to use. If there's nothing to do, `instantiate` does nothing. !!! note "Specifying project on startup" Instead of using `activate` from within Julia, you can specify the project on startup using @@ -103,7 +103,7 @@ also want a scratch space to try out a new package, or a sandbox to resolve vers between several incompatible packages. ```julia-repl -(@v1.9) pkg> activate --temp # requires Julia 1.5 or later +(@v1.10) pkg> activate --temp # requires Julia 1.5 or later Activating new environment at `/var/folders/34/km3mmt5930gc4pzq1d08jvjw0000gn/T/jl_a31egx/Project.toml` (jl_a31egx) pkg> add Example @@ -121,14 +121,14 @@ A "shared" environment is simply an environment that exists in `~/.julia/environ therefore a shared environment: ```julia-repl -(@v1.9) pkg> st +(@v1.10) pkg> st Status `~/.julia/environments/v1.9/Project.toml` ``` Shared environments can be activated with the `--shared` flag to `activate`: ```julia-repl -(@v1.9) pkg> activate --shared mysharedenv +(@v1.10) pkg> activate --shared mysharedenv Activating project at `~/.julia/environments/mysharedenv` (@mysharedenv) pkg> @@ -151,7 +151,7 @@ or using Pkg's precompile option, which can precompile the entire environment, o which can be significantly faster than the code-load route above. ```julia-repl -(@v1.9) pkg> precompile +(@v1.10) pkg> precompile Precompiling environment... 23 dependencies successfully precompiled in 36 seconds ``` @@ -165,7 +165,7 @@ By default, any package that is added to a project or updated in a Pkg action wi with its dependencies. ```julia-repl -(@v1.9) pkg> add Images +(@v1.10) pkg> add Images Resolving package versions... Updating `~/.julia/environments/v1.9/Project.toml` [916415d5] + Images v0.25.2 diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 58693bc583..93acb7c613 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -22,18 +22,18 @@ To get back to the Julia REPL, press `Ctrl+C` or backspace (when the REPL cursor Upon entering the Pkg REPL, you should see the following prompt: ```julia-repl -(@v1.9) pkg> +(@v1.10) pkg> ``` To add a package, use `add`: ```julia-repl -(@v1.9) pkg> add Example +(@v1.10) pkg> add Example Resolving package versions... Installed Example ─ v0.5.3 - Updating `~/.julia/environments/v1.9/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [7876af07] + Example v0.5.3 - Updating `~/.julia/environments/v1.9/Manifest.toml` + Updating `~/.julia/environments/v1.10/Manifest.toml` [7876af07] + Example v0.5.3 ``` @@ -49,14 +49,14 @@ julia> Example.hello("friend") We can also specify multiple packages at once to install: ```julia-repl -(@v1.9) pkg> add JSON StaticArrays +(@v1.10) pkg> add JSON StaticArrays ``` The `status` command (or the shorter `st` command) can be used to see installed packages. ```julia-repl -(@v1.9) pkg> st -Status `~/.julia/environments/v1.6/Project.toml` +(@v1.10) pkg> st +Status `~/.julia/environments/v1.10/Project.toml` [7876af07] Example v0.5.3 [682c06a0] JSON v0.21.3 [90137ffa] StaticArrays v1.5.9 @@ -68,13 +68,13 @@ Status `~/.julia/environments/v1.6/Project.toml` To remove packages, use `rm` (or `remove`): ```julia-repl -(@v1.9) pkg> rm JSON StaticArrays +(@v1.10) pkg> rm JSON StaticArrays ``` Use `up` (or `update`) to update the installed packages ```julia-repl -(@v1.9) pkg> up +(@v1.10) pkg> up ``` If you have been following this guide it is likely that the packages installed are at the latest version @@ -82,13 +82,13 @@ so `up` will not do anything. Below we show the status output in the case where an old version of the Example package and then upgrade it: ```julia-repl -(@v1.9) pkg> st -Status `~/.julia/environments/v1.9/Project.toml` +(@v1.10) pkg> st +Status `~/.julia/environments/v1.10/Project.toml` ⌃ [7876af07] Example v0.5.1 Info Packages marked with ⌃ have new versions available and may be upgradable. -(@v1.9) pkg> up - Updating `~/.julia/environments/v1.9/Project.toml` +(@v1.10) pkg> up + Updating `~/.julia/environments/v1.10/Project.toml` [7876af07] ↑ Example v0.5.1 ⇒ v0.5.3 ``` @@ -110,7 +110,7 @@ Let's set up a new environment so we may experiment. To set the active environment, use `activate`: ```julia-repl -(@v1.9) pkg> activate tutorial +(@v1.10) pkg> activate tutorial [ Info: activating new environment at `~/tutorial/Project.toml`. ``` @@ -166,16 +166,16 @@ For more information about environments, see the [Working with Environments](@re If you are ever stuck, you can ask `Pkg` for help: ```julia-repl -(@v1.9) pkg> ? +(@v1.10) pkg> ? ``` You should see a list of available commands along with short descriptions. You can ask for more detailed help by specifying a command: ```julia-repl -(@v1.9) pkg> ?develop +(@v1.10) pkg> ?develop ``` This guide should help you get started with `Pkg`. -`Pkg` has much more to offer in terms of powerful package management, -read the full manual to learn more! +`Pkg` has much more to offer in terms of powerful package management. +For more advanced topics, see [Managing Packages](@ref Managing-Packages), [Working with Environments](@ref Working-with-Environments), and [Creating Packages](@ref creating-packages-tutorial). diff --git a/docs/src/glossary.md b/docs/src/glossary.md index 60e0546039..54c00aa8ea 100644 --- a/docs/src/glossary.md +++ b/docs/src/glossary.md @@ -1,4 +1,4 @@ -# [**9.** Glossary](@id Glossary) +# [**10.** Glossary](@id Glossary) **Project:** a source tree with a standard layout, including a `src` directory for the main body of Julia code, a `test` directory for testing the project, @@ -46,7 +46,7 @@ since that could conflict with the configuration of the main application. **Environment:** the combination of the top-level name map provided by a project file combined with the dependency graph and map from packages to their entry points -provided by a manifest file. For more detail see the manual section on code loading. +provided by a manifest file. For more detail see the [manual section on code loading](https://docs.julialang.org/en/v1/manual/code-loading/). - **Explicit environment:** an environment in the form of an explicit project file and an optional corresponding manifest file together in a directory. If the diff --git a/docs/src/managing-packages.md b/docs/src/managing-packages.md index b5889221cf..ec6cf34be8 100644 --- a/docs/src/managing-packages.md +++ b/docs/src/managing-packages.md @@ -10,14 +10,14 @@ The most frequently used is `add` and its usage is described first. In the Pkg REPL, packages can be added with the `add` command followed by the name of the package, for example: ```julia-repl -(@v1.8) pkg> add JSON +(@v1.10) pkg> add JSON Installing known registries into `~/` Resolving package versions... Installed Parsers ─ v2.4.0 Installed JSON ──── v0.21.3 - Updating `~/.julia/environments/v1.8/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [682c06a0] + JSON v0.21.3 - Updating `~/environments/v1.9/Manifest.toml` + Updating `~/.julia/environments/v1.10/Manifest.toml` [682c06a0] + JSON v0.21.3 [69de0a69] + Parsers v2.4.0 [ade2ca70] + Dates @@ -40,16 +40,16 @@ It is possible to add multiple packages in one command as `pkg> add A B C`. The status output contains the packages you have added yourself, in this case, `JSON`: ```julia-repl -(@v1.11) pkg> st - Status `~/.julia/environments/v1.8/Project.toml` +(@v1.10) pkg> st + Status `~/.julia/environments/v1.10/Project.toml` [682c06a0] JSON v0.21.3 ``` The manifest status shows all the packages in the environment, including recursive dependencies: ```julia-repl -(@v1.11) pkg> st -m -Status `~/environments/v1.9/Manifest.toml` +(@v1.10) pkg> st -m +Status `~/.julia/environments/v1.10/Manifest.toml` [682c06a0] JSON v0.21.3 [69de0a69] Parsers v2.4.0 [ade2ca70] Dates @@ -64,18 +64,18 @@ To specify that you want a particular version (or set of versions) of a package, to require any patch release of the v0.21 series of JSON after v0.21.4, call `compat JSON 0.21.4`: ```julia-repl -(@1.11) pkg> compat JSON 0.21.4 +(@v1.10) pkg> compat JSON 0.21.4 Compat entry set: JSON = "0.21.4" Resolve checking for compliance with the new compat rules... Error empty intersection between JSON@0.21.3 and project compatibility 0.21.4 - 0.21 Suggestion Call `update` to attempt to meet the compatibility requirements. -(@1.11) pkg> update +(@v1.10) pkg> update Updating registry at `~/.julia/registries/General.toml` - Updating `~/.julia/environments/1.11/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [682c06a0] ↑ JSON v0.21.3 ⇒ v0.21.4 - Updating `~/.julia/environments/1.11/Manifest.toml` + Updating `~/.julia/environments/v1.10/Manifest.toml` [682c06a0] ↑ JSON v0.21.3 ⇒ v0.21.4 ``` @@ -96,11 +96,11 @@ julia> JSON.json(Dict("foo" => [1, "bar"])) |> print A specific version of a package can be installed by appending a version after a `@` symbol to the package name: ```julia-repl -(@v1.8) pkg> add JSON@0.21.1 +(@v1.10) pkg> add JSON@0.21.1 Resolving package versions... - Updating `~/.julia/environments/v1.8/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` ⌃ [682c06a0] + JSON v0.21.1 - Updating `~/environments/v1.9/Manifest.toml` + Updating `~/.julia/environments/v1.10/Manifest.toml` ⌃ [682c06a0] + JSON v0.21.1 ⌅ [69de0a69] + Parsers v1.1.2 [ade2ca70] + Dates @@ -118,12 +118,12 @@ If a branch (or a certain commit) of `Example` has a hotfix that is not yet incl we can explicitly track that branch (or commit) by appending `#branchname` (or `#commitSHA1`) to the package name: ```julia-repl -(@v1.8) pkg> add Example#master +(@v1.10) pkg> add Example#master Cloning git-repo `https://github.com/JuliaLang/Example.jl.git` Resolving package versions... - Updating `~/.julia/environments/v1.8/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [7876af07] + Example v0.5.4 `https://github.com/JuliaLang/Example.jl.git#master` - Updating `~/environments/v1.9/Manifest.toml` + Updating `~/.julia/environments/v1.10/Manifest.toml` [7876af07] + Example v0.5.4 `https://github.com/JuliaLang/Example.jl.git#master` ``` @@ -139,12 +139,12 @@ When updating packages, updates are pulled from that branch. To go back to tracking the registry version of `Example`, the command `free` is used: ```julia-repl -(@v1.8) pkg> free Example +(@v1.10) pkg> free Example Resolving package versions... Installed Example ─ v0.5.3 - Updating `~/.julia/environments/v1.8/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [7876af07] ~ Example v0.5.4 `https://github.com/JuliaLang/Example.jl.git#master` ⇒ v0.5.3 - Updating `~/environments/v1.9/Manifest.toml` + Updating `~/.julia/environments/v1.10/Manifest.toml` [7876af07] ~ Example v0.5.4 `https://github.com/JuliaLang/Example.jl.git#master` ⇒ v0.5.3 ``` @@ -153,12 +153,12 @@ To go back to tracking the registry version of `Example`, the command `free` is If a package is not in a registry, it can be added by specifying a URL to the Git repository: ```julia-repl -(@v1.8) pkg> add https://github.com/fredrikekre/ImportMacros.jl +(@v1.10) pkg> add https://github.com/fredrikekre/ImportMacros.jl Cloning git-repo `https://github.com/fredrikekre/ImportMacros.jl` Resolving package versions... - Updating `~/.julia/environments/v1.8/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [92a963f6] + ImportMacros v1.0.0 `https://github.com/fredrikekre/ImportMacros.jl#master` - Updating `~/environments/v1.9/Manifest.toml` + Updating `~/.julia/environments/v1.10/Manifest.toml` [92a963f6] + ImportMacros v1.0.0 `https://github.com/fredrikekre/ImportMacros.jl#master` ``` @@ -167,7 +167,7 @@ For unregistered packages, we could have given a branch name (or commit SHA1) to If you want to add a package using the SSH-based `git` protocol, you have to use quotes because the URL contains a `@`. For example, ```julia-repl -(@v1.8) pkg> add "git@github.com:fredrikekre/ImportMacros.jl.git" +(@v1.10) pkg> add "git@github.com:fredrikekre/ImportMacros.jl.git" Cloning git-repo `git@github.com:fredrikekre/ImportMacros.jl.git` Updating registry at `~/.julia/registries/General` Resolving package versions... @@ -188,7 +188,7 @@ repository: pkg> add https://github.com/timholy/SnoopCompile.jl.git:SnoopCompileCore Cloning git-repo `https://github.com/timholy/SnoopCompile.jl.git` Resolving package versions... - Updating `~/.julia/environments/v1.8/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [e2b509da] + SnoopCompileCore v2.9.0 `https://github.com/timholy/SnoopCompile.jl.git:SnoopCompileCore#master` Updating `~/.julia/environments/v1.8/Manifest.toml` [e2b509da] + SnoopCompileCore v2.9.0 `https://github.com/timholy/SnoopCompile.jl.git:SnoopCompileCore#master` @@ -214,15 +214,15 @@ from that local repo are pulled when packages are updated. By only using `add` your environment always has a "reproducible state", in other words, as long as the repositories and registries used are still accessible it is possible to retrieve the exact state of all the dependencies in the environment. This has the advantage that you can send your environment (`Project.toml` and `Manifest.toml`) to someone else and they can [`Pkg.instantiate`](@ref) that environment in the same state as you had it locally. -However, when you are developing a package, it is more convenient to load packages at their current state at some path. For this reason, the `dev` command exists. +However, when you are [developing a package](@ref developing), it is more convenient to load packages at their current state at some path. For this reason, the `dev` command exists. Let's try to `dev` a registered package: ```julia-repl -(@v1.8) pkg> dev Example +(@v1.10) pkg> dev Example Updating git-repo `https://github.com/JuliaLang/Example.jl.git` Resolving package versions... - Updating `~/.julia/environments/v1.8/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [7876af07] + Example v0.5.4 `~/.julia/dev/Example` Updating `~/.julia/environments/v1.8/Manifest.toml` [7876af07] + Example v0.5.4 `~/.julia/dev/Example` @@ -263,9 +263,9 @@ julia> Example.plusone(1) To stop tracking a path and use the registered version again, use `free`: ```julia-repl -(@v1.8) pkg> free Example +(@v1.10) pkg> free Example Resolving package versions... - Updating `~/.julia/environments/v1.8/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [7876af07] ~ Example v0.5.4 `~/.julia/dev/Example` ⇒ v0.5.3 Updating `~/.julia/environments/v1.8/Manifest.toml` [7876af07] ~ Example v0.5.4 `~/.julia/dev/Example` ⇒ v0.5.3 @@ -300,29 +300,29 @@ When new versions of packages are released, it is a good idea to update. Simply to the latest compatible version. Sometimes this is not what you want. You can specify a subset of the dependencies to upgrade by giving them as arguments to `up`, e.g: ```julia-repl -(@v1.8) pkg> up Example +(@v1.10) pkg> up Example ``` This will only allow Example do upgrade. If you also want to allow dependencies of Example to upgrade (with the exception of packages that are in the project) you can pass the `--preserve=direct` flag. ```julia-repl -(@v1.8) pkg> up --preserve=direct Example +(@v1.10) pkg> up --preserve=direct Example ``` And if you also want to allow dependencies of Example that are also in the project to upgrade, you can use `--preserve=none`: ```julia-repl -(@v1.8) pkg> up --preserve=none Example +(@v1.10) pkg> up --preserve=none Example ``` ## Pinning a package A pinned package will never be updated. A package can be pinned using `pin`, for example: ```julia-repl -(@v1.8) pkg> pin Example +(@v1.10) pkg> pin Example Resolving package versions... - Updating `~/.julia/environments/v1.8/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [7876af07] ~ Example v0.5.3 ⇒ v0.5.3 ⚲ Updating `~/.julia/environments/v1.8/Manifest.toml` [7876af07] ~ Example v0.5.3 ⇒ v0.5.3 ⚲ @@ -331,8 +331,8 @@ A pinned package will never be updated. A package can be pinned using `pin`, for Note the pin symbol `⚲` showing that the package is pinned. Removing the pin is done using `free` ```julia-repl -(@v1.8) pkg> free Example - Updating `~/.julia/environments/v1.8/Project.toml` +(@v1.10) pkg> free Example + Updating `~/.julia/environments/v1.10/Project.toml` [7876af07] ~ Example v0.5.3 ⚲ ⇒ v0.5.3 Updating `~/.julia/environments/v1.8/Manifest.toml` [7876af07] ~ Example v0.5.3 ⚲ ⇒ v0.5.3 @@ -343,7 +343,7 @@ Note the pin symbol `⚲` showing that the package is pinned. Removing the pin i The tests for a package can be run using `test` command: ```julia-repl -(@v1.8) pkg> test Example +(@v1.10) pkg> test Example ... Testing Example Testing Example tests passed @@ -356,7 +356,7 @@ The output of the build process is directed to a file. To explicitly run the build step for a package, the `build` command is used: ```julia-repl -(@v1.8) pkg> build IJulia +(@v1.10) pkg> build IJulia Building Conda ─→ `~/.julia/scratchspaces/44cfe95a-1eb2-52ea-b672-e2afdf69b78f/6e47d11ea2776bc5627421d59cdcc1296c058071/build.log` Building IJulia → `~/.julia/scratchspaces/44cfe95a-1eb2-52ea-b672-e2afdf69b78f/98ab633acb0fe071b671f6c1785c46cd70bb86bd/build.log` @@ -486,7 +486,7 @@ To fix such errors, you have a number of options: - remove either `A` or `B` from your environment. Perhaps `B` is left over from something you were previously working on, and you don't need it anymore. If you don't need `A` and `B` at the same time, this is the easiest way to fix the problem. - try reporting your conflict. In this case, we were able to deduce that `B` requires an outdated version of `D`. You could thus report an issue in the development repository of `B.jl` asking for an updated version. - try fixing the problem yourself. - This becomes easier once you understand `Project.toml` files and how they declare their compatibility requirements. We'll return to this example in [Fixing conflicts](@ref). + This becomes easier once you understand `Project.toml` files and how they declare their compatibility requirements. We'll return to this example in [Fixing conflicts](@ref Fixing-conflicts). ## Garbage collecting old, unused packages @@ -502,7 +502,7 @@ If you are short on disk space and want to clean out as many unused packages and To run a typical garbage collection with default arguments, simply use the `gc` command at the `pkg>` REPL: ```julia-repl -(@v1.8) pkg> gc +(@v1.10) pkg> gc Active manifests at: `~/BinaryProvider/Manifest.toml` ... diff --git a/docs/src/registries.md b/docs/src/registries.md index 7c50727204..cada0bdadf 100644 --- a/docs/src/registries.md +++ b/docs/src/registries.md @@ -1,4 +1,4 @@ -# **7.** Registries +# **8.** Registries Registries contain information about packages, such as available releases and dependencies, and where they can be downloaded. diff --git a/docs/src/toml-files.md b/docs/src/toml-files.md index b731c1c2ba..0249970a02 100644 --- a/docs/src/toml-files.md +++ b/docs/src/toml-files.md @@ -1,4 +1,4 @@ -# [**10.** `Project.toml` and `Manifest.toml`](@id Project-and-Manifest) +# [**11.** `Project.toml` and `Manifest.toml`](@id Project-and-Manifest) Two files that are central to Pkg are `Project.toml` and `Manifest.toml`. `Project.toml` and `Manifest.toml` are written in [TOML](https://github.com/toml-lang/toml) (hence the diff --git a/src/Pkg.jl b/src/Pkg.jl index a43fa91de9..399d9572cf 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -380,8 +380,13 @@ To get updates from the origin path or remote repository the package must first # Examples ```julia +# Pin a package to its current version Pkg.pin("Example") + +# Pin a package to a specific version Pkg.pin(name="Example", version="0.3.1") + +# Pin all packages in the project Pkg.pin(all_pkgs = true) ``` """ @@ -400,7 +405,13 @@ To free all dependencies set `all_pkgs=true`. # Examples ```julia +# Free a single package (remove pin or stop tracking path) Pkg.free("Package") + +# Free multiple packages +Pkg.free(["PackageA", "PackageB"]) + +# Free all packages in the project Pkg.free(all_pkgs = true) ``` @@ -711,7 +722,17 @@ Other choices for `protocol` are `"https"` or `"git"`. ```julia-repl julia> Pkg.setprotocol!(domain = "github.com", protocol = "ssh") +# Use HTTPS for GitHub (default, good for most users) +julia> Pkg.setprotocol!(domain = "github.com", protocol = "https") + +# Reset to default (let package developer decide) +julia> Pkg.setprotocol!(domain = "github.com", protocol = nothing) + +# Set protocol for custom domain without specifying protocol julia> Pkg.setprotocol!(domain = "gitlab.mycompany.com") + +# Use Git protocol for a custom domain +julia> Pkg.setprotocol!(domain = "gitlab.mycompany.com", protocol = "git") ``` """ const setprotocol! = API.setprotocol! @@ -786,8 +807,11 @@ If the manifest doesn't have the project hash recorded, or if there is no manife This function can be used in tests to verify that the manifest is synchronized with the project file: - using Pkg, Test, Package - @test Pkg.is_manifest_current(pkgdir(Package)) +```julia +using Pkg, Test +@test Pkg.is_manifest_current(pwd()) # Check current project +@test Pkg.is_manifest_current("/path/to/project") # Check specific project +``` """ const is_manifest_current = API.is_manifest_current diff --git a/src/REPLMode/command_declarations.jl b/src/REPLMode/command_declarations.jl index 98153e5b48..9e016c3f76 100644 --- a/src/REPLMode/command_declarations.jl +++ b/src/REPLMode/command_declarations.jl @@ -16,7 +16,7 @@ PSA[:name => "test", test [--coverage] [pkg[=uuid]] ... Run the tests for package `pkg`, or for the current project (which thus needs to be -a package) if `pkg` is ommitted. This is done by running the file `test/runtests.jl` +a package) if `pkg` is omitted. This is done by running the file `test/runtests.jl` in the package directory. The option `--coverage` can be used to run the tests with coverage enabled. The `startup.jl` file is disabled during testing unless julia is started with `--startup-file=yes`. @@ -592,7 +592,10 @@ pkg> registry status :completions => :complete_installed_apps, :description => "show status of apps", :help => md""" - show status of apps + app status [pkg[=uuid]] ... + +Show the status of installed apps. If packages are specified, only show +apps for those packages. """ ], PSA[:name => "add", @@ -603,9 +606,15 @@ PSA[:name => "add", :completions => :complete_add_dev, :description => "add app", :help => md""" - app add pkg + app add pkg[=uuid] ... + +Add apps provided by packages `pkg...`. This will make the apps available +as executables in `~/.julia/bin` (which should be added to PATH). -Adds the apps for packages `pkg...` or apps `app...`. +**Examples** +``` +pkg> app add Example +pkg> app add Example@0.5.0 ``` """, ], @@ -616,12 +625,17 @@ PSA[:name => "remove", :arg_count => 0 => Inf, :arg_parser => parse_package, :completions => :complete_installed_apps, - :description => "remove packages from project or manifest", + :description => "remove apps", :help => md""" - app [rm|remove] pkg ... - app [rm|remove] app ... + app [rm|remove] pkg[=uuid] ... + +Remove apps provided by packages `pkg...`. This will remove the executables +from `~/.julia/bin`. - Remove the apps for package `pkg`. +**Examples** +``` +pkg> app rm Example +``` """ ], PSA[:name => "develop", @@ -657,9 +671,14 @@ PSA[:name => "update", :arg_parser => parse_package, :description => "update app", :help => md""" - app update pkg + app [up|update] [pkg[=uuid]] ... -Updates the apps for packages `pkg...` or apps `app...`. +Update apps for packages `pkg...`. If no packages are specified, all apps will be updated. + +**Examples** +``` +pkg> app update +pkg> app update Example ``` """, ], # app From 95490a5e8fc4ac6374f5879008f18386fff2afbe Mon Sep 17 00:00:00 2001 From: abhro <5664668+abhro@users.noreply.github.com> Date: Tue, 1 Jul 2025 10:00:05 -0400 Subject: [PATCH 083/154] Create docs site favicon (#4275) --- docs/make.jl | 2 +- docs/src/assets/favicon.ico | Bin 0 -> 56558 bytes 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 docs/src/assets/favicon.ico diff --git a/docs/make.jl b/docs/make.jl index 976c1e4a21..8b3cb2c8b1 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -9,7 +9,7 @@ const formats = Any[ Documenter.HTML( prettyurls = get(ENV, "CI", nothing) == "true", canonical = "https://julialang.github.io/Pkg.jl/v1/", - assets = ["assets/custom.css"], + assets = ["assets/custom.css", "assets/favicon.ico"], ), ] if "pdf" in ARGS diff --git a/docs/src/assets/favicon.ico b/docs/src/assets/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..eeb1edd944276d4eaecc21b65c622718a395a991 GIT binary patch literal 56558 zcmeHw2Y4OFbuJt`@pGDDkpwFM0wmbGMDiB&-g~bXxM(Z@2=-2}h!m?>BvFN;x~OUu zt60^PWy`YcIF@@}@}Bj+6x&JcxWuw-6L;SK&)&J1-Mb3{A}O1N`c|`LcJJ=|=bSlp z5E1bS{MWxC0{@>J@xy-v5tsHN|6l5!*jv z%N@0}`4d;~Xk0yC4y-?eZS$D4x?hg3EEkb^)naU6)gMN=n;sqIYL32oM&j!GQsACC z+SB;n#HQlF)T3Sz-Ip!KSC@%tx4T7jZ;lv~QvWmb>BIIHSKsJdxnB%C$AZ64kIk$8 zeQbZKn0ALtZqciQ_y>=t9!U@r>PrVkd78d~dlIhP5xF|PY_zNKlfbcUbW+1lqE_Td zK3CYYjj`abuEp+16Op+!zX#3SHEL-S)v0U$F0&B;?(@ONXUA8S4@^0j7~*+_vBZCk zng8``@wd4})bhLlXy?axRx7qoU1mdYRlY88`Wx`=&5@b4znQ!#Q%pN)$a)=HwI4(L zYcT%@<-c7^+93_$nP>6rg;(XEUJf6fk`e)Yibi`I|7nuHFc5#-GbFCWI%e(p@s~6K z8k)Q*Lx83}z;7JH_P3YAkX$YA2s~oIpHGe}syY{UQ%V@_l;x8;3!vMXc|B%r7Wlu^ z)IV-lD&&5?;5}A(5P!lWCbbs^Mkh4>2)|iz{WYJ#AC<&lum1&b=!4z%^XOGMkoDo1 z#rCI(vAMM(uG}Zal~kE9jae6tfAp$cF+SZVBD3p6?6!1Me?<>d_9csPr4{EubI;@V zr~Od};VS)cjJtc3r|}!G*8;TL6gH}jKjsw^n@Yr_JfBE8>=WrHfPd-m@SnfZC(@4k zM0Al4@}XQzJ#3>tVosp^fFJ2sc$cl%{_-k?;$!`6jPv#2yW1jjYJTU$pOZIcipY!x zv7pB%iXQce(x-suG2mZTX`&O!|FvfMPyV0ZZQN7xxKAwJ=o4epn?%&gaQl(8MnBTT z=HdYSO5ek~WqcaedYq+c@pN_Ar5Tm_5v1F5h zA+eYEgP+HhTku~G`9B2y>fXHjd?LOMa{}g(*qv!68dCCxdc)WP=|_G9v4lxi3oakW zcLMhru;pJKS6(qNWq(rm_)Gt7uH;kVK6jZvxPvulqG+wVf>=w>Mdg zEvlCMnS0v6eBSbK{HX(suQKC5dBe!?pVj6Q#gF;KqBWQkG8zTv3Ok*1>S6ei>*z;r zd>1s}`E0_qIboz^fN{%&E&u(AzTyCNIh}90FKxQ^LNPwGStQ&9+@Boe_qoKsfAE<6 zn|u)XSC^Zf^HK0ub3)E3V@?=bP$S~@gxj2aN9rfgQ$7R@pTzf!9_bidjr}oBV=!J1 zjxDG@$2rxCyBt^g=V6~k<@*GAU16eHDgQ0_Cmk?!dHs3R6TzpZ-_Z3x^*pwe|KxxD z9QWsT`Jit!i;2Dx;Y6oV%kl%zTmB8-mU}f~I+Ab382K?L2RN7NSaCldx;E!t;+}eo zA>)UQ|9&5^Hv4}r2L6jyV?Mw}TZ1+SeTsJT6wiaMZym{4T#bE!f7_(?f-vl3psRCy z7xrNe4%&3o@tpAY>@@Rg;y&PS@&D1MJE5+`^jjn4e`3FQgYjEgclyMXD#Oo2ACXSK zG0-)x;eURLHt;Vrj~%d&F7yeGuP%Qb@rO*d;!pWa*`Z@{vGM1-l|T>i?DD||XfpYb zscT%re^~hY;bS-96y292=D{bYW6ymj{O8!<@7)9Z8}0BnpSuwLSMOKy1nh$};D762 z4C)&HpC5bn;QVik|B4af4<1x_u=VctNq=w{|Bt}_x6}W0`LBL2gufO;3gUlas8P6n zNgcocGeh;cT(9A~reS*o+j4B5cKQxn3&(#o_4 z)hz$%Yl8fD#2?=sJKEL!AhutQb~SCe{tBUk>z6ithTk(Iq5hY#-UgA7S~D=t-S{59 zw{Xl7Us%rd#loNbuH#RCz{Hkt{H^idizWYui$8o^e~0neGsWBX%Z98Cfk~cLQJS$f zFxKt+HpU_?qO#J7RsN*UfA)qJ%CdA|{hij_vcNmdX~LCKBOa*E5l_`-i}bYWKa6oT zy^KDKu3y?1`W8FzP1g6_#{5WE^FQRJ_nkXcaacT8bDP|Vf2`Lpo~gN2 z9)AvWuq=0LAllRVQ{2;r?UR4X@L%t0{uJJ^3v&La{Ut?#@3f|duqWoA2g(169@xHz zK08Ya15*XSLbK$4~akYeZBwEEx^CcY*%ppm-+vf0Dszl zdTfCH3(Ws7g4SPfC$IS3;nJH!xI5vm&fimUL}aG*4nP*1#rVwsQ=|dB*Nu_xrnf4x zDh9sTlwrbM;q6Tg$6sB0r#W3L&o3RA;BNfDiWBoRM}&Vn~C7XeNhZ%qB~ndEO>{SMJy8Ry z+HS~JL-)vuN%oGKg z)nbg>2VF1Mgnx2cwRo_`EdQq`G>SQ?K2e`jA>L|A7Z-|u`f<$jxkj#sQ{4}9cP2EJ zu*OS_$v_+#xqWu|-lT=N-Kk<+ah1#op5Y6pK|A@+9Fft<7W|RFLj3m??-TQrmWkYg z4PtUayLcS(T;WRmr&{=*Izd!In}|>96rH(S1!+h1SMB|kw}^uD)dR38m~Y~~*s|cS z7S&5zeyqFc+py&a-m&mCasM{-_zguRVn%YaIOz9_(xTQd{8NDcLl*p}0sn7oX&2j@ z{bEK^v&c-Z4wwJrfB5u=;{Sq&d}21{{m86(F?madiT4yw%ls_liAQGD3f2P>bLI}C z9vC^|HDdB+hx=InfoqXDbz*v>PvqP^_%62iBR*+|f3&w%EKKhf$GdJ9Z)|)_VyfeR zDu}zms}aZ!sc{*39^;m>)&1>1fc z=4;k?IP&pMyYJF$1dKkYpD^Sx_-P8}c#tD~$8TLQ)zY+S~Tjg05=j?3x??TptCd5q0 z>$pUpcyWyxdnf$i^A!)<$bZHEj`*+37IP5OQR7QlPkA6^J!?EQY(o3Y30AUxi1|I# zUwa1Yyr?&Fp5VF=x*u2XFY`PRQ=X93Eb|Q+_v1U#cKj&#bN(L={-lBD*FG$wK`Y5Q zYlJtYM-XqFZn#fB!$!~tbc$bLPS^(D+~yA$9I$+xEJ+B zhU{>t1>^6=A=ihs1*{iJIRyM42L5Ry=YQfqZ5aGd{9ge6(~`U7JksBCL`+NS626?x zCVN8B!^4;-Dl*pv#=3mp1-|K4K>*xo%U8koMq7Ts&X)fw{Bupv(HEw-h=;q|q>LxO zrV;szJF!yuzdTI*;V(EZ{wHkafBNaiq(iSqtB+}DUzhbky*_jeW3ToN zyRTGCgngKJvw1$DeG!?7{LKyabq4h1(+?S$U2Di5_=(hWO)caKb3G|H^z#=R{}+M( z)a1?({)!GL3r4$qA}@Wlpk1ir&wbjD%Z-@QPe2E)h|m7#EwtsK&*?fn_n)cD7D=hK zqOh=4G(xxE?emBD{-fZZ{Son}AAKmje%$$FeW*UDQR?feCQ#`Us?EIJC*mp~V^AmR zWCKXu5BaI21JZ+&ZlGh&ed6yskN!tpZ~8Fh|4Zv05mVt4P`Io1hJGO@ZGj%&D(0_j5wzb(3#TyF}jrNRYpUAy897`Qr^7lOO`b6-&cXx}-+t;s~_F;E21~e`mIb_Wxj(#CI zCn0u7{}E%0rue_)e=Gi7!i@hR#`i-QQD1YC|1ZM_6qh)B{AZ;u7Z2Lv-}oCH|2N^& zy8$+y;(OIzEc~tU{~_@Y%KEYN!we7a{NGYFH2(DA3>Sa; z0Ap?Nzmxo*Ys~*T|IdNXShoj){Qt{n5AfIGc#OdvSmpn;nKcC+o8+Tco_Vll9j77{|MDgP{-Ju&zbc zezT9P{jJBG09)UwMwN8Hm=WVj^hsbX!7%*kv(e||k>Wqu0smAR{O6{v5cgFK+JhSY z|3e#J>3?5={x>fzoc|^Mt>OCLNbzU>ulg;viBKt1O{kWENphZOGwc`0Lg6n_i&r|#KAw(u2w+J+- z>-zdW^%2GzM<+A`|DB=vn)CmK;Q#7|Gh%XLd+2$J*Y5)V+u=ViZKb%^iod6Y{Qqkm z|95=pVs>gc{%gVi`)u$}8X^DRZ^3_x1^<`eOJ0=TCdQZ6h$%OlYbf<`rCk@9W#I2P z*HhN-fvnFC%6j-^iCs40x6|sI#W;5ld?m2;5|)bv-OX|?AT7k#wus5_Z*$Ecg|)TK zxYZ}l=a1z3#_dXj&RT~UZl8Fl=X3}!JN!?MK>okF;ZYIei2s_`o`u)zsSi)K0qpBfi0!Bb@d!u$yMY7IB`VVnSN0 zyQg|BMwT5*?1$qmP!c?Wt9k;LC4@&A>M|KGPDMw%Rs{~GZB%{Kg>=zzan z{s;Ac=7lL*Q}X}0KC}Hm{CC3F$hhFz`gSo1@+i6oG8X!v8ehGgdK7+t=;bmFhxpL^ z?x3vSX~=rRzQ-C3nXM9M;(bHH*Ma}b5`XS_wRPVe^ExaV0=OOgNgqnga3c2=jk){GZk~1()(3A6#j3lAHe{GXLRbE~in%Zgq$O z1odvlhemr_LjL^>touhiqukpiz6xstZ{oj9Vy~Z{>3vL$_w*p<+$?iQlMi8zFF>4# zYXLg>co-j8mm_V&+OmG}?3#x{*sKNqoz;7!FXl+ctuhBFGp$eBb_x%+x5Gzg<$vPT zl)XtzN33US^I;k9BhIf6fq$|M|1*b(wxs$!iNB}i*HNCvU+DNVk1G~2eKq!~T@8QW zeh2<1{{ILat{*vE%n8_tJg{$pj&$s~PyA0KN8EazH6YuIi*aw0;P0%g@rzwHv4(~+ zKYSk04a%#b8`!V8LtbS57j=WlC6M*rUNO`Aco@ct-|g_{wM7YUiz%Ml#kj;aF&h5R zA$8H6>9AGm#gfcEaRbKL%9Cq=e<5O&o9pG5qySvo|OOYaQsON#GgKYg*{t|Kl1wYbKFmW{HLs5l-?q0 zvnvJt2UdJ}{AJh`Hze1JO-02*_c!Q$se3NVF9Yv2OBwP{{pNS@!%ufxXPY?I-tMRy zNSy$Ay^+bSVyfp(;P_Qr?9Ye4k_Yp>uZw8+Zo?kwH|GycJ(L7rWF_pS9&xDsR&l2H zUO_plkNq=i9u%2*>yX#cAvVZdo$&lkViJte3)BOOUgQSTK2un!R@Q(N?SX&33$fm< zViJ71`fqWcv|)$;lFVLHj>yx<4Vh%{|9_L`6%NFoIX(Y(r+MuAz?TOs_|xv60bMZ> zF~TRb9B%b2w$v?ll@yA3Np%D33X229nU#Whp!5UiaR>b#`k*VaDy1LlbTIxzS|k2< zbQ!q^%o$ii-2k?#(hXv_!vB|pxSV^hSdj2VN8A;*obR3ZU-eP#Z12;Em#qfgrP3~m z>L?bIF~_fMI4ET?$Nn+ITNM8E&ve)A6;olO`*XIK=6QuD+ov%9ugu#fW`RDCw3V!bSRWlsPIq9Ssg|fF31h)7wQ?z4FdlE1Nr}R z9e?6AC9zrj^APxdYiqk$UhNmNlN!Yl29Pe|sxRZ~kcy2RY zo9BHs4F5$5?}*vnXVC9mQ(w}-qJ*!Bc+V*@#?>jtx?3gQ9CNi1f7)5QT5l1vQkKgc zP2K0F#-1(5pY>b|V2d!nU-!-MI&*xNCa(x&r>zR;^S1sB>T_wStML83f_^`~$35VG zJN!9saDGwu>-cm2Rbx*4rviVg>}5P~FXsOP{x&fSIo`bmrGjx{b)Vjz#T-zZT@gUN zPvDFt-mChNW?290PN`*I@d|ieUDI2mU%ps~{9?{ax3mxD9vSvQtBCT@p1U7-eJu>L zg$ZAQ&VK@WK)1B@6+g2z@Lz7aXRh}P=m(#HdyBLW_5NyqLBgA2vU@+|{z}OG6CwN$ zVxPGeiO9#Z##`06v%RHsKl*fwl++$^y!1c_6TJ`jIY+O}?-#QZIs?twn*w^QK=tLE zu(RkUnfFtPIpHwoAv^p%*?rKdmoZMo`rcwMOE|HQIpbU!rBS{xYSJ|t>Q^BDRGsE@f^Th@}}uN z^&4}%&x1}jBF4T>GxRT>d4<`Fzk~ zyD4ngQmKSYqAIM1U4Y-q61}H=HYij=# zXe=Xbm9#^c3#4cu5jue`Yjpf|TaCWp>A?S8*zvUaX_JAMr5>kn)Z5p9X&H1i7kV!}lTo z3i2B^{6>kd!bxup9dNtM*D8jc$hMtXKIDLvi?2D%@!!Op0ev28l+*=Sv#ZAt6qc0j z+hEtTu8a0OeFUeLwZRwOBK7;18|>$N-p@RQ=G=0z1p2|se8k*QJ2lmFhpjzl*y<-S z$FCHg;?p9bLMYc0dlphY0$yvjs^rD{{95U^fK1P#37zk*epj zPkZhbS!c#v0bOR0KB51A0QvtPdcHU1Gwb+XsJGY4d7XOOV%YM_@=G1*gS!8clzN#n z{58lDrLVK4Zn+<}(`aAPVr$auF=BNYkH(v$sm%>jl$-UOm zo@4B0qI)xRv>s7d_qvJKhmZg3VfNop{7?LqOc;)CV92H0prdw))!<|5^KuTy{7gHM zwgdC?bj)qZ_TyP$nRe?m9x${0_3(4GsQ zXUT8W$tNZDiW>`VlRBQlM719W{_3|ftDY5OJspris~mL$`W~X(JEi_N1AhE1&1RpC z9p*g8{N1vWt?(tR5Zir6#O@ZuuTYq=##Toprkk<84DEzk=%QRZU`YJkEx!f+KT&+A zW6pi*0>^;&4ZwXB@ORS5>Cd68uR(nXYr=@@|AGJLWM@0*+%M?2|1#_dg|}17I6`3| z{CVzOVKS9|nW>(;po_1RIyw0-zxI_sI{rD;FNj%LH;IC}*EHSWg|PRhuW=r7aG1C0 zgtt1+`M(k2%PC>U?`z;4^})-c#EqzO~itSGLl^<9JSt1%L89 z`Dr3#~C<{hnu6 zKPc);wxM3IL+Xf}TdWvU@61i_3oK4v9@t;9&vaelZ}9(*8S_`XPkp}=ei-U^cS6sz zv)Ab(+5uloG~zTjqE_O3m=fdnwlRj(hIm0M;)HF|7O-N^<0pEF|4tMC(9aT|wk?Fa zlD&Uw_^bD#Y>G}^4gcbxe}el7+4lG|mcJfzJo&w@40;0AdeE`9+dsSMF|nuh7<8xQ zVqxk^F%N#^-NiSVFjn*c>;pF@Ee~M5!H^DU@P8}$|KEu}?YD!l<7jtpfd56e2^DRy zrL12JSznF(sORC6@2qM=zF4dD3;%D6&(IFvcQ!$8RrnzbTg8cvHaSn|{hkE=(RTQ! zUq1Mg4&qWbn((JynFt@8jwO9`yIPK7{8qyL+XDOVUek5GuiB@+cuU7^@bRrep5Hpe z`|cKR;yEW;Pl&~Mj+2f^`2br|+KcoPa*mkfZvBw>)6U}B3|PA_EN4T}O3BL4jr{(s@J;(r5w*ln2e zmCdT-PYjO$!y7YJA-1(%oL&hGaqfH@V?4<@YsJ#6K5?w`_OSkMtUDv>3%4RZ)#8|A zz&JD3Aqvcey@)mBJ{;?A{?~Z;d+5Klll40OtPSf#UEnO(?aT$U@;l{0F7o!M`#+*# zsxY^<%rSVZryYFZ$9hPHt?)UC|BleOas1|bUx044^%BVcH(>vDS$@me|D)S~O4cj= zPaUf^HpPnnOENA)43mJN3mBg0Iyo%+?}6@nL=JKY=co6Gz5e5&93Y)uXe*QyZx*xQ zOM(qxTl1;rgS*O)!1o|?U%!g{*SyiLwl|ZoUdfr7@Z207f3^Q=Fh0L1y+z6bJy(SL zXMjCvfSA5uvo?g44w!3j5^`ffnjiGt3jVj^|Dx2b)!3@`!sq`>iT`>TUmWEBm!QKU zFAljP`+(tD$ZQ=yyZtBn?ib~-|7ODe+tz#}ET+SIo?rVA>WZ<(2lT*|h#k=9r*O8m z4}tfqGB+ZRq5Y@8yc?TbhyVJeZJ)q?0k&@!Vm-9y>+Svc3V+pd-M;j+8o?T{r&pQn zJ7UVYU?Jv&>JqS=N9z3=vx(4Ln|TLbT;9H9*Y!jYXXd<0n@9r=%G%Sta|~UL5(~ zY5A>-GyhLcSt}S{R^KrPI?Pz)_A##WdB|)XSMu_jh6B?6YbfuB{r8~hy6Q`vk9mAa z$Q#jP`BvO%+w~Xig-y6J034seHro#8^PR(eqp%%_^R|4jwYb#LCe-PGw&dEv66w?5 z0zCEFZsPs-TROyw8iNk#7vg#{%<1A<6F$WI?ykxM|8EVAn<4*+$)-!9|1pEA6a-Wrxqrf|Q%`xNYib%-Y} z7t9GHzE;fD@e#xUJSnRXFKBxkThbV}KkRR;oDX;X`h*)gBJiAOY^P@>`u=dRtO&k> zq2mRl2j=^>0#C;HjBgcOY_+Q8Wd6A6x4RNB(=K4GnkiSXn?*qOHYrTCE`dea-eW;>gN!BXcIPv;_`fXuDLNj*mH#CTUy}a$amd}Z=8YKmcbI66 zc^nfVf0zT1SM!As4SXd06W$0tn>k4-LHreFhmpgxRK|cYk83^v&ih>7kv@K|`D+*3 zSI>9=HT{*)>BqU-e++DvW4oklKwr1Cnere7+uP}B_2Ti`;n|E__lIjqC81t`IYnC$ zD^$5XI?X&Q@!uR8H}WQZr}U?d1b^o1Fz;`iyIV~2ATJ;BBc0E=PyQU^GUIRbc~{a3 z^Sv3{DQ$d9=6{St{?q@f=PVHaRLt87H`Ou+WD9aYnA1;QUk#m}b-|3^(O0ATSzG$= z{Dw}CwH@1bVzbrh74Kf`cC@?g&+wb&kT*Xl%dAC=$Ua|0$CmL#)_-ws(9f&=*Wp(v z&P2Vu%ZmRiz<zKy_x@O#hu48pWn51fc3vK9df#u?>)_PTE-k1 zdm0h`$Q3Z-|D0Hyw8GIoCy#f4*Nc#+;gl~#8Zi90u=fp}{vJ6+Jg-HtG^aE%p6sR>F#NJ4TTEtN5NeAMcx!xDqtm_Bm`b{*?YU9Ny=zv2Bg= zwElM=u>3kOJsL%WwNjO61@7vZI5CYVu&GG}1nO9@{MSlE8oN1lPu4;h$OgjMV8TAP3*5*)i(4 z6<%yv|HnGOv96T^sQ(#Qh`LVwy1q}EU@rGKS1;;24w&S(-p|PU-zBgWdkGOXu>Z~88-=ZtcWGiZcZf+|Bgz`T z5_&(rXH-(J1^;Kr|DV$_!1w08lFXY$1yG|QHOUA zc-t()6*lBGJR>m4wQXRo_az(b3^_qu$?ppj-m>kp0Ow=fMg}<&gYZLNvJMcGv+qOea{Iy;G%kA7} z=Pe^0WMb>bHss$W0sb@X@Mn%C^ZVl*`O&E7i*oORZntIt^LaqggS9mZy}!!O!M3qaFPphPLMLYPtG5lM<}bA zQ#e2OFysn-1*dI&1^hkcAxuJS67(tUNBtetK4U)7?mf`e;NQ~Z$}9u_$FFq!F7*3= z|Fj9Nbte4P*jihIkME{#7eI|~z{%F1@ww9P2mbNn&x?Oh_j0WY#JKkbtYfX? zp!hL5X|>3xd>Xi${rWoo%-i%7-;4F&R!aZxc+{U5KI8B@1k%QI59*6us1bJG3|nE) z*Q@`&+9%Bb|3|NM{4VtSfd3TO;g0+-`Pj87FwwOoKzXdj*V;hLE0J$-geHrhe4C)qf?&FTSWDx%|7dk#y`u)H^ z&JKUt0IUNX?^+$;`dm)3*|7JGI+4W%w}21NB;Q)rWDw%&KS5 zhw^^A)B&ut#$)P#aUS>xrLAzc^gHPN=wAZ=KYXR*ccI@0{3kRs6~ zrpWn#^8)7wMW;dhAG*--xzg_k{xRV3bB1nrTH+q(J_g;+&Ie=I@s!QRdJo&fYC)|y zmNvWvf89RRXRJN1KA`SZHNR2NA?6?sLq@GCKZ>IMN^8bTZ zI(`@WeZW5&YjO>M1}N_XdJcuoXWSac-jrAN6k?;!*rPtT^zA2U#-X$iYPMp4a;-=&4w9lGyVji2aNstEY`tdyd zlB`2_@*z?VkVfIt`8K|*;Y!EvVtgOQim}8^V_aQ7qfVv!(X4b}$o`E%yZ+@6PSp1k zOCJL6z0zk~+5B$knjMzv9R2u|@zF_Zu+GC1(l$`nRLfdlt}Qek>&;QFt2thOH^XPR zpZO*4<6C@A{_cx0URUZfFmBgl+lMup{)cP7IQeHe{&LM;=6Gf97L@PAZw_*Rl#fqg z=hSk2up9ERCOYH<*N@R-{OX>;Tr$@BKWvM;VRs$_?p?@bXj_B)h#~*x>XnM!#r!Ue zBmM??+ez2}zjMmvBJa`fI}Pgq(gvG?b$ZI1hPDBSE9V0j=5gkYEGf7X_To3gmn=N)MA~EwfwYQu-HYwB7jq1*JqvMjSKD-Kk0O`zoWXzLI2d$5EDRq2 zd78Yfcv!WZAGl5k*Z5nIdkgaW-U;i&IYY)95!>VQ7#G%MyqV^DRgFc|HLNb(2n6F~ zH-ZjcqRmIU+KP!fRz4ZV4c#H#@4L_!pFzIRIx#DAub@m>W{*Q&H*1OQVnDSm?+Gvb z{}Vtjv@P`by`m|$)H$j1+l zBLl<1+B=1xYK{E8t+J+ozC6-{K|2nyT$%r|AN($Dxn}&1ZSK+)J;0aA#{rwqVCzF& z-%q(7tCI|;&B&M>X@xOGMH_lczeAiyx8>UY@-i8QtNt}%cV7oO*a=_qulRRdhRO}m zG3UO@|BQ64_zj-158E|c?lPq)U^f=q@krOo-!e~gsJ{m@=s)ODw%oAlKNzhq7P{^d_!kiorU#ylh%h~{AH%Kn$Ky;7#N$j4#N#3&K0SEDSsrT-?C0^|1Lfl)B9he& z@^RV&<>RzByjaISHh%Ce^}IKY;|O(ZjM=$}1@d(x{@QrM4%lRNQNJG?(;e#A z7*kDxzt0cWY><3Vi_5J|GbSSLz%lq2fu;+&{ I_#glMe~8&m(f|Me literal 0 HcmV?d00001 From 2097cdb86a5063f065067fc5e0baba17b84d95e8 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 1 Jul 2025 20:56:17 +0200 Subject: [PATCH 084/154] collect paths for input packages that are in a workspace (#4229) Co-authored-by: KristofferC --- src/API.jl | 85 +++++++++++-------- .../WorkspacePathResolution/Project.toml | 5 ++ .../SubProjectA/Project.toml | 9 ++ .../SubProjectA/src/SubProjectA.jl | 7 ++ .../SubProjectB/Project.toml | 3 + .../SubProjectB/src/SubProjectB.jl | 5 ++ test/workspaces.jl | 15 ++++ 7 files changed, 92 insertions(+), 37 deletions(-) create mode 100644 test/test_packages/WorkspacePathResolution/Project.toml create mode 100644 test/test_packages/WorkspacePathResolution/SubProjectA/Project.toml create mode 100644 test/test_packages/WorkspacePathResolution/SubProjectA/src/SubProjectA.jl create mode 100644 test/test_packages/WorkspacePathResolution/SubProjectB/Project.toml create mode 100644 test/test_packages/WorkspacePathResolution/SubProjectB/src/SubProjectB.jl diff --git a/src/API.jl b/src/API.jl index 103e7c99f3..d11ce3d453 100644 --- a/src/API.jl +++ b/src/API.jl @@ -187,46 +187,57 @@ for f in (:develop, :add, :rm, :up, :pin, :free, :test, :build, :status, :why, : end end -function update_source_if_set(project, pkg) +function update_source_if_set(env, pkg) + project = env.project source = get(project.sources, pkg.name, nothing) - source === nothing && return - if pkg.repo == GitRepo() - delete!(project.sources, pkg.name) - else - # This should probably not modify the dicts directly... - if pkg.repo.source !== nothing - source["url"] = pkg.repo.source - delete!(source, "path") + if source !== nothing + if pkg.repo == GitRepo() + delete!(project.sources, pkg.name) + else + # This should probably not modify the dicts directly... + if pkg.repo.source !== nothing + source["url"] = pkg.repo.source + delete!(source, "path") + end + if pkg.repo.rev !== nothing + source["rev"] = pkg.repo.rev + delete!(source, "path") + end + if pkg.repo.subdir !== nothing + source["subdir"] = pkg.repo.subdir + end + if pkg.path !== nothing + source["path"] = pkg.path + delete!(source, "url") + delete!(source, "rev") + end end - if pkg.repo.rev !== nothing - source["rev"] = pkg.repo.rev - delete!(source, "path") + if pkg.subdir !== nothing + source["subdir"] = pkg.subdir end - if pkg.repo.subdir !== nothing - source["subdir"] = pkg.repo.subdir + path, repo = get_path_repo(project, pkg.name) + if path !== nothing + pkg.path = path end - if pkg.path !== nothing - source["path"] = pkg.path - delete!(source, "url") - delete!(source, "rev") + if repo.source !== nothing + pkg.repo.source = repo.source + end + if repo.rev !== nothing + pkg.repo.rev = repo.rev + end + if repo.subdir !== nothing + pkg.repo.subdir = repo.subdir end end - if pkg.subdir !== nothing - source["subdir"] = pkg.subdir - end - path, repo = get_path_repo(project, pkg.name) - if path !== nothing - pkg.path = path - end - if repo.source !== nothing - pkg.repo.source = repo.source - end - if repo.rev !== nothing - pkg.repo.rev = repo.rev - end - if repo.subdir !== nothing - pkg.repo.subdir = repo.subdir + + # Packages in manifest should have their paths set to the path in the manifest + for (path, wproj) in env.workspace + if wproj.uuid == pkg.uuid + pkg.path = Types.relative_project_path(env.manifest_file, dirname(path)) + break + end end + return end function develop(ctx::Context, pkgs::Vector{PackageSpec}; shared::Bool=true, @@ -268,7 +279,7 @@ function develop(ctx::Context, pkgs::Vector{PackageSpec}; shared::Bool=true, if length(findall(x -> x.uuid == pkg.uuid, pkgs)) > 1 pkgerror("it is invalid to specify multiple packages with the same UUID: $(err_rep(pkg))") end - update_source_if_set(ctx.env.project, pkg) + update_source_if_set(ctx.env, pkg) end Operations.develop(ctx, pkgs, new_git; preserve=preserve, platform=platform) @@ -322,7 +333,7 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}; preserve::PreserveLevel=Op if length(findall(x -> x.uuid == pkg.uuid, pkgs)) > 1 pkgerror("it is invalid to specify multiple packages with the same UUID: $(err_rep(pkg))") end - update_source_if_set(ctx.env.project, pkg) + update_source_if_set(ctx.env, pkg) end Operations.add(ctx, pkgs, new_git; allow_autoprecomp, preserve, platform, target) @@ -401,7 +412,7 @@ function up(ctx::Context, pkgs::Vector{PackageSpec}; ensure_resolved(ctx, ctx.env.manifest, pkgs) end for pkg in pkgs - update_source_if_set(ctx.env.project, pkg) + update_source_if_set(ctx.env, pkg) end Operations.up(ctx, pkgs, level; skip_writing_project, preserve) return @@ -440,7 +451,7 @@ function pin(ctx::Context, pkgs::Vector{PackageSpec}; all_pkgs::Bool=false, kwar pkgerror("pinning a package requires a single version, not a versionrange") end end - update_source_if_set(ctx.env.project, pkg) + update_source_if_set(ctx.env, pkg) end project_deps_resolve!(ctx.env, pkgs) diff --git a/test/test_packages/WorkspacePathResolution/Project.toml b/test/test_packages/WorkspacePathResolution/Project.toml new file mode 100644 index 0000000000..793ff0e389 --- /dev/null +++ b/test/test_packages/WorkspacePathResolution/Project.toml @@ -0,0 +1,5 @@ +[workspace] +projects = [ + "SubProjectA", + "SubProjectB", +] \ No newline at end of file diff --git a/test/test_packages/WorkspacePathResolution/SubProjectA/Project.toml b/test/test_packages/WorkspacePathResolution/SubProjectA/Project.toml new file mode 100644 index 0000000000..e5aa2bbe50 --- /dev/null +++ b/test/test_packages/WorkspacePathResolution/SubProjectA/Project.toml @@ -0,0 +1,9 @@ +name = "SubProjectA" +uuid = "87654321-4321-4321-4321-210987654321" +version = "0.1.0" + +[deps] +SubProjectB = "12345678-1234-1234-1234-123456789012" + +[sources] +SubProjectB = {path = "SubProjectB"} diff --git a/test/test_packages/WorkspacePathResolution/SubProjectA/src/SubProjectA.jl b/test/test_packages/WorkspacePathResolution/SubProjectA/src/SubProjectA.jl new file mode 100644 index 0000000000..06b4efa7b4 --- /dev/null +++ b/test/test_packages/WorkspacePathResolution/SubProjectA/src/SubProjectA.jl @@ -0,0 +1,7 @@ +module SubProjectA + +using SubProjectB + +greet() = "Hello from SubProjectA! " * SubProjectB.greet() + +end \ No newline at end of file diff --git a/test/test_packages/WorkspacePathResolution/SubProjectB/Project.toml b/test/test_packages/WorkspacePathResolution/SubProjectB/Project.toml new file mode 100644 index 0000000000..f989c45989 --- /dev/null +++ b/test/test_packages/WorkspacePathResolution/SubProjectB/Project.toml @@ -0,0 +1,3 @@ +name = "SubProjectB" +uuid = "12345678-1234-1234-1234-123456789012" +version = "0.1.0" \ No newline at end of file diff --git a/test/test_packages/WorkspacePathResolution/SubProjectB/src/SubProjectB.jl b/test/test_packages/WorkspacePathResolution/SubProjectB/src/SubProjectB.jl new file mode 100644 index 0000000000..342bdf6c94 --- /dev/null +++ b/test/test_packages/WorkspacePathResolution/SubProjectB/src/SubProjectB.jl @@ -0,0 +1,5 @@ +module SubProjectB + +greet() = "Hello from SubProjectB!" + +end \ No newline at end of file diff --git a/test/workspaces.jl b/test/workspaces.jl index 33b75d2baa..ba848246f0 100644 --- a/test/workspaces.jl +++ b/test/workspaces.jl @@ -163,4 +163,19 @@ end end end +@testset "workspace path resolution issue #4222" begin + mktempdir() do dir + path = copy_test_package(dir, "WorkspacePathResolution") + cd(path) do + with_current_env() do + # First resolve SubProjectB (non-root project) without existing Manifest + Pkg.activate("SubProjectB") + @test !isfile("Manifest.toml") + # Should be able to find SubProjectA and succeed + Pkg.update() + end + end + end +end + end # module From fe73f6a71a2ad434a81fc6e3f1f0c589958ef6de Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 1 Jul 2025 20:57:28 +0200 Subject: [PATCH 085/154] Add readonly field support for Project.toml environments (#4284) Co-authored-by: KristofferC Co-authored-by: Claude --- docs/src/toml-files.md | 12 ++++++++++++ src/Types.jl | 6 ++++++ src/manifest.jl | 3 +++ src/project.jl | 14 +++++++++++++- test/new.jl | 27 +++++++++++++++++++++++++++ 5 files changed, 61 insertions(+), 1 deletion(-) diff --git a/docs/src/toml-files.md b/docs/src/toml-files.md index 0249970a02..0d73d33ffa 100644 --- a/docs/src/toml-files.md +++ b/docs/src/toml-files.md @@ -102,6 +102,18 @@ Note that Pkg.jl deviates from the SemVer specification when it comes to version the section on [pre-1.0 behavior](@ref compat-pre-1.0) for more details. +### The `readonly` field + +The `readonly` field is a boolean that, when set to `true`, marks the environment as read-only. This prevents any modifications to the environment, including adding, removing, or updating packages. For example: + +```toml +readonly = true +``` + +When an environment is marked as readonly, Pkg will throw an error if any operation that would modify the environment is attempted. +If the `readonly` field is not present or set to `false` (the default), the environment can be modified normally. + + ### The `[deps]` section All dependencies of the package/project are listed in the `[deps]` section. Each dependency diff --git a/src/Types.jl b/src/Types.jl index 18224c02c9..a3a939e575 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -266,6 +266,7 @@ Base.@kwdef mutable struct Project compat::Dict{String,Compat} = Dict{String,Compat}() sources::Dict{String,Dict{String, String}} = Dict{String,Dict{String, String}}() workspace::Dict{String, Any} = Dict{String, Any}() + readonly::Bool = false end Base.:(==)(t1::Project, t2::Project) = all(x -> (getfield(t1, x) == getfield(t2, x))::Bool, fieldnames(Project)) Base.hash(t::Project, h::UInt) = foldr(hash, [getfield(t, x) for x in fieldnames(Project)], init=h) @@ -1245,6 +1246,11 @@ function write_env(env::EnvCache; update_undo=true, end end + # Check if the environment is readonly before attempting to write + if env.project.readonly + pkgerror("Cannot modify a readonly environment. The project at $(env.project_file) is marked as readonly.") + end + if (env.project != env.original_project) && (!skip_writing_project) write_project(env) end diff --git a/src/manifest.jl b/src/manifest.jl index 1101883ac3..8b9714321c 100644 --- a/src/manifest.jl +++ b/src/manifest.jl @@ -361,6 +361,9 @@ function destructure(manifest::Manifest)::Dict end function write_manifest(env::EnvCache) + if env.project.readonly + pkgerror("Cannot write to readonly manifest file at $(env.manifest_file)") + end mkpath(dirname(env.manifest_file)) write_manifest(env.manifest, env.manifest_file) end diff --git a/src/project.jl b/src/project.jl index bb5aeb9b9b..55b47f0922 100644 --- a/src/project.jl +++ b/src/project.jl @@ -213,6 +213,7 @@ function Project(raw::Dict; file=nothing) project.targets = read_project_targets(get(raw, "targets", nothing), project) project.workspace = read_project_workspace(get(raw, "workspace", nothing), project) project.apps = read_project_apps(get(raw, "apps", nothing), project) + project.readonly = get(raw, "readonly", false)::Bool # Handle deps in both [deps] and [weakdeps] project._deps_weak = Dict(intersect(project.deps, project.weakdeps)) @@ -270,14 +271,25 @@ function destructure(project::Project)::Dict entry!("extras", project.extras) entry!("compat", Dict(name => x.str for (name, x) in project.compat)) entry!("targets", project.targets) + + # Only write readonly if it's true (not the default false) + if project.readonly + raw["readonly"] = true + else + delete!(raw, "readonly") + end + return raw end -const _project_key_order = ["name", "uuid", "keywords", "license", "desc", "version", "workspace", "deps", "weakdeps", "sources", "extensions", "compat"] +const _project_key_order = ["name", "uuid", "keywords", "license", "desc", "version", "readonly", "workspace", "deps", "weakdeps", "sources", "extensions", "compat"] project_key_order(key::String) = something(findfirst(x -> x == key, _project_key_order), length(_project_key_order) + 1) function write_project(env::EnvCache) + if env.project.readonly + pkgerror("Cannot write to readonly project file at $(env.project_file)") + end write_project(env.project, env.project_file) end write_project(project::Project, project_file::AbstractString) = diff --git a/test/new.jl b/test/new.jl index acf4ba34cc..0a62a8c916 100644 --- a/test/new.jl +++ b/test/new.jl @@ -3292,4 +3292,31 @@ end @test allunique(unique([Pkg.PackageSpec(path="foo"), Pkg.PackageSpec(path="foo")])) +# Test the readonly functionality +@testset "Readonly Environment Tests" begin + mktempdir() do dir + project_file = joinpath(dir, "Project.toml") + + # Test that normal environment works + cd(dir) do + # Activate the environment + Pkg.activate(".") + + # This should work fine + Pkg.add("Test") # Add Test package + + # Now make it readonly + project_data = Dict("readonly" => true) + open(project_file, "w") do io + TOML.print(io, project_data) + end + + # Now these should fail + @test_throws Pkg.Types.PkgError Pkg.add("Dates") + @test_throws Pkg.Types.PkgError Pkg.rm("Test") + @test_throws Pkg.Types.PkgError Pkg.update() + end + end +end + end #module From 4fc3086c8960fe5054a39980f3de9bfa1b0d43e4 Mon Sep 17 00:00:00 2001 From: Tanmay Mohapatra Date: Wed, 2 Jul 2025 00:55:34 +0530 Subject: [PATCH 086/154] Doc section for Pkg and Storage Protocols (#4234) Co-authored-by: Morten Piibeleht Co-authored-by: Kristoffer Carlsson --- docs/make.jl | 1 + docs/src/api.md | 7 ++ docs/src/protocol.md | 190 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 198 insertions(+) create mode 100644 docs/src/protocol.md diff --git a/docs/make.jl b/docs/make.jl index 8b3cb2c8b1..00794f72e8 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -43,6 +43,7 @@ makedocs( "toml-files.md", "repl.md", "api.md", + "protocol.md", ], ) diff --git a/docs/src/api.md b/docs/src/api.md index dc9e9e1794..afe34d8635 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -79,3 +79,10 @@ Pkg.Artifacts.ensure_artifact_installed Pkg.Artifacts.ensure_all_artifacts_installed Pkg.Artifacts.archive_artifact ``` + +## [Package Server Authentication Hooks](@id Package-Server-Authentication-Hooks) + +```@docs +Pkg.PlatformEngines.register_auth_error_handler +Pkg.PlatformEngines.deregister_auth_error_handler +``` diff --git a/docs/src/protocol.md b/docs/src/protocol.md new file mode 100644 index 0000000000..15a6cbee1b --- /dev/null +++ b/docs/src/protocol.md @@ -0,0 +1,190 @@ +# [**14.** Package and Storage Server Protocol Reference](@id Pkg-Server-Protocols) + +The Julia Package Server Protocol (Pkg Protocol) and the Package Storage Server Protocol (Storage Protocol) define how Julia's package manager, Pkg, obtains and manages packages and their associated resources. They aim to enhance the Julia package ecosystem, making it more efficient, reliable, and user-friendly, avoiding potential points of failure, and ensuring the permanent availability of package versions and artifacts, which is paramount for the stability and reproducibility of Julia projects. + +The Pkg client, by default, gets all resources over HTTPS from a single open source service run by the Julia community. This service for serving packages is additionally backed by multiple independent storage services which interface with proprietary origin services (GitHub, etc.) and guarantee persistent availability of resources into the future. + +The protocols also aim to address some of the limitations that existed prior to its introduction. + +- **Vanishing Resources.** It is possible for authors to delete code repositories of registered Julia packages. Without some kind of package server, no one can install a package which has been deleted. If someone happens to have a current fork of a deleted package, that can be made the new official repository for the package, but the chances of them having no or outdated forks are high. An even worse situation could happen for artifacts since they tend not to be kept in version control and are much more likely to be served from "random" web servers at a fixed URL with content changing over time. Artifact publishers are unlikely to retain all past versions of artifacts, so old versions of packages that depend on specific artifact content will not be reproducible in the future unless we do something to ensure that they are kept around after the publisher has stopped hosting them. By storing all package versions and artifacts in a single place, we can ensure that they are available forever. +- **Usage Insights.** It is valuable for the Julia community to know how many people are using Julia or what the relative popularity of different packages and operating systems is. Julia uses GitHub to host its ecosystem. GitHub - a commercial, proprietary service - has this information but does not make it available to the Julia community. We are of course using GitHub for free, so we can't complain, but it seems unfortunate that a commercial entity has this valuable information while the open source community remains in the dark. The Julia community really could use insight into who is using Julia and how, so that we can prioritize packages and platforms, and give real numbers when people ask "how many people are using Julia?" +- **Decoupling from Git and GitHub.** Prior to this, Julia package ecosystem was very deeply coupled to git and was even specialized on GitHub specifically in many ways. The Pkg and Storage Protocols allowed us to decouple ourselves from git as the primary mechanism for getting packages. Now Julia continues to support using git, but does not require it just to install packages from the default public registry anymore. This decoupling also paves the way for supporting other version control systems in the future, making git no longer so special. Special treatment of GitHub will also go away since we get the benefits of specializing for GitHub (fast tarball downloads) directly from the Pkg protocols. +- **Firewall problems.** Prior to this, Pkg's need to connect to arbitrary servers using a miscellany of protocols caused several problems with firewalls. A large set of protocols and an unbounded list of servers needed to be whitelisted just to support default Pkg operation. If Pkg only needed to talk to a single service over a single, secure protocol (i.e. HTTPS), then whitelisting Pkg for standard use would be dead simple. + +## Protocols & Services + +1. **Pkg Protocol:** what Julia Pkg Clients speak to Pkg Servers. The Pkg Server serves all resources that Pkg Clients need to install and use registered packages, including registry data, packages and artifacts. It is designed to be easily horizontally scalable and not to have any hard operational requirements: if service is slow, just start more servers; if a Pkg Server crashes, forget it and boot up a new one. +2. **Storage Protocol:** what Pkg Servers speak to get resources from Storage Services. Julia clients do not interact with Storage services directly and multiple independent Storage Services can symmetrically (all are treated equally) provide their service to a given Pkg Server. Since Pkg Servers cache what they serve to Clients and handle convenient content presentation, Storage Services can expose a much simpler protocol: all they do is serve up complete versions of registries, packages and artifacts, while guaranteeing persistence and completeness. Persistence means: once a version of a resource has been served, that version can be served forever. Completeness means: if the service serves a registry, it can serve all package versions referenced by that registry; if it serves a package version, it can serve all artifacts used by that package. + +Both protocols work over HTTPS, using only GET and HEAD requests. As is normal for HTTP, HEAD requests are used to get information about a resource, including whether it would be served, without actually downloading it. As described in what follows, the Pkg Protocol is client-to-server and may be unauthenticated, use basic auth, or OpenID; the Storage Protocol is server-to-server only and uses mutual authentication with TLS certificates. + +The following diagram shows how these services interact with each other and with external services such as GitHub, GitLab and BitBucket for source control, and S3 and HDFS for long-term persistence: + + ┌───────────┐ + + │ Amazon S3 │ + + │ Storage │ + + └───────────┘ + + ▲ + + ║ + + ▼ + + Storage ╔═══════════╗ ┌───────────┐ + + Pkg Protocol ║ Storage ║ ┌──▶│ GitHub │ + + Protocol ┌──▶║ Service A ║───┤ └───────────┘ + + ┏━━━━━━━━━━━━┓ ┏━━━━━━━━━━━━┓ │ ╚═══════════╝ │ ┌───────────┐ + + ┃ Pkg Client ┃────▶┃ Pkg Server ┃───┤ ╔═══════════╗ ├──▶│ GitLab │ + + ┗━━━━━━━━━━━━┛ ┗━━━━━━━━━━━━┛ │ ║ Storage ║ │ └───────────┘ + + └──▶║ Service B ║───┤ ┌───────────┐ + + ╚═══════════╝ └──▶│ BitBucket │ + + ▲ └───────────┘ + + ║ + + ▼ + + ┌───────────┐ + + │ HDFS │ + + │ Cluster │ + + └───────────┘ + +Each Julia Pkg Client is configured to talk to a Pkg Server. By default, they talk to `pkg.julialang.org`, a public, unauthenticated Pkg Server. If the environment variable `JULIA_PKG_SERVER` is set, the Pkg Client connects to that host instead. For example, if `JULIA_PKG_SERVER` is set to `pkg.company.com` then the Pkg Client will connect to `https://pkg.company.com`. So in typical operation, a Pkg Client will no longer rely on `libgit2` or a git command-line client, both of which have been an ongoing headache, especially behind firewalls and on Windows. If fact, git will only be necessary when working with git-hosted registries and unregistered packages - those will continue to work as they have previously, fetched using git. + +While the default Pkg Server at `pkg.julialang.org` is unauthenticated, other parties may host Pkg Server instances elsewhere, authenticated or unauthenticated, public or private, as they wish. People can connect to those servers by setting the `JULIA_PKG_SERVER` variable. There will be a configuration file for providing authentication information to Pkg Servers using either basic auth or OpenID. The Pkg Server implementation will be open source and have minimal operational requirements. Specifically, it needs: + +1. The ability to accept incoming connections on port 443; +2. The ability to connect to a configurable set of Storage Services; +3. Temporary disk storage for caching resources (registries, packages, artifacts). + +A Pkg Service may be backed by more than one actual server, as is typical for web services. The Pkg Service is stateless, so this kind of horizontal scaling is straightforward. Each Pkg Server serves registry, package and artifact resources to Pkg Clients and caches whatever it serves. Each Pkg Server, in turn, gets those resources from one or more Storage Services. Storage services are responsible for fetching resources from code hosting sites like GitHub, GitLab and BitBucket, and for persisting everything that they have ever served to long-term storage systems like Amazon S3, hosted HDFS clusters - or whatever an implementor wants to use. If the original copies of resources vanish, Pkg Servers must always serve up all previously served versions of resources. + +The Storage Protocol is designed to be extremely simple so that multiple independent implementations can coexist, and each Pkg Server may be symmetrically backed by multiple different Storage Services, providing both redundant backup and ensuring that no single implementation has a "choke hold" on the ecosystem - anyone can implement a new Storage Service and add it to the set of services backing the default Pkg Server at `pkg.julialang.org`. The simplest possible version of a Storage Service is a static HTTPS site serving files generated from a snapshot of a registry. Although this does not provide adequate long-term backup capabilities, and would need to be regenerated whenever a registry changes, it may be sufficient for some private uses. Having multiple independently operated Storage Services helps ensure that even if one Storage Service becomes unavailable or unreliable - for technical, financial, or political reasons - others will keep operating and so will the Pkg ecosystem. + +## The Pkg Protocol + +This section describes the protocol used by Pkg Clients to get resources from Pkg Servers, including the latest versions of registries, package source trees, and artifacts. There is also a standard system for asking for diffs of all of these from previous versions, to minimize how much data the client needs to download in order to update itself. There is additionally a bundle mechanism for requesting and receiving a set of resources in a single request. + +### Authentication + +The authentication scheme between a Pkg client and server will be HTTP authorization with bearer tokens, as standardized in RFC6750. This means that authenticated access is accomplished by the client by making an HTTPS request including a `Authorization: Bearer $access_token` header. + +The format of the token, its contents and validation mechanism are not specified by the Pkg Protocol. They are left to the server to define. The server is expected to validate the token and determine whether the client is authorized to access the requested resource. Similarly at the client side, the implementation of the token acquisition is not specified by the Pkg Protocol. However Pkg provides [hooks](#Authentication-Hooks) that can be implemented at the client side to trigger the token acquisition process. Tokens thus acquired are expected to be stored in a local file, the format of which is specified by the Pkg Protocol. Pkg will be able to read the token from this file and include it in the request to the server. Pkg can also, optionally, detect when the token is about to expire and trigger a refresh. The Pkg client also supports automatic token refresh, since bearer tokens are recommended to be short-lived (no more than a day). + +The authorization information is saved locally in `$(DEPOT_PATH[1])/servers/$server/auth.toml` which is a TOML file with the following fields: + +- `access_token` (REQUIRED): the bearer token used to authorize normal requests +- `expires_at` (OPTIONAL): an absolute expiration time +- `expires_in` (OPTIONAL): a relative expiration time +- `refresh_token` (OPTIONAL): bearer token used to authorize refresh requests +- `refresh_url` (OPTIONAL): URL to fetch a new token from + +The `auth.toml` file may contain other fields (e.g. user name, user email), but they are ignored by Pkg. The two other fields mentioned in RFC6750 are `token_type` and `scope`: these are omitted since only tokens of type `Bearer` are supported currently and the scope is always implicitly to provide access to Pkg protocol URLs. Pkg servers should, however, not send `auth.toml` files with `token_type` or `scope` fields, as these names may be used in the future, e.g. to support other kinds of tokens or to limit the scope of an authorization to a subset of Pkg protocol URLs. + +Initially, the user or user agent (IDE) must acquire a `auth.toml` file and save it to the correct location. After that, Pkg will determine whether the access token needs to be refreshed by examining the `expires_at` and/or `expires_in` fields of the auth file. The expiration time is the minimum of `expires_at` and `mtime(auth_file) + expires_in`. When the Pkg client downloads a new `auth.toml` file, if there is a relative `expires_in` field, an absolute `expires_at` value is computed based on the client's current clock time. This combination of policies allows expiration to work gracefully even in the presence of clock skew between the server and the client. + +If the access token is expired and there are `refresh_token` and `refresh_url` fields in `auth.toml`, a new auth file is requested by making a request to `refresh_url` with an `Authorization: Bearer $refresh_token` header. Pkg will refuse to make a refresh request unless `refresh_url` is an HTTPS URL. Note that `refresh_url` need not be a URL on the Pkg server: token refresh can be handled by a separate server. If the request is successful and the returned `auth.toml` file is a well-formed TOML file with at least an `access_token` field, it is saved to `$(DEPOT_PATH[1])/servers/$server/auth.toml`. + +Checking for access token expiry and refreshing `auth.toml` is done before each Pkg client request to a Pkg server, and if the auth file is updated the new access token is used, so the token should in theory always be up to date. Practice is different from theory, of course, and if the Pkg server considers the access token expired, it may return an HTTP 401 Unauthorized response, and the Pkg client should attempt to refresh the auth token. If, after attempting to refresh the access token, the server still returns HTTP 401 Unauthorized, the Pkg client will present the body of the error response to the user or user agent (IDE). + +## Authentication Hooks +A mechanism to register a hook at the client is provided to allow the user agent to handle an auth failure. It can, for example, present a login page and take the user through the necessary authentication flow to get a new auth token and store it in `auth.toml`. + +- A handler can also be registered using [`register_auth_error_handler`](@ref Pkg.PlatformEngines.register_auth_error_handler). It returns a function that can be called to deregister the handler. +- A handler can also be deregistered using [`deregister_auth_error_handler`](@ref Pkg.PlatformEngines.deregister_auth_error_handler). + +Example: + +```julia +# register a handler +dispose = Pkg.PlatformEngines.register_auth_error_handler((url, svr, err) -> begin + PkgAuth.authenticate(svr*"/auth") + return true, true +end) + +# ... client code ... + +# deregister the handler +dispose() +# or +Pkg.PlatformEngines.deregister_auth_error_handler(url, svr) +``` + +### Resources + +The client can make GET or HEAD requests to the following resources: + +- `/registries`: map of registry uuids at this server to their current tree hashes, each line of the response data is of the form `/registry/$uuid/$hash` representing a resource pointing to particular version of a registry +- `/registry/$uuid/$hash`: tarball of registry uuid at the given tree hash +- `/package/$uuid/$hash`: tarball of package uuid at the given tree hash +- `/artifact/$hash`: tarball of an artifact with the given tree hash + +Only the `/registries` changes - all other resources can be cached forever and the server will indicate this with the appropriate HTTP headers. + +### Reference Implementation + +A reference implementation of the Pkg Server protocol is available at [PkgServer.jl](https://github.com/JuliaPackaging/PkgServer.jl). + +## The Storage Protocol + +This section describes the protocol used by Pkg Servers to get resources from Storage Servers, including the latest versions of registries, package source trees, and artifacts. The Pkg Server requests each type of resource when it needs it and caches it for as long as it can, so Storage Services should not have to serve the same resources to the same Pkg Server instance many times. + +### Authentication + +Since the Storage protocol is a server-to-server protocol, it uses certificate-based mutual authentication: each side of the connection presents certificates of identity to the other. The operator of a Storage Service must issue a client certificate to the operator of a Pkg Service certifying that it is authorized to use the Storage Service. + +### Resources + +The Storage Protocol is similar to the Pkg Protocol: + +- `/registries`: map of registry uuids at this server to their current tree hashes +- `/registry/$uuid/$hash`: tarball of registry uuid at the given tree hash +- `/package/$uuid/$hash`: tarball of package uuid at the given tree hash +- `/artifact/$hash`: tarball of an artifact with the given tree hash + +As is the case with the Pkg Server protocol, only the `/registries` resource changes over time—all other resources are permanently cacheable and Pkg Servers are expected to cache resources indefinitely, only deleting them if they need to reclaim storage space. + +### Interaction + +Fetching resources from a single Storage Server is straightforward: the Pkg Server asks for a version of a registry by UUID and hash and the Storage Server returns a tarball of that registry tree if it knows about that registry and version, or an HTTP 404 error if it doesn't. + +Each Pkg Server may use multiple Storage Services for availability and depth of backup. For a given resource, the Pkg Server makes a HEAD request to each Storage Service requesting the resource, and then makes a GET request for the resource to the first Storage Server that replies to the HEAD request with a 200 OK. If no Storage Service responds with a 200 OK in enough time, the Pkg Server should respond to the request for the corresponding resource with a 404 error. Each Storage Service which responds with a 200 OK must behave as if it had served the resource, regardless of whether it does so or not - i.e. persist the resource to long-term storage. + +One subtlety is how the Pkg Server determines what the latest version of each registry is. It can get a map from registry UUIDs to version hashes from each Storage Server, but hashes are unordered - if multiple Storage Servers reply with different hashes, which one should the Pkg Server use? When Storage Servers disagree on the latest hash of a registry, the Pkg Server should ask each Storage Server about the hashes that the other servers returned: if Service A knows about Service B's hash but B doesn't know about A's hash, then A's hash is more recent and should be used. If each server doesn't know about the other's hash, then neither hash is strictly newer than the other one and either could be used. The Pkg Server can break the tie any way it wants, e.g. randomly or by using the lexicographically earlier hash. + +### Guarantees + +The primary guarantee that a Storage Server makes is that if it has ever successfully served a resource—registry tree, package source tree, artifact tree — it must be able to serve that same resource version forever. + +It's tempting to also require it to guarantee that if a Storage Server serves a registry tree, it can also serve every package source tree referred to within that registry tree. Similarly, it is tempting to require that if a Storage Server can serve a package source tree that it should be able to serve any artifacts referenced by that version of the package. However, this could fail for reasons entirely beyond the control of the server: what if the registry is published with wrong package hashes? What if someone registers a package version, doesn't git tag it, then force pushes the branch that the version was on? In both of these cases, the Storage Server may not be able to fetch a version of a package through no fault of its own. Similarly, artifact hashes in packages might be incorrect or vanish before the Storage Server can retrieve them. + +Therefore, we don't strictly require that Storage Servers guarantee this kind of closure under resource references. We do, however, recommend that Storage Servers proactively fetch resources referred to by other resources as soon as possible. When a new version of a registry is available, the Storage Server should fetch all the new package versions in the registry immediately. When a package version is fetched—for any reason, whether because it was included in a new registry snapshot or because an upstream Pkg Server requested it by hash—all artifacts that it references should be fetched immediately. + +## Verification + +Since all resources are content addressed, the Pkg Clients and Pkg Server can and should verify that resources that they receive from upstream have the correct content hash. If a resource does not have the right hash, it should not be used and not be served further downstream. Pkg Servers should try to fetch the resource from other Storage Services and serve one that has the correct content. Pkg Clients should error if they get a resource with an incorrect content hash. + +Git uses SHA1 for content hashing. There is a pure Julia implementation of git's content hashing algorithm, which is being used to verify artifacts in Julia 1.3 (among other things). The SHA1 hashing algorithm is considered to be cryptographically compromised at this point, and while it's not completely broken, git is already starting to plan how to move away from using SHA1 hashes. To that end, we should consider getting ahead of this problem by using a stronger hash like SHA3-256 in these protocols. Having control over these protocols actually makes this considerably easier than if we were continuing to rely on git for resource acquisition. + +The first step to using SHA3-256 instead of SHA1 is to populate registries with additional hashes for package versions. Currently each package version is identified by a git-tree-sha1 entry. We would add git-tree-sha3-256 entries that give the SHA3-256 hashes computed using the same git tree hashing logic. From this origin, the Pkg Client, Pkg Server and Storage Servers all just need to use SHA3-256 hashes rather than SHA1 hashes. + +## References + +1. Pkg & Storage Protocols [https://github.com/JuliaLang/Pkg.jl/issues/1377](https://github.com/JuliaLang/Pkg.jl/issues/1377) +2. Authenticated Pkg Client Support: [https://github.com/JuliaLang/Pkg.jl/pull/1538](https://github.com/JuliaLang/Pkg.jl/pull/1538) +3. Authentication Hooks: [https://github.com/JuliaLang/Pkg.jl/pull/1630](https://github.com/JuliaLang/Pkg.jl/pull/1630) From 833aa7faca554e1412435a32e790723ad56f0ab6 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 2 Jul 2025 09:57:31 +0200 Subject: [PATCH 087/154] update CHANGELOG with some new things (#4285) --- CHANGELOG.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5b9688414..5cab492167 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,20 @@ +Pkg v1.13 Release Notes +======================= + +- Project.toml environments now support a `readonly` field to mark environments as read-only, preventing modifications. ([#4284]) +- Pkg now automatically adds entries to `[sources]` when packages are added by URL or devved, improving workflow consistency. ([#4225]) + Pkg v1.12 Release Notes ======================= - Pkg now has support for "workspaces" which is a way to resolve multiple project files into a single manifest. The functions `Pkg.status`, `Pkg.why`, `Pkg.instantiate`, `Pkg.precompile` (and their REPL variants) have been updated - to take a `workspace` option. Read more about this feature in the manual about the TOML-files. + to take a `workspace` option. Read more about this feature in the manual about the TOML-files. ([#3841]) +- Pkg now supports "apps" which are Julia packages that can be run directly from the terminal after installation. + Apps can be defined in a package's Project.toml and installed via Pkg. ([#3772]) - `status` now shows when different versions/sources of dependencies are loaded than that which is expected by the manifest ([#4109]) +- When adding or developing a package that exists in the `[weakdeps]` section, it is now automatically removed from + weak dependencies and added as a regular dependency. ([#3865]) - Packages are also automatically added to `[sources]` when they are added by url or devved. Pkg v1.11 Release Notes @@ -85,6 +95,16 @@ Pkg v1.7 Release Notes - The `mode` keyword for `PackageSpec` has been removed ([#2454]). +[#4225]: https://github.com/JuliaLang/Pkg.jl/issues/4225 +[#4284]: https://github.com/JuliaLang/Pkg.jl/issues/4284 +[#3526]: https://github.com/JuliaLang/Pkg.jl/issues/3526 +[#3708]: https://github.com/JuliaLang/Pkg.jl/issues/3708 +[#3732]: https://github.com/JuliaLang/Pkg.jl/issues/3732 +[#3772]: https://github.com/JuliaLang/Pkg.jl/issues/3772 +[#3783]: https://github.com/JuliaLang/Pkg.jl/issues/3783 +[#3841]: https://github.com/JuliaLang/Pkg.jl/issues/3841 +[#3865]: https://github.com/JuliaLang/Pkg.jl/issues/3865 +[#4109]: https://github.com/JuliaLang/Pkg.jl/issues/4109 [#2284]: https://github.com/JuliaLang/Pkg.jl/issues/2284 [#2431]: https://github.com/JuliaLang/Pkg.jl/issues/2431 [#2432]: https://github.com/JuliaLang/Pkg.jl/issues/2432 From 5b95227597871853c086ac5bdd01b6bd8e9312b3 Mon Sep 17 00:00:00 2001 From: Raye Kimmerer Date: Wed, 2 Jul 2025 00:57:44 -0700 Subject: [PATCH 088/154] Clarify guidelines (#4096) Co-authored-by: Kristoffer Carlsson --- docs/src/creating-packages.md | 10 ++++++---- docs/src/toml-files.md | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/src/creating-packages.md b/docs/src/creating-packages.md index ef3a8fc734..2cee5bf6cc 100644 --- a/docs/src/creating-packages.md +++ b/docs/src/creating-packages.md @@ -562,10 +562,10 @@ duplicated into `[extras]`. This is an unfortunate duplication, but without doing this the project verifier under older Julia versions will throw an error if it finds packages under `[compat]` that is not listed in `[extras]`. -## Package naming rules +## Package naming guidelines Package names should be sensible to most Julia users, *even to those who are not domain experts*. -The following rules apply to the `General` registry but may be useful for other package +The following guidelines apply to the `General` registry but may be useful for other package registries as well. Since the `General` registry belongs to the entire community, people may have opinions about @@ -575,8 +575,10 @@ may fit your package better. 1. Avoid jargon. In particular, avoid acronyms unless there is minimal possibility of confusion. - * It's ok to say `USA` if you're talking about the USA. - * It's not ok to say `PMA`, even if you're talking about positive mental attitude. + * It's ok for package names to contain `DNA` if you're talking about the DNA, which has a universally agreed upon definition. + * It's more difficult to justify package names containing the acronym `CI` for instance, which may mean continuous integration, confidence interval, etc. + * If there is risk of confusion it may be best to disambiguate an acronym with additional words such as a lab group or field. + * If your acronym is unambiguous, easily searchable, and/or unlikely to be confused across domains a good justification is often enough for approval. 2. Avoid using `Julia` in your package name or prefixing it with `Ju`. * It is usually clear from context and to your users that the package is a Julia package. diff --git a/docs/src/toml-files.md b/docs/src/toml-files.md index 0d73d33ffa..4ad1bc569d 100644 --- a/docs/src/toml-files.md +++ b/docs/src/toml-files.md @@ -64,7 +64,7 @@ name = "Example" The name must be a valid [identifier](https://docs.julialang.org/en/v1/base/base/#Base.isidentifier) (a sequence of Unicode characters that does not start with a number and is neither `true` nor `false`). For packages, it is recommended to follow the -[package naming rules](@ref Package-naming-rules). The `name` field is mandatory +[package naming rules](@ref Package-naming-guidelines). The `name` field is mandatory for packages. From 175a1ffb636e2591bb2b4361251cd46990875ee6 Mon Sep 17 00:00:00 2001 From: Timothy Date: Wed, 2 Jul 2025 15:58:02 +0800 Subject: [PATCH 089/154] Switch to more portable shell shebang (#4162) Co-authored-by: Kristoffer Carlsson --- src/Apps/Apps.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Apps/Apps.jl b/src/Apps/Apps.jl index 770c872448..c9e4260415 100644 --- a/src/Apps/Apps.jl +++ b/src/Apps/Apps.jl @@ -468,7 +468,7 @@ function generate_shim(pkgname, app::AppInfo, env, julia) else julia_escaped = Base.shell_escape(julia) module_spec_escaped = Base.shell_escape(module_spec) - bash_shim(julia_escaped, module_spec_escaped, env) + shell_shim(julia_escaped, module_spec_escaped, env) end overwrite_file_if_different(julia_bin_filename, content) if Sys.isunix() @@ -477,9 +477,9 @@ function generate_shim(pkgname, app::AppInfo, env, julia) end -function bash_shim(julia_escaped::String, module_spec_escaped::String, env) +function shell_shim(julia_escaped::String, module_spec_escaped::String, env) return """ - #!/usr/bin/env bash + #!/bin/sh $SHIM_HEADER From ffdb668b81658f1c46041fc921fa03d2fdb9527a Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 2 Jul 2025 10:10:18 +0200 Subject: [PATCH 090/154] fix a header when cloning and use credentials when fetching (#4286) --- src/GitTools.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/GitTools.jl b/src/GitTools.jl index a3b2731d9d..bbce2fb95b 100644 --- a/src/GitTools.jl +++ b/src/GitTools.jl @@ -95,7 +95,7 @@ function clone(io::IO, url, source_path; header=nothing, credentials=nothing, kw @assert !isdir(source_path) || isempty(readdir(source_path)) url = normalize_url(url) printpkgstyle(io, :Cloning, header === nothing ? "git-repo `$url`" : header) - bar = MiniProgressBar(header = "Fetching:", color = Base.info_color()) + bar = MiniProgressBar(header = "Cloning:", color = Base.info_color()) fancyprint = can_fancyprint(io) fancyprint && start_progress(io, bar) if credentials === nothing @@ -122,7 +122,7 @@ function clone(io::IO, url, source_path; header=nothing, credentials=nothing, kw LibGit2.Callbacks() end mkpath(source_path) - return LibGit2.clone(url, source_path; callbacks=callbacks, credentials=credentials, kwargs...) + return LibGit2.clone(url, source_path; callbacks, credentials, kwargs...) end catch err rm(source_path; force=true, recursive=true) @@ -177,7 +177,7 @@ function fetch(io::IO, repo::LibGit2.GitRepo, remoteurl=nothing; header=nothing, end end else - return LibGit2.fetch(repo; remoteurl=remoteurl, callbacks=callbacks, refspecs=refspecs, kwargs...) + return LibGit2.fetch(repo; remoteurl, callbacks, credentials, refspecs, kwargs...) end catch err err isa LibGit2.GitError || rethrow() From c93d6f097808a4ac793db1e72ee78f31ab0fcbf0 Mon Sep 17 00:00:00 2001 From: jariji <96840304+jariji@users.noreply.github.com> Date: Wed, 2 Jul 2025 11:48:34 -0700 Subject: [PATCH 091/154] Show preserve strategy in resolve (#3761) Co-authored-by: Ian Butterworth --- src/Operations.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Operations.jl b/src/Operations.jl index e434f59d56..0ac59ab46f 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -1690,7 +1690,8 @@ end function _resolve(io::IO, env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, preserve::PreserveLevel, julia_version) - printpkgstyle(io, :Resolving, "package versions...") + usingstrategy = preserve != PRESERVE_TIERED ? " using $preserve" : "" + printpkgstyle(io, :Resolving, "package versions$(usingstrategy)...") if preserve == PRESERVE_TIERED_INSTALLED tiered_resolve(env, registries, pkgs, julia_version, true) elseif preserve == PRESERVE_TIERED From 96a10b570038418bbcfe0f0fb9d6d3db69b4a54b Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 2 Jul 2025 23:15:49 +0200 Subject: [PATCH 092/154] update the Pkg.test docs (#3173) Co-authored-by: Ian Butterworth --- src/Pkg.jl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Pkg.jl b/src/Pkg.jl index 399d9572cf..5f38cfa31d 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -289,17 +289,16 @@ const update = API.up !!! compat "Julia 1.9" Passing a string to `coverage` requires at least Julia 1.9. -Run the tests for package `pkg`, or for the current project (which thus needs to be a package) if no -positional argument is given to `Pkg.test`. A package is tested by running its -`test/runtests.jl` file. +Run the tests for the given package(s), or for the current project if no positional argument is given to `Pkg.test` +(the current project would need to be a package). The package is tested by running its `test/runtests.jl` file. -The tests are run by generating a temporary environment with only the `pkg` package -and its (recursive) dependencies in it. If a manifest file exists and the `allow_reresolve` -keyword argument is set to `false`, the versions in the manifest file are used. -Otherwise a feasible set of packages is resolved and installed. +The tests are run in a temporary environment that also includes the test specific dependencies +of the package. The versions of dependencies in the current project are used for the +test environment unless there is a compatibility conflict between the version of the dependencies and +the test-specific dependencies. In that case, if `allow_reresolve` is `false` an error is thrown and +if `allow_reresolve` is `true` a feasible set of versions of the dependencies is resolved and used. -During the tests, test-specific dependencies are active, which are -given in the project file as e.g. +Test-specific dependnecies are declared in the project file as: ```toml [extras] @@ -311,6 +310,7 @@ test = ["Test"] The tests are executed in a new process with `check-bounds=yes` and by default `startup-file=no`. If using the startup file (`~/.julia/config/startup.jl`) is desired, start julia with `--startup-file=yes`. + Inlining of functions during testing can be disabled (for better coverage accuracy) by starting julia with `--inline=no`. The tests can be run as if different command line arguments were passed to julia by passing the arguments instead to the `julia_args` keyword argument, e.g. From 5b4790c4eaef7b3dbd97d374a4177e3cf0a14112 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Wed, 2 Jul 2025 17:17:04 -0400 Subject: [PATCH 093/154] `Pkg.test`: add more detail to the "Could not use exact versions of packages in manifest, re-resolving" message (#3525) Co-authored-by: Ian Butterworth --- src/Operations.jl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Operations.jl b/src/Operations.jl index 0ac59ab46f..ea6b3ebfbc 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -2212,7 +2212,15 @@ function sandbox(fn::Function, ctx::Context, target::PackageSpec, err isa Resolve.ResolverError || rethrow() allow_reresolve || rethrow() @debug err - printpkgstyle(ctx.io, :Test, "Could not use exact versions of packages in manifest. Re-resolving dependencies", color=Base.warn_color()) + msg = string( + "Could not use exact versions of packages in manifest, re-resolving. ", + "Note: if you do not check your manifest file into source control, ", + "then you can probably ignore this message. ", + "However, if you do check your manifest file into source control, ", + "then you probably want to pass the `allow_reresolve = false` kwarg ", + "when calling the `Pkg.test` function.", + ) + printpkgstyle(ctx.io, :Test, msg, color=Base.warn_color()) Pkg.update(temp_ctx; skip_writing_project=true, update_registry=false, io=ctx.io) printpkgstyle(ctx.io, :Test, "Successfully re-resolved") @debug "Using _clean_ dep graph" From f3ed29e7ea4005e57657fc1cfe3a8339e9052fd4 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 2 Jul 2025 22:17:48 +0100 Subject: [PATCH 094/154] Update development instructions (#3625) Co-authored-by: Ian Butterworth --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5cc370c4c9..39b67ac995 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ If you want to develop this package do the following steps: - Make a fork and then clone the repo locally on your computer - Change the current directory to the Pkg repo you just cloned and start julia with `julia --project`. - `import Pkg` will now load the files in the cloned repo instead of the Pkg stdlib. -- To test your changes, simply do `include("test/runtests.jl")`. +- To test your changes, simply do `Pkg.test()`. If you need to build Julia from source with a Git checkout of Pkg, then instead use `make DEPS_GIT=Pkg` when building Julia. The `Pkg` repo is in `stdlib/Pkg`, and created initially with a detached `HEAD`. If you're doing this from a pre-existing Julia repository, you may need to `make clean` beforehand. From 799f7de320452f730bad5459ad97bee19ff25a65 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 2 Jul 2025 22:18:20 +0100 Subject: [PATCH 095/154] Fix bind_artifact! when platform has compare_strategy (#3624) Co-authored-by: Ian Butterworth --- src/Artifacts.jl | 2 +- test/artifacts.jl | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Artifacts.jl b/src/Artifacts.jl index 3509887bd4..4b9686f9de 100644 --- a/src/Artifacts.jl +++ b/src/Artifacts.jl @@ -220,7 +220,7 @@ function bind_artifact!(artifacts_toml::String, name::String, hash::SHA1; meta = artifact_dict[name] if !isa(meta, Vector) error("Mapping for '$name' within $(artifacts_toml) already exists!") - elseif any(isequal(platform), unpack_platform(x, name, artifacts_toml) for x in meta) + elseif any(p -> platforms_match(platform, p), unpack_platform(x, name, artifacts_toml) for x in meta) error("Mapping for '$name'/$(triplet(platform)) within $(artifacts_toml) already exists!") end end diff --git a/test/artifacts.jl b/test/artifacts.jl index feb1fec4db..12abc9e085 100644 --- a/test/artifacts.jl +++ b/test/artifacts.jl @@ -256,6 +256,12 @@ end @test ensure_artifact_installed("foo_txt", artifacts_toml; platform=linux64) == artifact_path(hash2) @test ensure_artifact_installed("foo_txt", artifacts_toml; platform=win32) == artifact_path(hash) + # Default HostPlatform() adds a compare_strategy key that doesn't get picked up from + # the Artifacts.toml + testhost = Platform("x86_64", "linux", Dict("libstdcxx_version" => "1.2.3")) + BinaryPlatforms.set_compare_strategy!(testhost, "libstdcxx_version", BinaryPlatforms.compare_version_cap) + @test_throws ErrorException bind_artifact!(artifacts_toml, "foo_txt", hash; download_info=download_info, platform=testhost) + # Next, check that we can get the download_info properly: meta = artifact_meta("foo_txt", artifacts_toml; platform=win32) @test meta["download"][1]["url"] == "http://google.com/hello_world" From 4f122a8ac5ff5cad6ab52cc581caaa438a1f6469 Mon Sep 17 00:00:00 2001 From: Dennis Hoelgaard Bal <61620837+KronosTheLate@users.noreply.github.com> Date: Wed, 2 Jul 2025 23:19:05 +0200 Subject: [PATCH 096/154] Add more info to deprecation warning of `installed` (#3452) Co-authored-by: Ian Butterworth --- src/Pkg.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Pkg.jl b/src/Pkg.jl index 5f38cfa31d..9bf7f14ceb 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -831,7 +831,7 @@ end ################ function installed() - @warn "Pkg.installed() is deprecated" + @warn "`Pkg.installed()` is deprecated. Use `Pkg.dependencies()` instead." maxlog=1 deps = dependencies() installs = Dict{String, VersionNumber}() for (uuid, dep) in deps From a704d2eb7011f49e7d5c761ec75173425b9c794a Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Wed, 2 Jul 2025 23:59:49 +0200 Subject: [PATCH 097/154] generate: use the same default code as Example.jl (#2601) Co-authored-by: Ian Butterworth --- src/generate.jl | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/generate.jl b/src/generate.jl index 6134a1686c..fdb729c969 100644 --- a/src/generate.jl +++ b/src/generate.jl @@ -61,10 +61,22 @@ end function entrypoint(io::IO, pkg::AbstractString, dir) genfile(io, joinpath(dir, "src"), "$pkg.jl") do file_io print(file_io, - """ + """ module $pkg - greet() = print("Hello World!") + \""" + hello(who::String) + + Return "Hello, `who`". + \""" + hello(who::String) = "Hello, \$who" + + \""" + domath(x::Number) + + Return `x + 5`. + \""" + domath(x::Number) = x + 5 end # module $pkg """ From eab1b8b494197af291e20a73e4c5ed8e75b86555 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 3 Jul 2025 00:00:09 +0200 Subject: [PATCH 098/154] propagate `include_lazy` through `download_artifacts` (#3106) Co-authored-by: KristofferC Co-authored-by: Ian Butterworth --- src/Operations.jl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Operations.jl b/src/Operations.jl index ea6b3ebfbc..a34859cbc5 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -824,7 +824,7 @@ function install_git( end end -function collect_artifacts(pkg_root::String; platform::AbstractPlatform=HostPlatform()) +function collect_artifacts(pkg_root::String; platform::AbstractPlatform=HostPlatform(), include_lazy::Bool=false) # Check to see if this package has an (Julia)Artifacts.toml artifacts_tomls = Tuple{String,Base.TOML.TOMLDict}[] for f in artifact_names @@ -848,7 +848,7 @@ function collect_artifacts(pkg_root::String; platform::AbstractPlatform=HostPlat end else # Otherwise, use the standard selector from `Artifacts` - artifacts = select_downloadable_artifacts(artifacts_toml; platform) + artifacts = select_downloadable_artifacts(artifacts_toml; platform, include_lazy) push!(artifacts_tomls, (artifacts_toml, artifacts)) end break @@ -868,7 +868,9 @@ end function download_artifacts(ctx::Context; platform::AbstractPlatform=HostPlatform(), julia_version = VERSION, - verbose::Bool=false) + verbose::Bool=false, + io::IO=stderr_f(), + include_lazy::Bool=false) env = ctx.env io = ctx.io fancyprint = can_fancyprint(io) @@ -894,7 +896,7 @@ function download_artifacts(ctx::Context; ansi_enablecursor = "\e[?25h" ansi_disablecursor = "\e[?25l" - all_collected_artifacts = reduce(vcat, map(pkg_root -> collect_artifacts(pkg_root; platform), pkg_roots)) + all_collected_artifacts = reduce(vcat, map(pkg_root -> collect_artifacts(pkg_root; platform, include_lazy), pkg_roots)) used_artifact_tomls = Set{String}(map(first, all_collected_artifacts)) longest_name_length = maximum(all_collected_artifacts; init=0) do (artifacts_toml, artifacts) maximum(textwidth, keys(artifacts); init=0) From 00ef215f5dab08adfa09b56266a9124b24ceb265 Mon Sep 17 00:00:00 2001 From: t-bltg Date: Thu, 3 Jul 2025 00:00:38 +0200 Subject: [PATCH 099/154] warn & strip on accidental leading `]` (#3122) Co-authored-by: Ian Butterworth --- src/REPLMode/REPLMode.jl | 4 ++++ test/repl.jl | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/src/REPLMode/REPLMode.jl b/src/REPLMode/REPLMode.jl index b0c70745b6..f3e1c249f7 100644 --- a/src/REPLMode/REPLMode.jl +++ b/src/REPLMode/REPLMode.jl @@ -225,6 +225,10 @@ end function tokenize(cmd::AbstractString) cmd = replace(replace(cmd, "\r\n" => "; "), "\n" => "; ") # for multiline commands + if startswith(cmd, ']') + @warn "Removing leading `]`, which should only be used once to switch to pkg> mode" + cmd = string(lstrip(cmd, ']')) + end qstrings = lex(cmd) statements = foldl(qstrings; init=[QString[]]) do collection, next (next.raw == ";" && !next.isquoted) ? diff --git a/test/repl.jl b/test/repl.jl index 0d2f0e94ad..709fcc0b52 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -25,6 +25,16 @@ using ..Utils @test_throws PkgError pkg"helpadd" end +@testset "accidental" begin + @test_logs (:warn, r"Removing leading.*") pkg"]?" + @test_logs (:warn, r"Removing leading.*") pkg"] ?" + @test_logs (:warn, r"Removing leading.*") pkg"]st" + @test_logs (:warn, r"Removing leading.*") pkg"] st" + @test_logs (:warn, r"Removing leading.*") pkg"]st -m" + @test_logs (:warn, r"Removing leading.*") pkg"] st -m" + @test_logs (:warn, r"Removing leading.*") pkg"]" # noop +end + temp_pkg_dir() do project_path with_pkg_env(project_path; change_dir=true) do; pkg"generate HelloWorld" From ae65b1eeee67134023a5bae643179c78e8e6e069 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Thu, 3 Jul 2025 10:15:30 +1200 Subject: [PATCH 100/154] Note disabling autoprecompilation in Pkg.instantiate docstring (#3613) Co-authored-by: Ian Butterworth --- src/Pkg.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Pkg.jl b/src/Pkg.jl index 9bf7f14ceb..89af928e11 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -520,10 +520,11 @@ dependencies in the manifest and instantiate the resulting project. `julia_version_strict=true` will turn manifest version check failures into errors instead of logging warnings. After packages have been installed the project will be precompiled. -See more at [Environment Precompilation](@ref). +See more and how to disable auto-precompilation at [Environment Precompilation](@ref). !!! compat "Julia 1.12" The `julia_version_strict` keyword argument requires at least Julia 1.12. + """ const instantiate = API.instantiate From d1d6a2b8995edd984fb54dc4532f68a7479ff92c Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 3 Jul 2025 03:15:14 -0400 Subject: [PATCH 101/154] `Pkg.Operations.install_archive`: when unpacking the archive, always rethrow the caught exception unless it is a `ProcessFailedException` (#2976) --- src/Operations.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Operations.jl b/src/Operations.jl index a34859cbc5..fcd2d04488 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -747,7 +747,7 @@ function install_archive( try unpack(path, dir; verbose=false) catch e - e isa InterruptException && rethrow() + e isa ProcessFailedException || rethrow() @warn "failed to extract archive downloaded from $(url)" url_success = false end From 3440600f60ec8207e920a483aa5ac531dc34c2be Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 3 Jul 2025 10:30:35 +0200 Subject: [PATCH 102/154] Refactor: Extract geturl function and improve error message (#4272) --- src/GitTools.jl | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/GitTools.jl b/src/GitTools.jl index bbce2fb95b..2e5c832998 100644 --- a/src/GitTools.jl +++ b/src/GitTools.jl @@ -141,11 +141,15 @@ function clone(io::IO, url, source_path; header=nothing, credentials=nothing, kw end end +function geturl(repo) + return LibGit2.with(LibGit2.get(LibGit2.GitRemote, repo, "origin")) do remote + LibGit2.url(remote) + end +end + function fetch(io::IO, repo::LibGit2.GitRepo, remoteurl=nothing; header=nothing, credentials=nothing, refspecs=[""], kwargs...) if remoteurl === nothing - remoteurl = LibGit2.with(LibGit2.get(LibGit2.GitRemote, repo, "origin")) do remote - LibGit2.url(remote) - end + remoteurl = geturl(repo) end fancyprint = can_fancyprint(io) remoteurl = normalize_url(remoteurl) @@ -341,7 +345,12 @@ tree_hash(root::AbstractString; debug_out::Union{IO,Nothing} = nothing) = tree_h function check_valid_HEAD(repo) try LibGit2.head(repo) catch err - Pkg.Types.pkgerror("invalid git HEAD ($(err.msg))") + url = try + geturl(repo) + catch + "(unknown url)" + end + Pkg.Types.pkgerror("invalid git HEAD in $url ($(err.msg))") end end From 3869055abc37a360ccf52b268975005dec7f237f Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 3 Jul 2025 12:13:09 +0200 Subject: [PATCH 103/154] Enhance fuzzy matching algorithm with multi-factor scoring (#4287) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrite the fuzzy sorting algorithm to provide more intuitive package name suggestions: Prefix matching prioritized for exact starts (e.g., "Plo" → "Plots") Subsequence matching with position weighting for scattered matches Weighted edit distance with common typo handling (a↔e, c↔k, etc.) Case preservation scoring and length penalties Optional popularity weighting for future use Co-authored-by: KristofferC Co-authored-by: Claude --- CHANGELOG.md | 1 + src/Types.jl | 2 +- src/fuzzysorting.jl | 356 ++++++++++++++++++++++++++++++-------------- test/new.jl | 23 ++- 4 files changed, 267 insertions(+), 115 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cab492167..ea7b9d5409 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ Pkg v1.12 Release Notes - When adding or developing a package that exists in the `[weakdeps]` section, it is now automatically removed from weak dependencies and added as a regular dependency. ([#3865]) - Packages are also automatically added to `[sources]` when they are added by url or devved. +- Enhanced fuzzy matching algorithm for package name suggestions. Pkg v1.11 Release Notes ======================= diff --git a/src/Types.jl b/src/Types.jl index a3a939e575..5df75b4f0f 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -1250,7 +1250,7 @@ function write_env(env::EnvCache; update_undo=true, if env.project.readonly pkgerror("Cannot modify a readonly environment. The project at $(env.project_file) is marked as readonly.") end - + if (env.project != env.original_project) && (!skip_writing_project) write_project(env) end diff --git a/src/fuzzysorting.jl b/src/fuzzysorting.jl index 0d8d842b7f..ba2d8f82fd 100644 --- a/src/fuzzysorting.jl +++ b/src/fuzzysorting.jl @@ -2,129 +2,283 @@ module FuzzySorting _displaysize(io::IO) = displaysize(io)::Tuple{Int,Int} -# This code is duplicated from REPL.jl -# Considering breaking this into an independent package +# Character confusion weights for fuzzy matching +const CHARACTER_CONFUSIONS = Dict( + ('a', 'e') => 0.5, ('e', 'a') => 0.5, + ('i', 'y') => 0.5, ('y', 'i') => 0.5, + ('u', 'o') => 0.5, ('o', 'u') => 0.5, + ('c', 'k') => 0.3, ('k', 'c') => 0.3, + ('s', 'z') => 0.3, ('z', 's') => 0.3, + # Keyboard proximity (QWERTY layout) + ('q', 'w') => 0.4, ('w', 'q') => 0.4, + ('w', 'e') => 0.4, ('e', 'w') => 0.4, + ('e', 'r') => 0.4, ('r', 'e') => 0.4, + ('r', 't') => 0.4, ('t', 'r') => 0.4, + ('t', 'y') => 0.4, ('y', 't') => 0.4, + ('y', 'u') => 0.4, ('u', 'y') => 0.4, + ('u', 'i') => 0.4, ('i', 'u') => 0.4, + ('i', 'o') => 0.4, ('o', 'i') => 0.4, + ('o', 'p') => 0.4, ('p', 'o') => 0.4, + ('a', 's') => 0.4, ('s', 'a') => 0.4, + ('s', 'd') => 0.4, ('d', 's') => 0.4, + ('d', 'f') => 0.4, ('f', 'd') => 0.4, + ('f', 'g') => 0.4, ('g', 'f') => 0.4, + ('g', 'h') => 0.4, ('h', 'g') => 0.4, + ('h', 'j') => 0.4, ('j', 'h') => 0.4, + ('j', 'k') => 0.4, ('k', 'j') => 0.4, + ('k', 'l') => 0.4, ('l', 'k') => 0.4, + ('z', 'x') => 0.4, ('x', 'z') => 0.4, + ('x', 'c') => 0.4, ('c', 'x') => 0.4, + ('c', 'v') => 0.4, ('v', 'c') => 0.4, + ('v', 'b') => 0.4, ('b', 'v') => 0.4, + ('b', 'n') => 0.4, ('n', 'b') => 0.4, + ('n', 'm') => 0.4, ('m', 'n') => 0.4, +) -# Search & Rescue -# Utilities for correcting user mistakes and (eventually) -# doing full documentation searches from the repl. +# Enhanced fuzzy scoring with multiple factors +function fuzzyscore(needle::AbstractString, haystack::AbstractString) + needle_lower, haystack_lower = lowercase(needle), lowercase(haystack) -# Fuzzy Search Algorithm + # Factor 1: Prefix matching bonus (highest priority) + prefix_score = prefix_match_score(needle_lower, haystack_lower) -function matchinds(needle, haystack; acronym::Bool = false) - chars = collect(needle) - is = Int[] - lastc = '\0' - for (i, char) in enumerate(haystack) - while !isempty(chars) && isspace(first(chars)) - popfirst!(chars) # skip spaces - end - isempty(chars) && break - if lowercase(char) == lowercase(chars[1]) && - (!acronym || !isletter(lastc)) - push!(is, i) - popfirst!(chars) + # Factor 2: Subsequence matching + subseq_score = subsequence_score(needle_lower, haystack_lower) + + # Factor 3: Character-level similarity (improved edit distance) + char_score = character_similarity_score(needle_lower, haystack_lower) + + # Factor 4: Case preservation bonus + case_score = case_preservation_score(needle, haystack) + + # Factor 5: Length penalty for very long matches + length_penalty = length_penalty_score(needle, haystack) + + # Weighted combination + base_score = 0.4 * prefix_score + 0.3 * subseq_score + 0.2 * char_score + 0.1 * case_score + final_score = base_score * length_penalty + + return final_score +end + +# Prefix matching: exact prefix gets maximum score +function prefix_match_score(needle::AbstractString, haystack::AbstractString) + if startswith(haystack, needle) + return 1.0 + elseif startswith(needle, haystack) + return 0.9 # Partial prefix match + else + # Check for prefix after common separators + for sep in ['_', '-', '.'] + parts = split(haystack, sep) + for part in parts + if startswith(part, needle) + return 0.7 # Component prefix match + end + end end - lastc = char + return 0.0 end - return is end -longer(x, y) = length(x) ≥ length(y) ? (x, true) : (y, false) +# Subsequence matching with position weighting +function subsequence_score(needle::AbstractString, haystack::AbstractString) + if isempty(needle) + return 1.0 + end -bestmatch(needle, haystack) = - longer(matchinds(needle, haystack, acronym = true), - matchinds(needle, haystack)) + needle_chars = collect(needle) + haystack_chars = collect(haystack) -# Optimal string distance: Counts the minimum number of insertions, deletions, -# transpositions or substitutions to go from one string to the other. -function string_distance(a::AbstractString, lena::Integer, b::AbstractString, lenb::Integer) - if lena > lenb - a, b = b, a - lena, lenb = lenb, lena - end - start = 0 - for (i, j) in zip(a, b) - if a == b - start += 1 - else - break + matched_positions = Int[] + haystack_idx = 1 + + for needle_char in needle_chars + found = false + for i in haystack_idx:length(haystack_chars) + if haystack_chars[i] == needle_char + push!(matched_positions, i) + haystack_idx = i + 1 + found = true + break + end + end + if !found + return 0.0 end end - start == lena && return lenb - start - vzero = collect(1:(lenb - start)) - vone = similar(vzero) - prev_a, prev_b = first(a), first(b) - current = 0 - for (i, ai) in enumerate(a) - i > start || (prev_a = ai; continue) - left = i - start - 1 - current = i - start - transition_next = 0 - for (j, bj) in enumerate(b) - j > start || (prev_b = bj; continue) - # No need to look beyond window of lower right diagonal - above = current - this_transition = transition_next - transition_next = vone[j - start] - vone[j - start] = current = left - left = vzero[j - start] - if ai != bj - # Minimum between substitution, deletion and insertion - current = min(current + 1, above + 1, left + 1) - if i > start + 1 && j > start + 1 && ai == prev_b && prev_a == bj - current = min(current, (this_transition += 1)) - end - end - vzero[j - start] = current - prev_b = bj + + # Calculate score based on how clustered the matches are + if length(matched_positions) <= 1 + return 1.0 + end + + # Penalize large gaps between matches + gaps = diff(matched_positions) + avg_gap = sum(gaps) / length(gaps) + gap_penalty = 1.0 / (1.0 + avg_gap / 3.0) + + # Bonus for matches at word boundaries + boundary_bonus = 0.0 + for pos in matched_positions + if pos == 1 || haystack_chars[pos-1] in ['_', '-', '.'] + boundary_bonus += 0.1 end - prev_a = ai end - current -end -function fuzzyscore(needle::AbstractString, haystack::AbstractString) - lena, lenb = length(needle), length(haystack) - 1 - (string_distance(needle, lena, haystack, lenb) / max(lena, lenb)) + coverage = length(needle) / length(haystack) + return min(1.0, gap_penalty + boundary_bonus) * coverage end -function fuzzysort(search::String, candidates::Vector{String}) - scores = map(cand -> (FuzzySorting.fuzzyscore(search, cand), -Float64(FuzzySorting.levenshtein(search, cand))), candidates) - candidates[sortperm(scores)] |> reverse, any(s -> s[1] >= print_score_threshold, scores) +# Improved character-level similarity +function character_similarity_score(needle::AbstractString, haystack::AbstractString) + if isempty(needle) || isempty(haystack) + return 0.0 + end + + # Use Damerau-Levenshtein distance with character confusion weights + distance = weighted_edit_distance(needle, haystack) + max_len = max(length(needle), length(haystack)) + + return max(0.0, 1.0 - distance / max_len) end -# Levenshtein Distance +# Weighted edit distance accounting for common typos +function weighted_edit_distance(s1::AbstractString, s2::AbstractString) -function levenshtein(s1, s2) a, b = collect(s1), collect(s2) - m = length(a) - n = length(b) - d = Matrix{Int}(undef, m+1, n+1) + m, n = length(a), length(b) + # Initialize distance matrix + d = Matrix{Float64}(undef, m+1, n+1) d[1:m+1, 1] = 0:m d[1, 1:n+1] = 0:n for i = 1:m, j = 1:n - d[i+1,j+1] = min(d[i , j+1] + 1, - d[i+1, j ] + 1, - d[i , j ] + (a[i] != b[j])) + if a[i] == b[j] + d[i+1, j+1] = d[i, j] # No cost for exact match + else + # Standard operations + insert_cost = d[i, j+1] + 1.0 + delete_cost = d[i+1, j] + 1.0 + + # Check for repeated character deletion (common typo) + if i > 1 && a[i] == a[i-1] && a[i-1] == b[j] + delete_cost = d[i, j+1] + 0.3 # Low cost for deleting repeated char + end + + # Check for repeated character insertion (common typo) + if j > 1 && b[j] == b[j-1] && a[i] == b[j-1] + insert_cost = d[i, j+1] + 0.3 # Low cost for inserting repeated char + end + + # Substitution with confusion weighting + confusion_key = (a[i], b[j]) + subst_cost = d[i, j] + get(CHARACTER_CONFUSIONS, confusion_key, 1.0) + + d[i+1, j+1] = min(insert_cost, delete_cost, subst_cost) + + # Transposition + if i > 1 && j > 1 && a[i] == b[j-1] && a[i-1] == b[j] + d[i+1, j+1] = min(d[i+1, j+1], d[i-1, j-1] + 1.0) + end + end end return d[m+1, n+1] end -function levsort(search::String, candidates::Vector{String}) - scores = map(cand -> (Float64(levenshtein(search, cand)), -fuzzyscore(search, cand)), candidates) - candidates = candidates[sortperm(scores)] - i = 0 - for outer i = 1:length(candidates) - levenshtein(search, candidates[i]) > 3 && break +# Case preservation bonus +function case_preservation_score(needle::AbstractString, haystack::AbstractString) + if isempty(needle) || isempty(haystack) + return 0.0 + end + + matches = 0 + min_len = min(length(needle), length(haystack)) + + for i in 1:min_len + if needle[i] == haystack[i] + matches += 1 + end + end + + return matches / min_len +end + +# Length penalty for very long matches +function length_penalty_score(needle::AbstractString, haystack::AbstractString) + needle_len = length(needle) + haystack_len = length(haystack) + + if needle_len == 0 + return 0.0 + end + + # Strong preference for similar lengths + length_ratio = haystack_len / needle_len + length_diff = abs(haystack_len - needle_len) + + # Bonus for very close lengths (within 1-2 characters) + if length_diff <= 1 + return 1.1 # Small bonus for near-exact length + elseif length_diff <= 2 + return 1.05 + elseif length_ratio <= 1.5 + return 1.0 + elseif length_ratio <= 2.0 + return 0.8 + elseif length_ratio <= 3.0 + return 0.6 + else + return 0.4 # Heavy penalty for very long matches end - return candidates[1:i] end -# Result printing +# Main sorting function with optional popularity weighting +function fuzzysort(search::String, candidates::Vector{String}; popularity_weights::Dict{String,Float64} = Dict{String,Float64}()) + scores = map(candidates) do cand + base_score = fuzzyscore(search, cand) + weight = get(popularity_weights, cand, 1.0) + score = base_score * weight + return (score, cand) + end + + # Sort by score descending, then by candidate name for ties + sorted_scores = sort(scores, by = x -> (-x[1], x[2])) + + # Extract candidates and check if any meet threshold + result_candidates = [x[2] for x in sorted_scores] + has_good_matches = any(x -> x[1] >= print_score_threshold, sorted_scores) + + return result_candidates, has_good_matches +end + +# Keep existing interface functions for compatibility +function matchinds(needle, haystack; acronym::Bool = false) + chars = collect(needle) + is = Int[] + lastc = '\0' + for (i, char) in enumerate(haystack) + while !isempty(chars) && isspace(first(chars)) + popfirst!(chars) # skip spaces + end + isempty(chars) && break + if lowercase(char) == lowercase(chars[1]) && + (!acronym || !isletter(lastc)) + push!(is, i) + popfirst!(chars) + end + lastc = char + end + return is +end + +longer(x, y) = length(x) ≥ length(y) ? (x, true) : (y, false) + +bestmatch(needle, haystack) = + longer(matchinds(needle, haystack, acronym = true), + matchinds(needle, haystack)) function printmatch(io::IO, word, match) is, _ = bestmatch(word, match) @@ -137,7 +291,7 @@ function printmatch(io::IO, word, match) end end -const print_score_threshold = 0.5 +const print_score_threshold = 0.25 function printmatches(io::IO, word, matches; cols::Int = _displaysize(io)[2]) total = 0 @@ -152,25 +306,5 @@ end printmatches(args...; cols::Int = _displaysize(stdout)[2]) = printmatches(stdout, args..., cols = cols) -function print_joined_cols(io::IO, ss::Vector{String}, delim = "", last = delim; cols::Int = _displaysize(io)[2]) - i = 0 - total = 0 - for outer i = 1:length(ss) - total += length(ss[i]) - total + max(i-2,0)*length(delim) + (i>1 ? 1 : 0)*length(last) > cols && (i-=1; break) - end - join(io, ss[1:i], delim, last) -end - -print_joined_cols(args...; cols::Int = _displaysize(stdout)[2]) = print_joined_cols(stdout, args...; cols=cols) - -function print_correction(io::IO, word::String, mod::Module) - cors = map(quote_spaces, levsort(word, accessible(mod))) - pre = "Perhaps you meant " - print(io, pre) - print_joined_cols(io, cors, ", ", " or "; cols = _displaysize(io)[2] - length(pre)) - println(io) - return -end end diff --git a/test/new.jl b/test/new.jl index 0a62a8c916..76983c5b43 100644 --- a/test/new.jl +++ b/test/new.jl @@ -511,14 +511,31 @@ end ) Pkg.add(name="Example", rev="master", version="0.5.0") # Adding with a slight typo gives suggestions try - Pkg.add("Examplle") + io = IOBuffer() + Pkg.add("Examplle"; io) @test false # to fail if add doesn't error catch err @test err isa PkgError @test occursin("The following package names could not be resolved:", err.msg) @test occursin("Examplle (not found in project, manifest or registry)", err.msg) - @test occursin("Suggestions:", err.msg) - # @test occursin("Example", err.msg) # can't test this as each char in "Example" is individually colorized + @test occursin("Suggestions: Example", err.msg) + end + # Adding with lowercase suggests uppercase + try + io = IOBuffer() + Pkg.add("http"; io) + @test false # to fail if add doesn't error + catch err + @test err isa PkgError + @test occursin("Suggestions: HTTP", err.msg) + end + try + io = IOBuffer() + Pkg.add("Flix"; io) + @test false # to fail if add doesn't error + catch err + @test err isa PkgError + @test occursin("Suggestions: Flux", err.msg) end @test_throws PkgError( "name, UUID, URL, or filesystem path specification required when calling `add`" From 69b50d778b154eeed5dc8ab86b8a263e1f163cde Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 3 Jul 2025 14:02:44 +0200 Subject: [PATCH 104/154] fixup CHANGELOG for automatic source additions (#4294) --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea7b9d5409..a9ad60bf16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Pkg v1.13 Release Notes ======================= - Project.toml environments now support a `readonly` field to mark environments as read-only, preventing modifications. ([#4284]) -- Pkg now automatically adds entries to `[sources]` when packages are added by URL or devved, improving workflow consistency. ([#4225]) +- Packages are now automatically added to `[sources]` when they are added by url or devved. Pkg v1.12 Release Notes ======================= @@ -15,7 +15,6 @@ Pkg v1.12 Release Notes - `status` now shows when different versions/sources of dependencies are loaded than that which is expected by the manifest ([#4109]) - When adding or developing a package that exists in the `[weakdeps]` section, it is now automatically removed from weak dependencies and added as a regular dependency. ([#3865]) -- Packages are also automatically added to `[sources]` when they are added by url or devved. - Enhanced fuzzy matching algorithm for package name suggestions. Pkg v1.11 Release Notes From 6636197fd565055f177e662bd9ae13d72d793559 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 3 Jul 2025 14:03:33 +0200 Subject: [PATCH 105/154] Implement atomic TOML writes to prevent data corruption (#4293) Co-authored-by: KristofferC Co-authored-by: Claude --- src/API.jl | 10 +++------- src/Apps/Apps.jl | 5 ++--- src/Artifacts.jl | 12 +++--------- src/PlatformEngines.jl | 9 ++------- src/Registry/Registry.jl | 14 ++++---------- src/Types.jl | 11 +++-------- src/utils.jl | 24 ++++++++++++++++++++++++ 7 files changed, 41 insertions(+), 44 deletions(-) diff --git a/src/API.jl b/src/API.jl index d11ce3d453..4982b1ba04 100644 --- a/src/API.jl +++ b/src/API.jl @@ -14,7 +14,7 @@ import Base: StaleCacheKey import ..depots, ..depots1, ..logdir, ..devdir, ..printpkgstyle import ..Operations, ..GitTools, ..Pkg, ..Registry -import ..can_fancyprint, ..pathrepr, ..isurl, ..PREV_ENV_PATH +import ..can_fancyprint, ..pathrepr, ..isurl, ..PREV_ENV_PATH, ..atomic_toml_write using ..Types, ..TOML using ..Types: VersionTypes using Base.BinaryPlatforms @@ -654,9 +654,7 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, usage_path = joinpath(logdir(depot), fname) if !(isempty(usage)::Bool) || isfile(usage_path) let usage=usage - open(usage_path, "w") do io - TOML.print(io, usage, sorted=true) - end + atomic_toml_write(usage_path, usage, sorted=true) end end end @@ -986,9 +984,7 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, # Write out the `new_orphanage` for this depot mkpath(dirname(orphanage_file)) - open(orphanage_file, "w") do io - TOML.print(io, new_orphanage, sorted=true) - end + atomic_toml_write(orphanage_file, new_orphanage, sorted=true) end function recursive_dir_size(path) diff --git a/src/Apps/Apps.jl b/src/Apps/Apps.jl index c9e4260415..3126a65478 100644 --- a/src/Apps/Apps.jl +++ b/src/Apps/Apps.jl @@ -1,6 +1,7 @@ module Apps using Pkg +using Pkg: atomic_toml_write using Pkg.Versions using Pkg.Types: AppInfo, PackageSpec, Context, EnvCache, PackageEntry, Manifest, handle_repo_add!, handle_repo_develop!, write_manifest, write_project, pkgerror, projectfile_path, manifestfile_path @@ -151,9 +152,7 @@ function _resolve(manifest::Manifest, pkgname=nothing) # TODO: What if project file has its own entryfile? project_data = TOML.parsefile(projectfile) project_data["entryfile"] = joinpath(sourcepath, "src", "$(pkg.name).jl") - open(projectfile, "w") do io - TOML.print(io, project_data) - end + atomic_toml_write(projectfile, project_data) else error("could not find project file for package $pkg") end diff --git a/src/Artifacts.jl b/src/Artifacts.jl index 4b9686f9de..83b71737c4 100644 --- a/src/Artifacts.jl +++ b/src/Artifacts.jl @@ -6,7 +6,7 @@ using Tar: can_symlink using FileWatching: FileWatching import ..set_readonly, ..GitTools, ..TOML, ..pkg_server, ..can_fancyprint, - ..stderr_f, ..printpkgstyle, ..mv_temp_dir_retries + ..stderr_f, ..printpkgstyle, ..mv_temp_dir_retries, ..atomic_toml_write import Base: get, SHA1 import Artifacts: artifact_names, ARTIFACTS_DIR_OVERRIDE, ARTIFACT_OVERRIDES, artifact_paths, @@ -267,11 +267,7 @@ function bind_artifact!(artifacts_toml::String, name::String, hash::SHA1; # Spit it out onto disk let artifact_dict = artifact_dict parent_dir = dirname(artifacts_toml) - temp_artifacts_toml = isempty(parent_dir) ? tempname(pwd()) : tempname(parent_dir) - open(temp_artifacts_toml, "w") do io - TOML.print(io, artifact_dict, sorted=true) - end - mv(temp_artifacts_toml, artifacts_toml; force=true) + atomic_toml_write(artifacts_toml, artifact_dict, sorted=true) end # Mark that we have used this Artifact.toml @@ -302,9 +298,7 @@ function unbind_artifact!(artifacts_toml::String, name::String; ) end - open(artifacts_toml, "w") do io - TOML.print(io, artifact_dict, sorted=true) - end + atomic_toml_write(artifacts_toml, artifact_dict, sorted=true) return end diff --git a/src/PlatformEngines.jl b/src/PlatformEngines.jl index 98f1934559..2d705cfb63 100644 --- a/src/PlatformEngines.jl +++ b/src/PlatformEngines.jl @@ -5,7 +5,7 @@ module PlatformEngines using SHA, Downloads, Tar -import ...Pkg: Pkg, TOML, pkg_server, depots1, can_fancyprint, stderr_f +import ...Pkg: Pkg, TOML, pkg_server, depots1, can_fancyprint, stderr_f, atomic_toml_write using ..MiniProgressBars using Base.BinaryPlatforms, p7zip_jll @@ -188,12 +188,7 @@ function get_auth_header(url::AbstractString; verbose::Bool = false) auth_info["expires_at"] = expires_at end end - let auth_info = auth_info - open(tmp, write=true) do io - TOML.print(io, auth_info, sorted=true) - end - end - mv(tmp, auth_file, force=true) + atomic_toml_write(auth_file, auth_info, sorted=true) access_token = auth_info["access_token"]::String return "Authorization" => "Bearer $access_token" end diff --git a/src/Registry/Registry.jl b/src/Registry/Registry.jl index af5cab05e1..60916fd0f8 100644 --- a/src/Registry/Registry.jl +++ b/src/Registry/Registry.jl @@ -2,7 +2,7 @@ module Registry import ..Pkg using ..Pkg: depots, depots1, printpkgstyle, stderr_f, isdir_nothrow, pathrepr, pkg_server, - GitTools + GitTools, atomic_toml_write using ..Pkg.PlatformEngines: download_verify_unpack, download, download_verify, exe7z, verify_archive_tree_hash using UUIDs, LibGit2, TOML, Dates import FileWatching @@ -212,9 +212,7 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depots::Union{S end mv(tmp, joinpath(regdir, reg.name * ".tar.gz"); force=true) reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(_hash), "path" => reg.name * ".tar.gz") - open(joinpath(regdir, reg.name * ".toml"), "w") do io - TOML.print(io, reg_info) - end + atomic_toml_write(joinpath(regdir, reg.name * ".toml"), reg_info) printpkgstyle(io, :Added, "`$(reg.name)` registry to $(Base.contractuser(regdir))") else mktempdir() do tmp @@ -368,9 +366,7 @@ function save_registry_update_log(d::Dict) pkg_scratch_space = joinpath(DEPOT_PATH[1], "scratchspaces", "44cfe95a-1eb2-52ea-b672-e2afdf69b78f") mkpath(pkg_scratch_space) pkg_reg_updated_file = joinpath(pkg_scratch_space, "registry_updates.toml") - open(pkg_reg_updated_file, "w") do io - TOML.print(io, d) - end + atomic_toml_write(pkg_reg_updated_file, d) end """ @@ -450,9 +446,7 @@ function update(regs::Vector{RegistrySpec}; io::IO=stderr_f(), force::Bool=true, registry_path = dirname(reg.path) mv(tmp, joinpath(registry_path, reg.name * ".tar.gz"); force=true) reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(hash), "path" => reg.name * ".tar.gz") - open(joinpath(registry_path, reg.name * ".toml"), "w") do io - TOML.print(io, reg_info) - end + atomic_toml_write(joinpath(registry_path, reg.name * ".toml"), reg_info) registry_update_log[string(reg.uuid)] = now() @label done_tarball_read else diff --git a/src/Types.jl b/src/Types.jl index 5df75b4f0f..fb16dac000 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -10,7 +10,7 @@ import Base.string using TOML import ..Pkg, ..Registry -import ..Pkg: GitTools, depots, depots1, logdir, set_readonly, safe_realpath, pkg_server, stdlib_dir, stdlib_path, isurl, stderr_f, RESPECT_SYSIMAGE_VERSIONS +import ..Pkg: GitTools, depots, depots1, logdir, set_readonly, safe_realpath, pkg_server, stdlib_dir, stdlib_path, isurl, stderr_f, RESPECT_SYSIMAGE_VERSIONS, atomic_toml_write import Base.BinaryPlatforms: Platform using ..Pkg.Versions import FileWatching @@ -668,15 +668,10 @@ function write_env_usage(source_files, usage_filepath::AbstractString) usage[k] = [Dict("time" => maximum(times))] end - tempfile = tempname() try - open(tempfile, "w") do io - TOML.print(io, usage, sorted=true) - end - TOML.parsefile(tempfile) # compare to `usage` ? - mv(tempfile, usage_file; force=true) # only mv if parse succeeds + atomic_toml_write(usage_file, usage, sorted=true) catch err - @error "Failed to write valid usage file `$usage_file`" tempfile + @error "Failed to write valid usage file `$usage_file`" exception=err end end return diff --git a/src/utils.jl b/src/utils.jl index b71b802617..62f6da6be4 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -152,6 +152,30 @@ function casesensitive_isdir(dir::String) isdir_nothrow(dir) && lastdir in readdir(joinpath(dir, "..")) end +""" + atomic_toml_write(path::String, data; kws...) + +Write TOML data to a file atomically by first writing to a temporary file and then moving it into place. +This prevents "teared" writes if the process is interrupted or if multiple processes write to the same file. + +The `kws` are passed to `TOML.print`. +""" +function atomic_toml_write(path::String, data; kws...) + dir = dirname(path) + isempty(dir) && (dir = pwd()) + + temp_path, temp_io = mktemp(dir) + return try + TOML.print(temp_io, data; kws...) + close(temp_io) + mv(temp_path, path; force = true) + catch + close(temp_io) + rm(temp_path; force = true) + rethrow() + end +end + ## ordering of UUIDs ## if VERSION < v"1.2.0-DEV.269" # Defined in Base as of #30947 Base.isless(a::UUID, b::UUID) = a.value < b.value From 7381ccf45a93674ce94f054a465e7132f50f5092 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 3 Jul 2025 08:18:22 -0600 Subject: [PATCH 106/154] Change refs/* to refs/heads/* to speed up repo cloning w/ many branches (#2330) Co-authored-by: Ian Butterworth Co-authored-by: KristofferC --- src/Types.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Types.jl b/src/Types.jl index fb16dac000..a9aba5b501 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -694,7 +694,7 @@ function read_package(path::String) return project end -const refspecs = ["+refs/*:refs/remotes/cache/*"] +const refspecs = ["+refs/heads/*:refs/remotes/cache/heads/*"] function relative_project_path(project_file::String, path::String) # compute path relative the project From d0c6d50ff79d34cf19204b9ab0a0dc8b03fb6b15 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 3 Jul 2025 16:19:54 +0200 Subject: [PATCH 107/154] also use a bare repo when cloning for `add` using cli GIT (#4296) Co-authored-by: KristofferC --- src/GitTools.jl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/GitTools.jl b/src/GitTools.jl index 2e5c832998..6977a8907b 100644 --- a/src/GitTools.jl +++ b/src/GitTools.jl @@ -89,7 +89,7 @@ function checkout_tree_to_path(repo::LibGit2.GitRepo, tree::LibGit2.GitObject, p end end -function clone(io::IO, url, source_path; header=nothing, credentials=nothing, kwargs...) +function clone(io::IO, url, source_path; header=nothing, credentials=nothing, isbare=false, kwargs...) url = String(url)::String source_path = String(source_path)::String @assert !isdir(source_path) || isempty(readdir(source_path)) @@ -103,7 +103,9 @@ function clone(io::IO, url, source_path; header=nothing, credentials=nothing, kw end try if use_cli_git() - cmd = `git clone --quiet $url $source_path` + args = ["--quiet", url, source_path] + isbare && pushfirst!(args, "--bare") + cmd = `git clone $args` try run(pipeline(cmd; stdout=devnull)) catch err @@ -122,7 +124,7 @@ function clone(io::IO, url, source_path; header=nothing, credentials=nothing, kw LibGit2.Callbacks() end mkpath(source_path) - return LibGit2.clone(url, source_path; callbacks, credentials, kwargs...) + return LibGit2.clone(url, source_path; callbacks, credentials, isbare, kwargs...) end catch err rm(source_path; force=true, recursive=true) @@ -155,7 +157,6 @@ function fetch(io::IO, repo::LibGit2.GitRepo, remoteurl=nothing; header=nothing, remoteurl = normalize_url(remoteurl) printpkgstyle(io, :Updating, header === nothing ? "git-repo `$remoteurl`" : header) bar = MiniProgressBar(header = "Fetching:", color = Base.info_color()) - fancyprint = can_fancyprint(io) callbacks = if fancyprint LibGit2.Callbacks( :transfer_progress => ( From ad9551b264acf92e56f447ad83255097759addc3 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 4 Jul 2025 09:22:54 +0300 Subject: [PATCH 108/154] improve error message info for bad compat entries (#4302) --- src/precompile.jl | 2 +- src/project.jl | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/precompile.jl b/src/precompile.jl index 3a7e2ea32f..bcbbf62864 100644 --- a/src/precompile.jl +++ b/src/precompile.jl @@ -150,7 +150,7 @@ let end Base.precompile(Tuple{typeof(Pkg.API.status)}) - Base.precompile(Tuple{typeof(Pkg.Types.read_project_compat),Base.Dict{String,Any},Pkg.Types.Project,},) + Base.precompile(Tuple{typeof(Pkg.Types.read_project_compat),Base.Dict{String,Any},Pkg.Types.Project},) Base.precompile(Tuple{typeof(Pkg.Versions.semver_interval),Base.RegexMatch}) Base.precompile(Tuple{typeof(Pkg.REPLMode.do_cmds), Array{Pkg.REPLMode.Command, 1}, Base.TTY}) diff --git a/src/project.jl b/src/project.jl index 55b47f0922..2e4fe7c0d6 100644 --- a/src/project.jl +++ b/src/project.jl @@ -90,21 +90,22 @@ function read_project_apps(raw::Dict{String,Any}, project::Project) return appinfos end -read_project_compat(::Nothing, project::Project) = Dict{String,Compat}() -function read_project_compat(raw::Dict{String,Any}, project::Project) +read_project_compat(::Nothing, project::Project; file=nothing) = Dict{String,Compat}() +function read_project_compat(raw::Dict{String,Any}, project::Project; file=nothing) compat = Dict{String,Compat}() + location_string = file === nothing ? "" : " in $(repr(file))" for (name, version) in raw version = version::String try compat[name] = Compat(semver_spec(version), version) catch err - pkgerror("Could not parse compatibility version for dependency `$name`") + pkgerror("Could not parse compatibility version spec $(repr(version)) for dependency `$name`$location_string") end end return compat end -read_project_compat(raw, project::Project) = - pkgerror("Expected `compat` section to be a key-value list") +read_project_compat(raw, project::Project; file=nothing) = + pkgerror("Expected `compat` section to be a key-value list" * (file === nothing ? "" : " in $(repr(file))")) read_project_sources(::Nothing, project::Project) = Dict{String,Dict{String,String}}() function read_project_sources(raw::Dict{String,Any}, project::Project) @@ -209,7 +210,7 @@ function Project(raw::Dict; file=nothing) project.exts = get(Dict{String, String}, raw, "extensions") project.sources = read_project_sources(get(raw, "sources", nothing), project) project.extras = read_project_deps(get(raw, "extras", nothing), "extras") - project.compat = read_project_compat(get(raw, "compat", nothing), project) + project.compat = read_project_compat(get(raw, "compat", nothing), project; file) project.targets = read_project_targets(get(raw, "targets", nothing), project) project.workspace = read_project_workspace(get(raw, "workspace", nothing), project) project.apps = read_project_apps(get(raw, "apps", nothing), project) @@ -271,14 +272,14 @@ function destructure(project::Project)::Dict entry!("extras", project.extras) entry!("compat", Dict(name => x.str for (name, x) in project.compat)) entry!("targets", project.targets) - + # Only write readonly if it's true (not the default false) if project.readonly raw["readonly"] = true else delete!(raw, "readonly") end - + return raw end From 8793b13d57e05d4c859cbded30b35f8765b24268 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 4 Jul 2025 09:23:54 +0300 Subject: [PATCH 109/154] Make tips show the repl mode when in repl mode (#3854) --- src/API.jl | 13 +++++++++---- src/Operations.jl | 19 ++++++++++++++----- src/Pkg.jl | 13 ++++++++++++- src/REPLMode/REPLMode.jl | 23 +++++++++++++---------- src/Registry/Registry.jl | 3 ++- test/repl.jl | 26 ++++++++++++++++++++++++++ 6 files changed, 76 insertions(+), 21 deletions(-) diff --git a/src/API.jl b/src/API.jl index 4982b1ba04..2c8f61ce23 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1230,17 +1230,22 @@ function instantiate(ctx::Context; manifest::Union{Bool, Nothing}=nothing, Types.check_manifest_julia_version_compat(ctx.env.manifest, ctx.env.manifest_file; julia_version_strict) if Operations.is_manifest_current(ctx.env) === false + resolve_cmd = Pkg.in_repl_mode() ? "pkg> resolve" : "Pkg.resolve()" + update_cmd = Pkg.in_repl_mode() ? "pkg> update" : "Pkg.update()" @warn """The project dependencies or compat requirements have changed since the manifest was last resolved. - It is recommended to `Pkg.resolve()` or consider `Pkg.update()` if necessary.""" + It is recommended to `$resolve_cmd` or consider `$update_cmd` if necessary.""" end Operations.prune_manifest(ctx.env) for (name, uuid) in ctx.env.project.deps get(ctx.env.manifest, uuid, nothing) === nothing || continue + resolve_cmd = Pkg.in_repl_mode() ? "pkg> resolve" : "Pkg.resolve()" + rm_cmd = Pkg.in_repl_mode() ? "pkg> rm $name" : "Pkg.rm(\"$name\")" + instantiate_cmd = Pkg.in_repl_mode() ? "pkg> instantiate" : "Pkg.instantiate()" pkgerror("`$name` is a direct dependency, but does not appear in the manifest.", - " If you intend `$name` to be a direct dependency, run `Pkg.resolve()` to populate the manifest.", - " Otherwise, remove `$name` with `Pkg.rm(\"$name\")`.", - " Finally, run `Pkg.instantiate()` again.") + " If you intend `$name` to be a direct dependency, run `$resolve_cmd` to populate the manifest.", + " Otherwise, remove `$name` with `$rm_cmd`.", + " Finally, run `$instantiate_cmd` again.") end # check if all source code and artifacts are downloaded to exit early if Operations.is_instantiated(ctx.env, workspace; platform) diff --git a/src/Operations.jl b/src/Operations.jl index fcd2d04488..0a24273d91 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -1627,21 +1627,21 @@ function assert_can_add(ctx::Context, pkgs::Vector{PackageSpec}) existing_uuid == pkg.uuid || pkgerror("""Refusing to add package $(err_rep(pkg)). Package `$(pkg.name)=$(existing_uuid)` with the same name already exists as a direct dependency. - To remove the existing package, use `import Pkg; Pkg.rm("$(pkg.name)")`. + To remove the existing package, use `$(Pkg.in_repl_mode() ? """pkg> rm $(pkg.name)""" : """import Pkg; Pkg.rm("$(pkg.name)")""")`. """) # package with the same uuid exist in the project: assert they have the same name name = findfirst(==(pkg.uuid), ctx.env.project.deps) name === nothing || name == pkg.name || pkgerror("""Refusing to add package $(err_rep(pkg)). Package `$name=$(pkg.uuid)` with the same UUID already exists as a direct dependency. - To remove the existing package, use `import Pkg; Pkg.rm("$name")`. + To remove the existing package, use `$(Pkg.in_repl_mode() ? """pkg> rm $name""" : """import Pkg; Pkg.rm("$name")""")`. """) # package with the same uuid exist in the manifest: assert they have the same name entry = get(ctx.env.manifest, pkg.uuid, nothing) entry === nothing || entry.name == pkg.name || pkgerror("""Refusing to add package $(err_rep(pkg)). Package `$(entry.name)=$(pkg.uuid)` with the same UUID already exists in the manifest. - To remove the existing package, use `import Pkg; Pkg.rm(Pkg.PackageSpec(uuid="$(pkg.uuid)"); mode=Pkg.PKGMODE_MANIFEST)`. + To remove the existing package, use `$(Pkg.in_repl_mode() ? """pkg> rm --manifest $(entry.name)=$(pkg.uuid)""" : """import Pkg; Pkg.rm(Pkg.PackageSpec(uuid="$(pkg.uuid)"); mode=Pkg.PKGMODE_MANIFEST)""")`. """) end end @@ -1932,7 +1932,8 @@ end function update_package_pin!(registries::Vector{Registry.RegistryInstance}, pkg::PackageSpec, entry::Union{Nothing, PackageEntry}) if entry === nothing - pkgerror("package $(err_rep(pkg)) not found in the manifest, run `Pkg.resolve()` and retry.") + cmd = Pkg.in_repl_mode() ? "pkg> resolve" : "Pkg.resolve()" + pkgerror("package $(err_rep(pkg)) not found in the manifest, run `$cmd` and retry.") end #if entry.pinned && pkg.version == VersionSpec() @@ -2961,7 +2962,15 @@ function status(env::EnvCache, registries::Vector{Registry.RegistryInstance}, pk print_status(env, old_env, registries, header, filter_uuids, filter_names; diff, ignore_indent, io, workspace, outdated, extensions, mode, hidden_upgrades_info, show_usagetips) end if is_manifest_current(env) === false - tip = show_usagetips ? " It is recommended to `Pkg.resolve()` or consider `Pkg.update()` if necessary." : "" + tip = if show_usagetips + if Pkg.in_repl_mode() + " It is recommended to `pkg> resolve` or consider `pkg> update` if necessary." + else + " It is recommended to `Pkg.resolve()` or consider `Pkg.update()` if necessary." + end + else + "" + end printpkgstyle(io, :Warning, "The project dependencies or compat requirements have changed since the manifest was last resolved.$tip", ignore_indent; color=Base.warn_color()) end diff --git a/src/Pkg.jl b/src/Pkg.jl index 89af928e11..83fde0c646 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -52,6 +52,9 @@ const RESPECT_SYSIMAGE_VERSIONS = Ref(true) # For globally overriding in e.g. tests const DEFAULT_IO = Ref{Union{IO,Nothing}}(nothing) +# ScopedValue to track whether we're currently in REPL mode +const IN_REPL_MODE = Base.ScopedValues.ScopedValue{Bool}() + # See discussion in https://github.com/JuliaLang/julia/pull/52249 function unstableio(@nospecialize(io::IO)) # Needed to prevent specialization https://github.com/JuliaLang/julia/pull/52249#discussion_r1401199265 @@ -69,6 +72,14 @@ usable_io(io) = (io isa Base.TTY) || (io isa IOContext{IO} && io.io isa Base.TTY can_fancyprint(io::IO) = (usable_io(io)) && (get(ENV, "CI", nothing) != "true") should_autoprecompile() = Base.JLOptions().use_compiled_modules == 1 && Base.get_bool_env("JULIA_PKG_PRECOMPILE_AUTO", true) +""" + in_repl_mode() + +Check if we're currently executing in REPL mode. This is used to determine +whether to show tips in REPL format (`pkg> add Foo`) or API format (`Pkg.add("Foo")`). +""" +in_repl_mode() = @something(Base.ScopedValues.get(IN_REPL_MODE), false) + include("utils.jl") include("MiniProgressBars.jl") include("GitTools.jl") @@ -723,7 +734,7 @@ Other choices for `protocol` are `"https"` or `"git"`. ```julia-repl julia> Pkg.setprotocol!(domain = "github.com", protocol = "ssh") -# Use HTTPS for GitHub (default, good for most users) +# Use HTTPS for GitHub (default, good for most users) julia> Pkg.setprotocol!(domain = "github.com", protocol = "https") # Reset to default (let package developer decide) diff --git a/src/REPLMode/REPLMode.jl b/src/REPLMode/REPLMode.jl index f3e1c249f7..030bc4dba0 100644 --- a/src/REPLMode/REPLMode.jl +++ b/src/REPLMode/REPLMode.jl @@ -6,7 +6,7 @@ module REPLMode using Markdown, UUIDs, Dates -import ..casesensitive_isdir, ..OFFLINE_MODE, ..linewrap, ..pathrepr +import ..casesensitive_isdir, ..OFFLINE_MODE, ..linewrap, ..pathrepr, ..IN_REPL_MODE using ..Types, ..Operations, ..API, ..Registry, ..Resolve, ..Apps import ..stdout_f, ..stderr_f @@ -402,15 +402,18 @@ function do_cmds(commands::Vector{Command}, io) end function do_cmd(command::Command, io) - # REPL specific commands - command.spec === SPECS["package"]["help"] && return Base.invokelatest(do_help!, command, io) - # API commands - if command.spec.should_splat - TEST_MODE[] && return command.spec.api, command.arguments..., command.options - command.spec.api(command.arguments...; collect(command.options)...) # TODO is invokelatest still needed? - else - TEST_MODE[] && return command.spec.api, command.arguments, command.options - command.spec.api(command.arguments; collect(command.options)...) + # Set the scoped value to indicate we're in REPL mode + Base.ScopedValues.@with IN_REPL_MODE => true begin + # REPL specific commands + command.spec === SPECS["package"]["help"] && return Base.invokelatest(do_help!, command, io) + # API commands + if command.spec.should_splat + TEST_MODE[] && return command.spec.api, command.arguments..., command.options + command.spec.api(command.arguments...; collect(command.options)...) # TODO is invokelatest still needed? + else + TEST_MODE[] && return command.spec.api, command.arguments, command.options + command.spec.api(command.arguments; collect(command.options)...) + end end end diff --git a/src/Registry/Registry.jl b/src/Registry/Registry.jl index 60916fd0f8..fcaf05ae24 100644 --- a/src/Registry/Registry.jl +++ b/src/Registry/Registry.jl @@ -161,10 +161,11 @@ function check_registry_state(reg) reg_currently_uses_pkg_server = reg.tree_info !== nothing reg_should_use_pkg_server = registry_use_pkg_server() if reg_currently_uses_pkg_server && !reg_should_use_pkg_server + pkg_cmd = Pkg.in_repl_mode() ? "pkg> registry rm $(reg.name); registry add $(reg.name)" : "using Pkg; Pkg.Registry.rm(\"$(reg.name)\"); Pkg.Registry.add(\"$(reg.name)\")" msg = string( "Your registry may be outdated. We recommend that you run the ", "following command: ", - "using Pkg; Pkg.Registry.rm(\"$(reg.name)\"); Pkg.Registry.add(\"$(reg.name)\")", + pkg_cmd, ) @warn(msg) end diff --git a/test/repl.jl b/test/repl.jl index 709fcc0b52..c6926eb77d 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -761,4 +761,30 @@ end end end +@testset "in_repl_mode" begin + # Test that in_repl_mode() returns false by default (API mode) + @test Pkg.in_repl_mode() == false + + # Test that in_repl_mode() returns true when running REPL commands + # This is tested indirectly by running a simple REPL command + temp_pkg_dir() do project_path + cd(project_path) do + # The pkg"" macro should set IN_REPL_MODE => true during execution + # We can't directly test the scoped value here, but we can test + # that REPL commands work correctly + pkg"status" + # The fact that this doesn't error confirms REPL mode is working + @test true + end + end + + # Test manual scoped value setting (for completeness) + Base.ScopedValues.@with Pkg.IN_REPL_MODE => true begin + @test Pkg.in_repl_mode() == true + end + + # Verify we're back to false after the scoped block + @test Pkg.in_repl_mode() == false +end + end # module From 232c62317e71c2754dafd547d21f294dc36dce7e Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 4 Jul 2025 10:34:03 +0200 Subject: [PATCH 110/154] Implement lazy loading for RegistryInstance to improve startup performance (#4304) Fix #4301 Co-authored-by: Claude --- src/Registry/registry_instance.jl | 88 +++++++++++++++++++++---------- 1 file changed, 59 insertions(+), 29 deletions(-) diff --git a/src/Registry/registry_instance.jl b/src/Registry/registry_instance.jl index 93990d2c8c..d697bb6f0a 100644 --- a/src/Registry/registry_instance.jl +++ b/src/Registry/registry_instance.jl @@ -271,21 +271,75 @@ function uncompress_registry(tar_gz::AbstractString) return data end -struct RegistryInstance +mutable struct RegistryInstance path::String + tree_info::Union{Base.SHA1, Nothing} + compressed_file::Union{String, Nothing} + + # Lazily loaded fields name::String uuid::UUID repo::Union{String, Nothing} description::Union{String, Nothing} pkgs::Dict{UUID, PkgEntry} - tree_info::Union{Base.SHA1, Nothing} in_memory_registry::Union{Nothing, Dict{String, String}} # various caches name_to_uuids::Dict{String, Vector{UUID}} + + # Inner constructor for lazy loading - leaves fields undefined + function RegistryInstance(path::String, tree_info::Union{Base.SHA1, Nothing}, compressed_file::Union{String, Nothing}) + new(path, tree_info, compressed_file) + end + + # Full constructor for when all fields are known + function RegistryInstance(path::String, tree_info::Union{Base.SHA1, Nothing}, compressed_file::Union{String, Nothing}, + name::String, uuid::UUID, repo::Union{String, Nothing}, description::Union{String, Nothing}, + pkgs::Dict{UUID, PkgEntry}, in_memory_registry::Union{Nothing, Dict{String, String}}, + name_to_uuids::Dict{String, Vector{UUID}}) + new(path, tree_info, compressed_file, name, uuid, repo, description, pkgs, in_memory_registry, name_to_uuids) + end end const REGISTRY_CACHE = Dict{String, Tuple{Base.SHA1, Bool, RegistryInstance}}() +@noinline function _ensure_registry_loaded_slow!(r::RegistryInstance) + isdefined(r, :pkgs) && return r + + if getfield(r, :compressed_file) !== nothing + r.in_memory_registry = uncompress_registry(joinpath(dirname(getfield(r, :path)), getfield(r, :compressed_file))) + else + r.in_memory_registry = nothing + end + + d = parsefile(r.in_memory_registry, getfield(r, :path), "Registry.toml") + r.name = d["name"]::String + r.uuid = UUID(d["uuid"]::String) + r.repo = get(d, "repo", nothing)::Union{String, Nothing} + r.description = get(d, "description", nothing)::Union{String, Nothing} + + r.pkgs = Dict{UUID, PkgEntry}() + for (uuid, info) in d["packages"]::Dict{String, Any} + uuid = UUID(uuid::String) + info::Dict{String, Any} + name = info["name"]::String + pkgpath = info["path"]::String + pkg = PkgEntry(pkgpath, getfield(r, :path), name, uuid, r.in_memory_registry) + r.pkgs[uuid] = pkg + end + + r.name_to_uuids = Dict{String, Vector{UUID}}() + + return r +end + +# Property accessors that trigger lazy loading +@inline function Base.getproperty(r::RegistryInstance, f::Symbol) + if f === :name || f === :uuid || f === :repo || f === :description || f === :pkgs || f === :name_to_uuids + isdefined(r, :pkgs) || _ensure_registry_loaded_slow!(r) + end + return getfield(r, f) +end + function get_cached_registry(path, tree_info::Base.SHA1, compressed::Bool) if !ispath(path) delete!(REGISTRY_CACHE, path) @@ -326,33 +380,9 @@ function RegistryInstance(path::AbstractString) end end - in_memory_registry = if compressed_file !== nothing - uncompress_registry(joinpath(dirname(path), compressed_file)) - else - nothing - end - - d = parsefile(in_memory_registry, path, "Registry.toml") - pkgs = Dict{UUID, PkgEntry}() - for (uuid, info) in d["packages"]::Dict{String, Any} - uuid = UUID(uuid::String) - info::Dict{String, Any} - name = info["name"]::String - pkgpath = info["path"]::String - pkg = PkgEntry(pkgpath, path, name, uuid, in_memory_registry) - pkgs[uuid] = pkg - end - reg = RegistryInstance( - path, - d["name"]::String, - UUID(d["uuid"]::String), - get(d, "repo", nothing)::Union{String, Nothing}, - get(d, "description", nothing)::Union{String, Nothing}, - pkgs, - tree_info, - in_memory_registry, - Dict{String, UUID}(), - ) + # Create partially initialized registry - defer expensive operations + reg = RegistryInstance(path, tree_info, compressed_file) + if tree_info !== nothing REGISTRY_CACHE[path] = (tree_info, compressed_file !== nothing, reg) end From e7a2dfecbfe43cf1c32f1ccd1e98a4dca52726ee Mon Sep 17 00:00:00 2001 From: Quan-feng WU <32387646+Fenyutanchan@users.noreply.github.com> Date: Fri, 4 Jul 2025 18:50:14 +0800 Subject: [PATCH 111/154] add support for `Pkg.add(path="/path/to/git-submodule.jl")` directly. (#3344) Co-authored-by: KristofferC --- src/Types.jl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Types.jl b/src/Types.jl index a9aba5b501..a7121d423e 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -874,7 +874,16 @@ function handle_repo_add!(ctx::Context, pkg::PackageSpec) repo_source = pkg.repo.source if !isurl(pkg.repo.source) if isdir(pkg.repo.source) - if !isdir(joinpath(pkg.repo.source, ".git")) + git_path = joinpath(pkg.repo.source, ".git") + if isfile(git_path) + # Git submodule: .git is a file containing path to actual git directory + git_ref_content = readline(git_path) + git_info_path = joinpath(dirname(git_path), last(split(git_ref_content))) + else + # Regular git repo: .git is a directory + git_info_path = git_path + end + if !isdir(git_info_path) msg = "Did not find a git repository at `$(pkg.repo.source)`" if isfile(joinpath(pkg.repo.source, "Project.toml")) || isfile(joinpath(pkg.repo.source, "JuliaProject.toml")) msg *= ", perhaps you meant `Pkg.develop`?" From 95e0ab7a8ffc6a51d5708cb0d00157ac80306b50 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 4 Jul 2025 14:21:03 +0200 Subject: [PATCH 112/154] Fix reference_manifest_isolated_test to use temporary directory (#4306) Co-authored-by: Claude Co-authored-by: KristofferC --- test/manifests.jl | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/test/manifests.jl b/test/manifests.jl index a1780673d1..78bca1feba 100644 --- a/test/manifests.jl +++ b/test/manifests.jl @@ -7,12 +7,20 @@ using ..Utils # used with the reference manifests in `test/manifest/formats` # ensures the manifests are valid and restored after test function reference_manifest_isolated_test(f, dir::String; v1::Bool=false) - env_dir = joinpath(@__DIR__, "manifest", "formats", dir) - env_manifest = joinpath(env_dir, "Manifest.toml") - env_project = joinpath(env_dir, "Project.toml") - cp(env_manifest, string(env_manifest, "_backup")) - cp(env_project, string(env_project, "_backup")) + source_env_dir = joinpath(@__DIR__, "manifest", "formats", dir) + source_env_manifest = joinpath(source_env_dir, "Manifest.toml") + source_env_project = joinpath(source_env_dir, "Project.toml") + + # Create a temporary directory for the test files + temp_base_dir = mktempdir() try + # Copy entire directory structure to preserve paths that tests expect + env_dir = joinpath(temp_base_dir, dir) + cp(source_env_dir, env_dir) + + env_manifest = joinpath(env_dir, "Manifest.toml") + env_project = joinpath(env_dir, "Project.toml") + isfile(env_manifest) || error("Reference manifest is missing") if Base.is_v1_format_manifest(Base.parsed_toml(env_manifest)) == !v1 error("Reference manifest file at $(env_manifest) is invalid") @@ -21,10 +29,8 @@ function reference_manifest_isolated_test(f, dir::String; v1::Bool=false) f(env_dir, env_manifest) end finally - cp(string(env_manifest, "_backup"), env_manifest, force = true) - rm(string(env_manifest, "_backup")) - cp(string(env_project, "_backup"), env_project, force = true) - rm(string(env_project, "_backup")) + # Clean up temporary directory + rm(temp_base_dir, recursive=true) end end From 06dc7eb2837cc101758a3d7ef224c853065f0feb Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 4 Jul 2025 14:33:01 +0200 Subject: [PATCH 113/154] prevent error when updating sources with a non-existing manifest (#4307) Co-authored-by: KristofferC --- src/Types.jl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/Types.jl b/src/Types.jl index a7121d423e..d0f8a4f451 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -1240,13 +1240,15 @@ function write_env(env::EnvCache; update_undo=true, @assert entry.repo.subdir == repo.subdir end end - if entry.path !== nothing - env.project.sources[pkg] = Dict("path" => entry.path) - elseif entry.repo != GitRepo() - d = Dict("url" => entry.repo.source) - entry.repo.rev !== nothing && (d["rev"] = entry.repo.rev) - entry.repo.subdir !== nothing && (d["subdir"] = entry.repo.subdir) - env.project.sources[pkg] = d + if entry !== nothing + if entry.path !== nothing + env.project.sources[pkg] = Dict("path" => entry.path) + elseif entry.repo != GitRepo() + d = Dict("url" => entry.repo.source) + entry.repo.rev !== nothing && (d["rev"] = entry.repo.rev) + entry.repo.subdir !== nothing && (d["subdir"] = entry.repo.subdir) + env.project.sources[pkg] = d + end end end From cb6099af80b09f31ee309c7fc19117346f40c50e Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 4 Jul 2025 15:43:10 +0300 Subject: [PATCH 114/154] Add more info when package is missing (#4303) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/Operations.jl | 20 +++++++++++++++++++- test/pkg.jl | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/Operations.jl b/src/Operations.jl index 0a24273d91..f3ba5e1297 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -417,7 +417,25 @@ function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UU path = project_rel_path(env, source_path(env.manifest_file, pkg)) end if !isdir(path) - pkgerror("expected package $(err_rep(pkg)) to exist at path `$path`") + # Find which packages depend on this missing package for better error reporting + dependents = String[] + for (dep_uuid, dep_entry) in env.manifest.deps + if pkg.uuid in values(dep_entry.deps) || pkg.uuid in values(dep_entry.weakdeps) + push!(dependents, dep_entry.name === nothing ? "unknown package [$dep_uuid]" : dep_entry.name) + end + end + + error_msg = "expected package $(err_rep(pkg)) to exist at path `$path`" + error_msg *= "\n\nThis package is referenced in the manifest file: $(env.manifest_file)" + + if !isempty(dependents) + if length(dependents) == 1 + error_msg *= "\nIt is required by: $(dependents[1])" + else + error_msg *= "\nIt is required by:\n$(join([" - $dep" for dep in dependents], "\n"))" + end + end + pkgerror(error_msg) end deps, weakdeps = collect_project(pkg, path) deps_map[pkg.uuid] = deps diff --git a/test/pkg.jl b/test/pkg.jl index f0e9950ce1..f18e8d1619 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -730,6 +730,39 @@ end @test sprint(showerror, err) == "foobar" end +@testset "issue #2191: better diagnostic for missing package" begin + temp_pkg_dir() do project_path; cd_tempdir() do tmpdir + Pkg.activate(".") + + # Create a package A that depends on package B + Pkg.generate("A") + Pkg.generate("B") + git_init_and_commit("A") + git_init_and_commit("B") + + # Add B as a dependency of A + cd("A") do + Pkg.develop(PackageSpec(path="../B")) + end + + # Now remove the B directory to simulate the missing package scenario + rm("B", recursive=true) + + # Try to perform an operation that would trigger the missing package error + cd("A") do + try + Pkg.resolve() + @test false # a PkgError should be thrown" + catch e + @test e isa PkgError + error_msg = sprint(showerror, e) + # Check that the improved error message contains helpful information + @test occursin("This package is referenced in the manifest file:", error_msg) + end + end + end end +end + @testset "issue #1066: package with colliding name/uuid exists in project" begin temp_pkg_dir() do project_path; cd_tempdir() do tmpdir Pkg.activate(".") From c1d5f73fc522255ca69da1783301a517cf274090 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 4 Jul 2025 14:49:51 +0200 Subject: [PATCH 115/154] fix resolve to not suceed when giving a non-existing version for a package (#4308) Co-authored-by: KristofferC --- src/Operations.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Operations.jl b/src/Operations.jl index f3ba5e1297..8dfece6c2b 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -552,7 +552,7 @@ function resolve_versions!(env::EnvCache, registries::Vector{Registry.RegistryIn # We only fixup a JLL if the old major/minor/patch matches the new major/minor/patch if old_v !== nothing && Base.thispatch(old_v) == Base.thispatch(vers_fix[uuid]) new_v = vers_fix[uuid] - if old_v != new_v + if old_v != new_v && haskey(compat_map[uuid], old_v) compat_map[uuid][old_v] = compat_map[uuid][new_v] # Note that we don't delete!(compat_map[uuid], old_v) because we want to keep the compat info around # in case there's JLL version confusion between the sysimage pkgorigins version and manifest @@ -587,6 +587,10 @@ function resolve_versions!(env::EnvCache, registries::Vector{Registry.RegistryIn deps_fixed else d = Dict{String, UUID}() + if !haskey(compat_map[pkg.uuid], pkg.version) + available_versions = sort!(collect(keys(compat_map[pkg.uuid]))) + pkgerror("version $(pkg.version) of package $(pkg.name) is not available. Available versions: $(join(available_versions, ", "))") + end for (uuid, _) in compat_map[pkg.uuid][pkg.version] d[names[uuid]] = uuid end From 115e303d8f8010d38bf2cf5340bc20ba72a52a3e Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 4 Jul 2025 18:41:56 +0300 Subject: [PATCH 116/154] add option for Pkg.build to allow_reresolve=false (#3329) --- CHANGELOG.md | 1 + src/API.jl | 4 +- src/Operations.jl | 8 +-- src/Pkg.jl | 12 +++- test/api.jl | 29 +++++++++ test/test_packages/.gitignore | 1 + .../AllowReresolveTest/Manifest.toml | 62 +++++++++++++++++++ .../AllowReresolveTest/Project.toml | 16 +++++ .../AllowReresolveTest/deps/build.jl | 3 + .../src/AllowReresolveTest.jl | 7 +++ .../AllowReresolveTest/test/runtests.jl | 6 ++ 11 files changed, 141 insertions(+), 8 deletions(-) create mode 100644 test/test_packages/AllowReresolveTest/Manifest.toml create mode 100644 test/test_packages/AllowReresolveTest/Project.toml create mode 100644 test/test_packages/AllowReresolveTest/deps/build.jl create mode 100644 test/test_packages/AllowReresolveTest/src/AllowReresolveTest.jl create mode 100644 test/test_packages/AllowReresolveTest/test/runtests.jl diff --git a/CHANGELOG.md b/CHANGELOG.md index a9ad60bf16..4151a9c7ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ Pkg v1.13 Release Notes ======================= - Project.toml environments now support a `readonly` field to mark environments as read-only, preventing modifications. ([#4284]) +- `Pkg.build` now supports an `allow_reresolve` keyword argument to control whether the build process can re-resolve package versions, similar to the existing option for `Pkg.test`. ([#3329]) - Packages are now automatically added to `[sources]` when they are added by url or devved. Pkg v1.12 Release Notes diff --git a/src/API.jl b/src/API.jl index 2c8f61ce23..52c4362bfe 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1118,7 +1118,7 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, return end -function build(ctx::Context, pkgs::Vector{PackageSpec}; verbose=false, kwargs...) +function build(ctx::Context, pkgs::Vector{PackageSpec}; verbose=false, allow_reresolve::Bool=true, kwargs...) Context!(ctx; kwargs...) if isempty(pkgs) @@ -1133,7 +1133,7 @@ function build(ctx::Context, pkgs::Vector{PackageSpec}; verbose=false, kwargs... project_resolve!(ctx.env, pkgs) manifest_resolve!(ctx.env.manifest, pkgs) ensure_resolved(ctx, ctx.env.manifest, pkgs) - Operations.build(ctx, Set{UUID}(pkg.uuid for pkg in pkgs), verbose) + Operations.build(ctx, Set{UUID}(pkg.uuid for pkg in pkgs), verbose; allow_reresolve) end function get_or_make_pkgspec(pkgspecs::Vector{PackageSpec}, ctx::Context, uuid) diff --git a/src/Operations.jl b/src/Operations.jl index 8dfece6c2b..b26aa7100d 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -1312,12 +1312,12 @@ function any_package_not_installed(manifest::Manifest) return false end -function build(ctx::Context, uuids::Set{UUID}, verbose::Bool) +function build(ctx::Context, uuids::Set{UUID}, verbose::Bool; allow_reresolve::Bool=true) if any_package_not_installed(ctx.env.manifest) || !isfile(ctx.env.manifest_file) Pkg.instantiate(ctx, allow_build = false, allow_autoprecomp = false) end all_uuids = get_deps(ctx.env, uuids) - build_versions(ctx, all_uuids; verbose) + build_versions(ctx, all_uuids; verbose, allow_reresolve) end function dependency_order_uuids(env::EnvCache, uuids::Vector{UUID})::Dict{UUID,Int} @@ -1377,7 +1377,7 @@ pkg_scratchpath() = joinpath(depots1(), "scratchspaces", PkgUUID) builddir(source_path::String) = joinpath(source_path, "deps") buildfile(source_path::String) = joinpath(builddir(source_path), "build.jl") -function build_versions(ctx::Context, uuids::Set{UUID}; verbose=false) +function build_versions(ctx::Context, uuids::Set{UUID}; verbose=false, allow_reresolve::Bool=true) # collect builds for UUIDs with `deps/build.jl` files builds = Tuple{UUID,String,String,VersionNumber}[] for uuid in uuids @@ -1459,7 +1459,7 @@ function build_versions(ctx::Context, uuids::Set{UUID}; verbose=false) fancyprint && show_progress(ctx.io, bar) let log_file=log_file - sandbox(ctx, pkg, builddir(source_path), build_project_override; preferences=build_project_preferences) do + sandbox(ctx, pkg, builddir(source_path), build_project_override; preferences=build_project_preferences, allow_reresolve) do flush(ctx.io) ok = open(log_file, "w") do log std = verbose ? ctx.io : log diff --git a/src/Pkg.jl b/src/Pkg.jl index 83fde0c646..488693073d 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -366,14 +366,22 @@ const gc = API.gc Pkg.build(pkg::Union{String, Vector{String}}; verbose = false, io::IO=stderr) Pkg.build(pkgs::Union{PackageSpec, Vector{PackageSpec}}; verbose = false, io::IO=stderr) +**Keyword arguments:** + - `verbose::Bool=false`: print the build output to `stdout`/`stderr` instead of redirecting to the `build.log` file. + - `allow_reresolve::Bool=true`: allow Pkg to reresolve the package versions in the build environment + +!!! compat "Julia 1.13" + `allow_reresolve` requires at least Julia 1.13. + Run the build script in `deps/build.jl` for `pkg` and all of its dependencies in depth-first recursive order. If no argument is given to `build`, the current project is built, which thus needs to be a package. This function is called automatically on any package that gets installed for the first time. -`verbose = true` prints the build output to `stdout`/`stderr` instead of -redirecting to the `build.log` file. + +The build takes place in a new process matching the current process with default of `startup-file=no`. +If using the startup file (`~/.julia/config/startup.jl`) is desired, start julia with an explicit `--startup-file=yes`. """ const build = API.build diff --git a/test/api.jl b/test/api.jl index 242b5af2e5..0441f3f020 100644 --- a/test/api.jl +++ b/test/api.jl @@ -313,4 +313,33 @@ end end end end +@testset "allow_reresolve parameter" begin + isolate(loaded_depot=false) do; mktempdir() do tempdir + Pkg.Registry.add(url = "https://github.com/JuliaRegistries/Test") + # AllowReresolveTest has Example v0.5.1 which is yanked in the test registry. + test_dir = joinpath(tempdir, "AllowReresolveTest") + + # Test that we can build and test with allow_reresolve=true + copy_test_package(tempdir, "AllowReresolveTest") + Pkg.activate(joinpath(tempdir, "AllowReresolveTest")) + @test Pkg.build(; allow_reresolve=true) == nothing + + rm(test_dir, force=true, recursive=true) + copy_test_package(tempdir, "AllowReresolveTest") + Pkg.activate(joinpath(tempdir, "AllowReresolveTest")) + @test Pkg.test(; allow_reresolve=true) == nothing + + # Test that allow_reresolve=false fails with the broken manifest + rm(test_dir, force=true, recursive=true) + copy_test_package(tempdir, "AllowReresolveTest") + Pkg.activate(joinpath(tempdir, "AllowReresolveTest")) + @test_throws Pkg.Resolve.ResolverError Pkg.build(; allow_reresolve=false) + + rm(test_dir, force=true, recursive=true) + copy_test_package(tempdir, "AllowReresolveTest") + Pkg.activate(joinpath(tempdir, "AllowReresolveTest")) + @test_throws Pkg.Resolve.ResolverError Pkg.test(; allow_reresolve=false) + end end +end + end # module APITests diff --git a/test/test_packages/.gitignore b/test/test_packages/.gitignore index ba39cc531e..3d68ab37d6 100644 --- a/test/test_packages/.gitignore +++ b/test/test_packages/.gitignore @@ -1 +1,2 @@ Manifest.toml +!AllowReresolveTest/Manifest.toml diff --git a/test/test_packages/AllowReresolveTest/Manifest.toml b/test/test_packages/AllowReresolveTest/Manifest.toml new file mode 100644 index 0000000000..518f8d04bc --- /dev/null +++ b/test/test_packages/AllowReresolveTest/Manifest.toml @@ -0,0 +1,62 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.13.0-DEV" +manifest_format = "2.0" +project_hash = "a100b4eee2a8dd47230a6724ae4de850bddbb7a5" + +[[deps.AllowReresolveTest]] +deps = ["Example"] +path = "." +uuid = "12345678-1234-1234-1234-123456789abc" +version = "0.1.0" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.Example]] +deps = ["Test"] +git-tree-sha1 = "8eb7b4d4ca487caade9ba3e85932e28ce6d6e1f8" +uuid = "7876af07-990d-54b4-ab0e-23690620f79a" +version = "0.5.1" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.JuliaSyntaxHighlighting]] +deps = ["StyledStrings"] +uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" +version = "1.12.0" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + +[[deps.Markdown]] +deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" diff --git a/test/test_packages/AllowReresolveTest/Project.toml b/test/test_packages/AllowReresolveTest/Project.toml new file mode 100644 index 0000000000..643237b7b5 --- /dev/null +++ b/test/test_packages/AllowReresolveTest/Project.toml @@ -0,0 +1,16 @@ +name = "AllowReresolveTest" +uuid = "12345678-1234-1234-1234-123456789abc" +version = "0.1.0" + +[deps] +Example = "7876af07-990d-54b4-ab0e-23690620f79a" + +[compat] +Example = "0.5" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] +build = ["Test"] diff --git a/test/test_packages/AllowReresolveTest/deps/build.jl b/test/test_packages/AllowReresolveTest/deps/build.jl new file mode 100644 index 0000000000..28e53db871 --- /dev/null +++ b/test/test_packages/AllowReresolveTest/deps/build.jl @@ -0,0 +1,3 @@ +# Build script for AllowReresolveTest +using Test +println("Build completed successfully!") diff --git a/test/test_packages/AllowReresolveTest/src/AllowReresolveTest.jl b/test/test_packages/AllowReresolveTest/src/AllowReresolveTest.jl new file mode 100644 index 0000000000..e549c3a22e --- /dev/null +++ b/test/test_packages/AllowReresolveTest/src/AllowReresolveTest.jl @@ -0,0 +1,7 @@ +module AllowReresolveTest + +import Example + +greet() = "Hello from AllowReresolveTest using Example!" + +end diff --git a/test/test_packages/AllowReresolveTest/test/runtests.jl b/test/test_packages/AllowReresolveTest/test/runtests.jl new file mode 100644 index 0000000000..a1c953c162 --- /dev/null +++ b/test/test_packages/AllowReresolveTest/test/runtests.jl @@ -0,0 +1,6 @@ +using Test +using AllowReresolveTest + +@testset "AllowReresolveTest.jl" begin + @test AllowReresolveTest.greet() == "Hello from AllowReresolveTest using Example!" +end From 4ef5a5d4bba6d5e6439856cebe6ab81446116e46 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 4 Jul 2025 20:21:17 +0300 Subject: [PATCH 117/154] Add a tip when trying to upgrade a specific package that's possible to upgrade, just not optimal (#4266) --- CHANGELOG.md | 3 ++- src/Operations.jl | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4151a9c7ef..8191c2c271 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Pkg v1.13 Release Notes - Project.toml environments now support a `readonly` field to mark environments as read-only, preventing modifications. ([#4284]) - `Pkg.build` now supports an `allow_reresolve` keyword argument to control whether the build process can re-resolve package versions, similar to the existing option for `Pkg.test`. ([#3329]) - Packages are now automatically added to `[sources]` when they are added by url or devved. +- `update` now shows a helpful tip when trying to upgrade a specific package that can be upgraded but is held back because it's part of a less optimal resolver solution ([#4266]) Pkg v1.12 Release Notes ======================= @@ -14,7 +15,7 @@ Pkg v1.12 Release Notes - Pkg now supports "apps" which are Julia packages that can be run directly from the terminal after installation. Apps can be defined in a package's Project.toml and installed via Pkg. ([#3772]) - `status` now shows when different versions/sources of dependencies are loaded than that which is expected by the manifest ([#4109]) -- When adding or developing a package that exists in the `[weakdeps]` section, it is now automatically removed from +- When adding or developing a package that exists in the `[weakdeps]` section, it is now automatically removed from weak dependencies and added as a regular dependency. ([#3865]) - Enhanced fuzzy matching algorithm for package name suggestions. diff --git a/src/Operations.jl b/src/Operations.jl index b26aa7100d..309de62867 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -1922,6 +1922,9 @@ end function up(ctx::Context, pkgs::Vector{PackageSpec}, level::UpgradeLevel; skip_writing_project::Bool=false, preserve::Union{Nothing,PreserveLevel}=nothing) + + requested_pkgs = pkgs + new_git = Set{UUID}() # TODO check all pkg.version == VersionSpec() # set version constraints according to `level` @@ -1949,6 +1952,31 @@ function up(ctx::Context, pkgs::Vector{PackageSpec}, level::UpgradeLevel; download_artifacts(ctx, julia_version=ctx.julia_version) write_env(ctx.env; skip_writing_project) # write env before building show_update(ctx.env, ctx.registries; io=ctx.io, hidden_upgrades_info = true) + + if length(requested_pkgs) == 1 + pkg = only(requested_pkgs) + entry = manifest_info(ctx.env.manifest, pkg.uuid) + if entry === nothing || (entry.path === nothing && entry.repo.source === nothing) + # Get current version after the update + current_version = entry !== nothing ? entry.version : nothing + original_entry = manifest_info(ctx.env.original_manifest, pkg.uuid) + original_version = original_entry !== nothing ? original_entry.version : nothing + + # Check if version didn't change and there's a newer version available + if current_version == original_version && current_version !== nothing + temp_pkg = PackageSpec(name=pkg.name, uuid=pkg.uuid, version=current_version) + cinfo = status_compat_info(temp_pkg, ctx.env, ctx.registries) + if cinfo !== nothing + packages_holding_back, max_version, max_version_compat = cinfo + if current_version < max_version + printpkgstyle(ctx.io, :Info, "$(pkg.name) can be updated but at the cost of downgrading other packages. " * + "To force upgrade to the latest version, try `add $(pkg.name)@$(max_version)`", color=Base.info_color()) + end + end + end + end + end + build_versions(ctx, union(new_apply, new_git)) end @@ -2801,8 +2829,8 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie pkg_downloaded = !is_instantiated(new) || is_package_downloaded(env.manifest_file, new) new_ver_avail = !latest_version && !Operations.is_tracking_repo(new) && !Operations.is_tracking_path(new) - pkg_upgradable = new_ver_avail && isempty(cinfo[1]) - pkg_heldback = new_ver_avail && !isempty(cinfo[1]) + pkg_upgradable = new_ver_avail && cinfo !== nothing && isempty(cinfo[1]) + pkg_heldback = new_ver_avail && cinfo !== nothing && !isempty(cinfo[1]) if !pkg_downloaded && (pkg_upgradable || pkg_heldback) # allow space in the gutter for two icons on a single line From 94d9551cf9a01a397101296fa82873e3c74a78ef Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sat, 5 Jul 2025 09:12:24 +0200 Subject: [PATCH 118/154] Fix parsing to better handle branch/tag/subdir specifiers in complex URLs and paths (#4299) Co-authored-by: Claude Co-authored-by: KristofferC --- src/REPLMode/REPLMode.jl | 2 +- src/REPLMode/argument_parsers.jl | 382 ++++++++++++++++++++++++++----- src/utils.jl | 5 - test/new.jl | 223 +++++++++++++++++- test/repl.jl | 13 +- test/subdir.jl | 8 +- 6 files changed, 549 insertions(+), 84 deletions(-) diff --git a/src/REPLMode/REPLMode.jl b/src/REPLMode/REPLMode.jl index 030bc4dba0..40e6646e4f 100644 --- a/src/REPLMode/REPLMode.jl +++ b/src/REPLMode/REPLMode.jl @@ -6,7 +6,7 @@ module REPLMode using Markdown, UUIDs, Dates -import ..casesensitive_isdir, ..OFFLINE_MODE, ..linewrap, ..pathrepr, ..IN_REPL_MODE +import ..OFFLINE_MODE, ..linewrap, ..pathrepr, ..IN_REPL_MODE using ..Types, ..Operations, ..API, ..Registry, ..Resolve, ..Apps import ..stdout_f, ..stderr_f diff --git a/src/REPLMode/argument_parsers.jl b/src/REPLMode/argument_parsers.jl index 0901b58a1c..a5e68df298 100644 --- a/src/REPLMode/argument_parsers.jl +++ b/src/REPLMode/argument_parsers.jl @@ -1,4 +1,5 @@ import ..isdir_nothrow, ..Registry.RegistrySpec, ..isurl +using UUIDs struct PackageIdentifier val::String @@ -21,75 +22,336 @@ const PackageToken = Union{PackageIdentifier, Rev, Subdir} -packagetoken(word::String)::PackageToken = - first(word) == '@' ? VersionToken(word[2:end]) : - first(word) == '#' ? Rev(word[2:end]) : - first(word) == ':' ? Subdir(word[2:end]) : - PackageIdentifier(word) - -############### -# PackageSpec # -############### -""" -Parser for PackageSpec objects. -""" -function parse_package(args::Vector{QString}, options; add_or_dev=false)::Vector{PackageSpec} - words′ = package_lex(args) - words = String[] - for word in words′ - if (m = match(r"https://github.com/(.*?)/(.*?)/(?:tree|commit)/(.*?)$", word)) !== nothing - push!(words, "https://github.com/$(m.captures[1])/$(m.captures[2])") - push!(words, "#$(m.captures[3])") - else - push!(words, word) +# Check if a string is a valid UUID +function is_valid_uuid(str::String) + try + UUID(str) + return true + catch + return false + end +end + +# Simple URL detection +function looks_like_url(str::String) + return startswith(str, "http://") || startswith(str, "https://") || + startswith(str, "git@") || startswith(str, "ssh://") || + contains(str, ".git") +end + +# Simple path detection +function looks_like_path(str::String) + return contains(str, '/') || contains(str, '\\') || str == "." || str == ".." || + (length(str) >= 2 && isletter(str[1]) && str[2] == ':') # Windows drive letters +end + +# Check if a string looks like a complete URL +function looks_like_complete_url(str::String) + return (startswith(str, "http://") || startswith(str, "https://") || + startswith(str, "git@") || startswith(str, "ssh://")) && + (contains(str, '.') || contains(str, '/')) +end + +# Check if a colon at given position is part of a Windows drive letter +function is_windows_drive_colon(input::String, colon_pos::Int) + # Windows drive letters are single letters followed by colon at beginning + # Examples: "C:", "D:", etc. + if colon_pos == 2 && length(input) >= 2 + first_char = input[1] + return isletter(first_char) && input[2] == ':' + end + return false +end + +# Extract subdir specifier from the end of input (rightmost : that's not a Windows drive letter) +function extract_subdir(input::String) + colon_pos = findlast(':', input) + if colon_pos === nothing + return input, nothing + end + + # Skip Windows drive letters (e.g., C:, D:) + if is_windows_drive_colon(input, colon_pos) + return input, nothing + end + + subdir_part = input[nextind(input, colon_pos):end] + remaining = input[1:prevind(input, colon_pos)] + return remaining, subdir_part +end + +# Extract revision specifier from input (first # that separates base from revision) +function extract_revision(input::String) + hash_pos = findfirst('#', input) + if hash_pos === nothing + return input, nothing + end + + rev_part = input[nextind(input, hash_pos):end] + remaining = input[1:prevind(input, hash_pos)] + return remaining, rev_part +end + +# Extract version specifier from the end of input (rightmost @) +function extract_version(input::String) + at_pos = findlast('@', input) + if at_pos === nothing + return input, nothing + end + + version_part = input[nextind(input, at_pos):end] + remaining = input[1:prevind(input, at_pos)] + return remaining, version_part +end + +# Handle GitHub tree/commit URLs by converting them to standard URL + rev format +function preprocess_github_tree_commit_url(input::String) + m = match(r"https://github.com/(.*?)/(.*?)/(?:tree|commit)/(.*?)$", input) + if m !== nothing + base_url = "https://github.com/$(m.captures[1])/$(m.captures[2])" + rev = m.captures[3] + return [PackageIdentifier(base_url), Rev(rev)] + end + return nothing +end + +# Check if a colon in a URL string is part of URL structure (not a subdir separator) +function is_url_structure_colon(input::String, colon_pos::Int) + after_colon = input[nextind(input, colon_pos):end] + + # Check for git@host:path syntax + if startswith(input, "git@") + at_pos = findfirst('@', input) + if at_pos !== nothing + between_at_colon = input[nextind(input, at_pos):prevind(input, colon_pos)] + if !contains(between_at_colon, '/') + return true + end + end + end + + # Check for protocol:// syntax + if colon_pos <= lastindex(input) - 2 + next_pos = nextind(input, colon_pos) + if next_pos <= lastindex(input) - 1 && + input[colon_pos:nextind(input, nextind(input, colon_pos))] == "://" + return true end end - args = PackageToken[packagetoken(pkgword) for pkgword in words] - return parse_package_args(args; add_or_dev=add_or_dev) + # Check for user:password@ syntax (: followed by text then @) + if contains(after_colon, '@') + at_in_after = findfirst('@', after_colon) + if at_in_after !== nothing + text_before_at = after_colon[1:prevind(after_colon, at_in_after)] + if !contains(text_before_at, '/') + return true + end + end + end + + # Check for port numbers (: followed by digits then /) + if occursin(r"^\d+(/|$)", after_colon) + return true + end + + return false +end + +# Extract subdir from URL, being careful about URL structure +function extract_url_subdir(input::String) + colon_pos = findlast(':', input) + if colon_pos === nothing + return input, nothing + end + + # Check if this colon is part of URL structure + if is_url_structure_colon(input, colon_pos) + return input, nothing + end + + after_colon = input[nextind(input, colon_pos):end] + before_colon = input[1:prevind(input, colon_pos)] + + # Only treat as subdir if it looks like one and the part before looks like a URL + if (contains(after_colon, '/') || (!contains(after_colon, '@') && !contains(after_colon, '#'))) && + (contains(before_colon, "://") || contains(before_colon, ".git") || contains(before_colon, '@')) + return before_colon, after_colon + end + + return input, nothing +end + +# Extract revision from URL, only after a complete URL +function extract_url_revision(input::String) + hash_pos = findfirst('#', input) + if hash_pos === nothing + return input, nothing + end + + before_hash = input[1:prevind(input, hash_pos)] + after_hash = input[nextind(input, hash_pos):end] + + if looks_like_complete_url(before_hash) + return before_hash, after_hash + end + + return input, nothing +end + +# Parse URLs with specifiers +# URLs can only have revisions (#) and subdirs (:), NOT versions (@) +function parse_url_with_specifiers(input::String) + tokens = PackageToken[] + remaining = input + + # Extract subdir if present (rightmost : that looks like a subdir) + remaining, subdir_part = extract_url_subdir(remaining) + + # Extract revision (first # that comes after a complete URL) + remaining, rev_part = extract_url_revision(remaining) + + # What's left is the base URL + push!(tokens, PackageIdentifier(remaining)) + + # Add the specifiers in the correct order + if rev_part !== nothing + push!(tokens, Rev(rev_part)) + end + if subdir_part !== nothing + push!(tokens, Subdir(subdir_part)) + end + + return tokens end - # Match a git repository URL. This includes uses of `@` and `:` but - # requires that it has `.git` at the end. -let url = raw"((git|ssh|http(s)?)|(git@[\w\-\.]+))(:(//)?)([\w\.@\:/\-~]+)(\.git$)(/)?", +function parse_path_with_specifiers(input::String) + tokens = PackageToken[] + remaining = input + + # Extract subdir if present (rightmost :) + remaining, subdir_part = extract_subdir(remaining) - # Match a `NAME=UUID` package specifier. - name_uuid = raw"[^@\#\s:]+\s*=\s*[^@\#\s:]+", + # Extract revision if present (rightmost #) + remaining, rev_part = extract_revision(remaining) + + # What's left is the base path + push!(tokens, PackageIdentifier(remaining)) + + # Add specifiers in correct order + if rev_part !== nothing + push!(tokens, Rev(rev_part)) + end + if subdir_part !== nothing + push!(tokens, Subdir(subdir_part)) + end + + return tokens +end + +# Parse package names with specifiers +function parse_name_with_specifiers(input::String) + tokens = PackageToken[] + remaining = input + + # Extract subdir if present (rightmost :) + remaining, subdir_part = extract_subdir(remaining) + + # Extract version if present (rightmost @) + remaining, version_part = extract_version(remaining) + + # Extract revision if present (rightmost #) + remaining, rev_part = extract_revision(remaining) + + # What's left is the base name + push!(tokens, PackageIdentifier(remaining)) + + # Add specifiers in correct order + if rev_part !== nothing + push!(tokens, Rev(rev_part)) + end + if version_part !== nothing + push!(tokens, VersionToken(version_part)) + end + if subdir_part !== nothing + push!(tokens, Subdir(subdir_part)) + end - # Match a `#BRANCH` branch or tag specifier. - branch = raw"\#\s*[^@^:\s]+", + return tokens +end - # Match an `@VERSION` version specifier. - version = raw"@\s*[^@\#\s]*", +# Parse a single package specification +function parse_package_spec_new(input::String) + # Handle quoted strings + if (startswith(input, '"') && endswith(input, '"')) || + (startswith(input, '\'') && endswith(input, '\'')) + input = input[2:end-1] + end - # Match a `:SUBDIR` subdir specifier. - subdir = raw":[^@\#\s]+", + # Handle GitHub tree/commit URLs first (special case) + github_result = preprocess_github_tree_commit_url(input) + if github_result !== nothing + return github_result + end - # Match any other way to specify a package. This includes package - # names, local paths, and URLs that don't match the `url` part. In - # order not to clash with the branch, version, and subdir - # specifiers, these cannot include `@` or `#`, and `:` is only - # allowed if followed by `/` or `\`. For URLs matching this part - # of the regex, that means that `@` (e.g. user names) and `:` - # (e.g. port) cannot be used but it doesn't have to end with - # `.git`. - other = raw"([^@\#\s:] | :(/|\\))+" + # Handle name=uuid format + if contains(input, '=') + parts = split(input, '=', limit=2) + if length(parts) == 2 + name = String(strip(parts[1])) + uuid_str = String(strip(parts[2])) + if is_valid_uuid(uuid_str) + return [PackageIdentifier("$name=$uuid_str")] + end + end + end - # Combine all of the above. - global const package_id_re = Regex( - "$url | $name_uuid | $branch | $version | $subdir | $other", "x") + # Check what type of input this is and parse accordingly + if looks_like_url(input) + return parse_url_with_specifiers(input) + elseif looks_like_path(input) + return parse_path_with_specifiers(input) + else + return parse_name_with_specifiers(input) + end end -function package_lex(qwords::Vector{QString})::Vector{String} - words = String[] - for qword in qwords - qword.isquoted ? - push!(words, qword.raw) : - append!(words, map(m->m.match, eachmatch(package_id_re, qword.raw))) +function parse_package(args::Vector{QString}, options; add_or_dev=false)::Vector{PackageSpec} + tokens = PackageToken[] + + i = 1 + while i <= length(args) + arg = args[i] + input = arg.isquoted ? arg.raw : arg.raw + + # Check if this argument is a standalone modifier (like #dev, @v1.0, :subdir) + if !arg.isquoted && (startswith(input, '#') || startswith(input, '@') || startswith(input, ':')) + # This is a standalone modifier - it should be treated as a token + if startswith(input, '#') + push!(tokens, Rev(input[2:end])) + elseif startswith(input, '@') + push!(tokens, VersionToken(input[2:end])) + elseif startswith(input, ':') + push!(tokens, Subdir(input[2:end])) + end + else + # Parse this argument normally + if arg.isquoted + # For quoted arguments, treat as literal without specifier extraction + arg_tokens = [PackageIdentifier(input)] + else + arg_tokens = parse_package_spec_new(input) + end + append!(tokens, arg_tokens) + end + + i += 1 end - return words + + return parse_package_args(tokens; add_or_dev=add_or_dev) end + function parse_package_args(args::Vector{PackageToken}; add_or_dev=false)::Vector{PackageSpec} # check for and apply PackageSpec modifier (e.g. `#foo` or `@v1.0.2`) function apply_modifier!(pkg::PackageSpec, args::Vector{PackageToken}) @@ -155,17 +417,13 @@ end function parse_package_identifier(pkg_id::PackageIdentifier; add_or_develop=false)::PackageSpec word = pkg_id.val if add_or_develop + if occursin(name_re, word) && isdir(expanduser(word)) + @info "Use `./$word` to add or develop the local directory at `$(Base.contractuser(abspath(word)))`." + end if isurl(word) return PackageSpec(; url=word) elseif any(occursin.(['\\','/'], word)) || word == "." || word == ".." - if casesensitive_isdir(expanduser(word)) - return PackageSpec(; path=normpath(expanduser(word))) - else - pkgerror("`$word` appears to be a local path, but directory does not exist") - end - end - if occursin(name_re, word) && casesensitive_isdir(expanduser(word)) - @info "Use `./$word` to add or develop the local directory at `$(Base.contractuser(abspath(word)))`." + return PackageSpec(; path=normpath(expanduser(word))) end end if occursin(uuid_re, word) @@ -195,7 +453,7 @@ end function parse_registry(word::AbstractString; add=false)::RegistrySpec word = expanduser(word) registry = RegistrySpec() - if add && isdir_nothrow(word) # TODO: Should be casesensitive_isdir + if add && isdir_nothrow(word) if isdir(joinpath(word, ".git")) # add path as url and clone it from there registry.url = abspath(word) else # put the path diff --git a/src/utils.jl b/src/utils.jl index 62f6da6be4..2e40e98ab6 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -146,11 +146,6 @@ function isfile_nothrow(path::String) end end -function casesensitive_isdir(dir::String) - dir = abspath(dir) - lastdir = splitpath(dir)[end] - isdir_nothrow(dir) && lastdir in readdir(joinpath(dir, "..")) -end """ atomic_toml_write(path::String, data; kws...) diff --git a/test/new.jl b/test/new.jl index 76983c5b43..bde938f347 100644 --- a/test/new.jl +++ b/test/new.jl @@ -1149,6 +1149,226 @@ end @test api == Pkg.add @test args == [Pkg.PackageSpec(;url="https://github.com/00vareladavid/Unregistered.jl", rev="0.1.0")] @test isempty(opts) + + api, args, opts = first(Pkg.pkg"add a/path/with/@/deal/with/it") + @test normpath(args[1].path) == normpath("a/path/with/@/deal/with/it") + + # Test GitHub URLs with tree/commit paths + @testset "GitHub tree/commit URLs" begin + api, args, opts = first(Pkg.pkg"add https://github.com/user/repo/tree/feature-branch") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "https://github.com/user/repo" + @test args[1].rev == "feature-branch" + + api, args, opts = first(Pkg.pkg"add https://github.com/user/repo/commit/abc123def") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "https://github.com/user/repo" + @test args[1].rev == "abc123def" + end + + # Test Git URLs with branch specifiers + @testset "Git URLs with branch specifiers" begin + api, args, opts = first(Pkg.pkg"add https://github.com/user/repo.git#main") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "https://github.com/user/repo.git" + @test args[1].rev == "main" + + api, args, opts = first(Pkg.pkg"add https://bitbucket.org/user/repo.git#develop") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "https://bitbucket.org/user/repo.git" + @test args[1].rev == "develop" + + api, args, opts = first(Pkg.pkg"add git@github.com:user/repo.git#feature") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "git@github.com:user/repo.git" + @test args[1].rev == "feature" + + api, args, opts = first(Pkg.pkg"add ssh://git@server.com/path/repo.git#branch-name") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "ssh://git@server.com/path/repo.git" + @test args[1].rev == "branch-name" + end + + + # Test Git URLs with subdir specifiers + @testset "Git URLs with subdir specifiers" begin + api, args, opts = first(Pkg.pkg"add https://github.com/user/monorepo.git:packages/MyPackage") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "https://github.com/user/monorepo.git" + @test args[1].subdir == "packages/MyPackage" + + api, args, opts = first(Pkg.pkg"add ssh://git@server.com/repo.git:subdir/nested") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "ssh://git@server.com/repo.git" + @test args[1].subdir == "subdir/nested" + end + + # Test complex URLs (with username in URL + branch/tag/subdir) + @testset "Complex Git URLs" begin + api, args, opts = first(Pkg.pkg"add https://username@bitbucket.org/org/repo.git#dev") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "https://username@bitbucket.org/org/repo.git" + @test args[1].rev == "dev" + + api, args, opts = first(Pkg.pkg"add https://user:token@gitlab.company.com/group/project.git") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "https://user:token@gitlab.company.com/group/project.git" + + api, args, opts = first(Pkg.pkg"add https://example.com:8080/git/repo.git:packages/core") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "https://example.com:8080/git/repo.git" + @test args[1].subdir == "packages/core" + + # Test URLs with complex authentication and branch names containing # + api, args, opts = first(Pkg.pkg"add https://user:pass123@gitlab.example.com:8443/group/project.git#feature/fix-#42") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "https://user:pass123@gitlab.example.com:8443/group/project.git" + @test args[1].rev == "feature/fix-#42" + + # Test URLs with complex authentication and subdirs + api, args, opts = first(Pkg.pkg"add https://api_key:secret@company.git.server.com/team/monorepo.git:libs/julia/pkg") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "https://api_key:secret@company.git.server.com/team/monorepo.git" + @test args[1].subdir == "libs/julia/pkg" + + # Test URLs with authentication, branch with #, and subdir + api, args, opts = first(Pkg.pkg"add https://deploy:token123@internal.git.company.com/product/backend.git#hotfix/issue-#789:packages/core") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "https://deploy:token123@internal.git.company.com/product/backend.git" + @test args[1].rev == "hotfix/issue-#789" + @test args[1].subdir == "packages/core" + + # Test SSH URLs with port numbers and subdirs + api, args, opts = first(Pkg.pkg"add ssh://git@custom.server.com:2222/path/to/repo.git:src/package") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "ssh://git@custom.server.com:2222/path/to/repo.git" + @test args[1].subdir == "src/package" + + # Test URL with username in URL and multiple # in branch name + api, args, opts = first(Pkg.pkg"add https://ci_user@build.company.net/team/project.git#release/v2.0-#123-#456") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "https://ci_user@build.company.net/team/project.git" + @test args[1].rev == "release/v2.0-#123-#456" + + # Test complex case: auth + port + branch with # + subdir + api, args, opts = first(Pkg.pkg"add https://robot:abc123@git.enterprise.com:9443/division/platform.git#bugfix/handle-#special-chars:modules/julia-pkg") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "https://robot:abc123@git.enterprise.com:9443/division/platform.git" + @test args[1].rev == "bugfix/handle-#special-chars" + @test args[1].subdir == "modules/julia-pkg" + + # Test local paths with branch specifiers (paths can be repos) + api, args, opts = first(Pkg.pkg"add ./local/repo#feature-branch") + @test api == Pkg.add + @test length(args) == 1 + @test normpath(args[1].path) == normpath("local/repo") # normpath removes "./" + @test args[1].rev == "feature-branch" + + # Test local paths with subdir specifiers + api, args, opts = first(Pkg.pkg"add ./monorepo:packages/subpkg") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].path == "monorepo" # normpath removes "./" + @test args[1].subdir == "packages/subpkg" + + # Test local paths with both branch and subdir + api, args, opts = first(Pkg.pkg"add ./project#develop:src/package") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].path == "project" # normpath removes "./" + @test args[1].rev == "develop" + @test args[1].subdir == "src/package" + + # Test local paths with branch containing # characters + api, args, opts = first(Pkg.pkg"add ../workspace/repo#bugfix/issue-#123") + @test api == Pkg.add + @test length(args) == 1 + @test normpath(args[1].path) == normpath("../workspace/repo") + @test args[1].rev == "bugfix/issue-#123" + + # Test complex local path case: relative path + branch with # + subdir + if !Sys.iswindows() + api, args, opts = first(Pkg.pkg"add ~/projects/myrepo#feature/fix-#456:libs/core") + @test api == Pkg.add + @test length(args) == 1 + @test startswith(args[1].path, "/") # ~ gets expanded to absolute path + @test endswith(normpath(args[1].path), normpath("/projects/myrepo")) + @test args[1].rev == "feature/fix-#456" + @test args[1].subdir == "libs/core" + end + + # Test quoted URL with separate revision specifier (regression test) + api, args, opts = first(Pkg.pkg"add \"https://username@bitbucket.org/orgname/reponame.git\"#dev") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "https://username@bitbucket.org/orgname/reponame.git" + @test args[1].rev == "dev" + + # Test quoted URL with separate version specifier + api, args, opts = first(Pkg.pkg"add \"https://company.git.server.com/project.git\"@v2.1.0") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "https://company.git.server.com/project.git" + @test args[1].version == "v2.1.0" + + # Test quoted URL with separate subdir specifier + api, args, opts = first(Pkg.pkg"add \"https://gitlab.example.com/monorepo.git\":packages/core") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "https://gitlab.example.com/monorepo.git" + @test args[1].subdir == "packages/core" + end + + # Test that regular URLs without .git still work + @testset "Non-.git URLs (unchanged behavior)" begin + api, args, opts = first(Pkg.pkg"add https://github.com/user/repo") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].url == "https://github.com/user/repo" + @test args[1].rev === nothing + @test args[1].subdir === nothing + end + + @testset "Windows path handling" begin + # Test that Windows drive letters are not treated as subdir separators + api, args, opts = first(Pkg.pkg"add C:\\Users\\test\\project") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].path == normpath("C:\\\\Users\\\\test\\\\project") + @test args[1].subdir === nothing + + # Test with forward slashes too + api, args, opts = first(Pkg.pkg"add C:/Users/test/project") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].path == normpath("C:/Users/test/project") + @test args[1].subdir === nothing + + # Test that actual subdir syntax still works with Windows paths + api, args, opts = first(Pkg.pkg"add C:\\Users\\test\\project:subdir") + @test api == Pkg.add + @test length(args) == 1 + @test args[1].path == normpath("C:\\\\Users\\\\test\\\\project") + @test args[1].subdir == "subdir" + end + # Add using preserve option api, args, opts = first(Pkg.pkg"add --preserve=none Example") @test api == Pkg.add @@ -1183,7 +1403,6 @@ end @test api == Pkg.add @test args == [Pkg.PackageSpec(;name="example")] @test isempty(opts) - @test_throws PkgError Pkg.pkg"add ./Example" api, args, opts = first(Pkg.pkg"add ./example") @test api == Pkg.add @test args == [Pkg.PackageSpec(;path="example")] @@ -1196,7 +1415,7 @@ end end end isolate() do; cd_tempdir() do dir # adding a nonexistent directory - @test_throws PkgError("`some/really/random/Dir` appears to be a local path, but directory does not exist" + @test_throws PkgError("Path `$(normpath("some/really/random/Dir"))` does not exist." ) Pkg.pkg"add some/really/random/Dir" # warn if not explicit about adding directory mkdir("Example") diff --git a/test/repl.jl b/test/repl.jl index c6926eb77d..69a0f529af 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -600,13 +600,6 @@ end @test typeof(pkg_spec) == Pkg.Types.PackageSpec end -@testset "parse git url (issue #1935) " begin - urls = ["https://github.com/abc/ABC.jl.git", "https://abc.github.io/ABC.jl"] - for url in urls - @test Pkg.REPLMode.package_lex([Pkg.REPLMode.QString((url), false)]) == [url] - end -end - @testset "unit test for REPLMode.promptf" begin function set_name(projfile_path, newname) sleep(1.1) @@ -764,7 +757,7 @@ end @testset "in_repl_mode" begin # Test that in_repl_mode() returns false by default (API mode) @test Pkg.in_repl_mode() == false - + # Test that in_repl_mode() returns true when running REPL commands # This is tested indirectly by running a simple REPL command temp_pkg_dir() do project_path @@ -777,12 +770,12 @@ end @test true end end - + # Test manual scoped value setting (for completeness) Base.ScopedValues.@with Pkg.IN_REPL_MODE => true begin @test Pkg.in_repl_mode() == true end - + # Verify we're back to false after the scoped block @test Pkg.in_repl_mode() == false end diff --git a/test/subdir.jl b/test/subdir.jl index a9833d2829..814eab04ca 100644 --- a/test/subdir.jl +++ b/test/subdir.jl @@ -258,13 +258,13 @@ end pkg"rm Dep" # Add from path at branch, REPL subdir syntax - pkgstr("add $(packages_dir):julia#master") + pkgstr("add $(packages_dir)#master:julia") @test isinstalled("Package") @test !isinstalled("Dep") @test isinstalled(dep) pkg"rm Package" - pkgstr("add $(packages_dir):dependencies/Dep#master") + pkgstr("add $(packages_dir)#master:dependencies/Dep") @test !isinstalled("Package") @test isinstalled("Dep") pkg"rm Dep" @@ -331,13 +331,13 @@ end pkg"rm Dep" # Add from url at branch, REPL subdir syntax. - pkgstr("add $(packages_dir_url):julia#master") + pkgstr("add $(packages_dir_url)#master:julia") @test isinstalled("Package") @test !isinstalled("Dep") @test isinstalled(dep) pkg"rm Package" - pkgstr("add $(packages_dir_url):dependencies/Dep#master") + pkgstr("add $(packages_dir_url)#master:dependencies/Dep") @test !isinstalled("Package") @test isinstalled("Dep") pkg"rm Dep" From 902b2c2fc0b59ec9443c50e9813f75dc34c4922e Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sat, 5 Jul 2025 09:13:07 +0200 Subject: [PATCH 119/154] warn if adding a local path containing a dirty repo (#4309) Co-authored-by: KristofferC --- src/Types.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Types.jl b/src/Types.jl index d0f8a4f451..0608eb21f3 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -891,6 +891,11 @@ function handle_repo_add!(ctx::Context, pkg::PackageSpec) pkgerror(msg) end LibGit2.with(GitTools.check_valid_HEAD, LibGit2.GitRepo(pkg.repo.source)) # check for valid git HEAD + LibGit2.with(LibGit2.GitRepo(pkg.repo.source)) do repo + if LibGit2.isdirty(repo) + @warn "The repository at `$(pkg.repo.source)` has uncommitted changes. Consider using `Pkg.develop` instead of `Pkg.add` if you want to work with the current state of the repository." + end + end pkg.repo.source = isabspath(pkg.repo.source) ? safe_realpath(pkg.repo.source) : relative_project_path(ctx.env.manifest_file, pkg.repo.source) repo_source = normpath(joinpath(dirname(ctx.env.manifest_file), pkg.repo.source)) else From d2c81dbe7288e9cf6acea2098a2f3de22e24782a Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 7 Jul 2025 09:57:15 +0200 Subject: [PATCH 120/154] Add support for GitHub pull request URLs (#4295) Co-authored-by: Claude --- CHANGELOG.md | 1 + src/REPLMode/argument_parsers.jl | 21 +++++++++++---------- src/Types.jl | 19 ++++++++++++++++++- test/new.jl | 5 +++++ 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8191c2c271..5201b39403 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Pkg v1.12 Release Notes - When adding or developing a package that exists in the `[weakdeps]` section, it is now automatically removed from weak dependencies and added as a regular dependency. ([#3865]) - Enhanced fuzzy matching algorithm for package name suggestions. +- The Pkg REPL now supports GitHub pull request URLs, allowing direct package installation from PRs via `pkg> add https://github.com/Org/Package.jl/pull/123` ([#4295]) Pkg v1.11 Release Notes ======================= diff --git a/src/REPLMode/argument_parsers.jl b/src/REPLMode/argument_parsers.jl index a5e68df298..ab71079a48 100644 --- a/src/REPLMode/argument_parsers.jl +++ b/src/REPLMode/argument_parsers.jl @@ -104,15 +104,16 @@ function extract_version(input::String) return remaining, version_part end -# Handle GitHub tree/commit URLs by converting them to standard URL + rev format -function preprocess_github_tree_commit_url(input::String) - m = match(r"https://github.com/(.*?)/(.*?)/(?:tree|commit)/(.*?)$", input) - if m !== nothing - base_url = "https://github.com/$(m.captures[1])/$(m.captures[2])" - rev = m.captures[3] - return [PackageIdentifier(base_url), Rev(rev)] - end - return nothing +function preprocess_github_url(input::String) + # Handle GitHub tree/commit URLs + if (m = match(r"https://github.com/(.*?)/(.*?)/(?:tree|commit)/(.*?)$", input)) !== nothing + return [PackageIdentifier("https://github.com/$(m.captures[1])/$(m.captures[2])"), Rev(m.captures[3])] + # Handle GitHub pull request URLs + elseif (m = match(r"https://github.com/(.*?)/(.*?)/pull/(\d+)$", input)) !== nothing + return [PackageIdentifier("https://github.com/$(m.captures[1])/$(m.captures[2])"), Rev("pull/$(m.captures[3])/head")] + else + return nothing + end end # Check if a colon in a URL string is part of URL structure (not a subdir separator) @@ -289,7 +290,7 @@ function parse_package_spec_new(input::String) end # Handle GitHub tree/commit URLs first (special case) - github_result = preprocess_github_tree_commit_url(input) + github_result = preprocess_github_url(input) if github_result !== nothing return github_result end diff --git a/src/Types.jl b/src/Types.jl index 0608eb21f3..45060dc5d6 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -918,7 +918,14 @@ function handle_repo_add!(ctx::Context, pkg::PackageSpec) fetched = false if obj_branch === nothing fetched = true - GitTools.fetch(ctx.io, repo, repo_source_typed; refspecs=refspecs) + # For pull requests, fetch the specific PR ref + if startswith(rev_or_hash, "pull/") && endswith(rev_or_hash, "/head") + pr_number = rev_or_hash[6:end-5] # Extract number from "pull/X/head" + pr_refspecs = ["+refs/pull/$(pr_number)/head:refs/remotes/cache/pull/$(pr_number)/head"] + GitTools.fetch(ctx.io, repo, repo_source_typed; refspecs=pr_refspecs) + else + GitTools.fetch(ctx.io, repo, repo_source_typed; refspecs=refspecs) + end obj_branch = get_object_or_branch(repo, rev_or_hash) if obj_branch === nothing pkgerror("Did not find rev $(rev_or_hash) in repository") @@ -1003,6 +1010,16 @@ get_object_or_branch(repo, rev::SHA1) = # Returns nothing if rev could not be found in repo function get_object_or_branch(repo, rev) + # Handle pull request references + if startswith(rev, "pull/") && endswith(rev, "/head") + try + gitobject = LibGit2.GitObject(repo, "remotes/cache/" * rev) + return gitobject, true + catch err + err isa LibGit2.GitError && err.code == LibGit2.Error.ENOTFOUND || rethrow() + end + end + try gitobject = LibGit2.GitObject(repo, "remotes/cache/heads/" * rev) return gitobject, true diff --git a/test/new.jl b/test/new.jl index bde938f347..57c8c32ff1 100644 --- a/test/new.jl +++ b/test/new.jl @@ -432,6 +432,11 @@ end @test arg.url == "https://github.com/JuliaLang/Pkg.jl" @test arg.rev == "aa/gitlab" + api, args, opts = first(Pkg.pkg"add https://github.com/JuliaPy/PythonCall.jl/pull/529") + arg = args[1] + @test arg.url == "https://github.com/JuliaPy/PythonCall.jl" + @test arg.rev == "pull/529/head" + api, args, opts = first(Pkg.pkg"add https://github.com/TimG1964/XLSX.jl#Bug-fixing-post-#289:subdir") arg = args[1] @test arg.url == "https://github.com/TimG1964/XLSX.jl" From 393f6012250001c488a30b3bd3d2e3a57ed2b97a Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 7 Jul 2025 13:12:31 +0200 Subject: [PATCH 121/154] consider a package with a rev (but without a source) to not be "tracked" by a registry (#4311) Co-authored-by: KristofferC --- src/Operations.jl | 2 +- src/Types.jl | 4 ++-- test/new.jl | 21 +++++++++++++++++-- .../sources_only_rev/Project.toml | 5 +++++ 4 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 test/test_packages/sources_only_rev/Project.toml diff --git a/src/Operations.jl b/src/Operations.jl index 309de62867..f85b8b6671 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -351,7 +351,7 @@ function collect_project(pkg::Union{PackageSpec, Nothing}, path::String) end is_tracking_path(pkg) = pkg.path !== nothing -is_tracking_repo(pkg) = pkg.repo.source !== nothing +is_tracking_repo(pkg) = (pkg.repo.source !== nothing || pkg.repo.rev !== nothing) is_tracking_registry(pkg) = !is_tracking_path(pkg) && !is_tracking_repo(pkg) isfixed(pkg) = !is_tracking_registry(pkg) || pkg.pinned diff --git a/src/Types.jl b/src/Types.jl index 45060dc5d6..f6f8a9249d 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -1254,7 +1254,6 @@ function write_env(env::EnvCache; update_undo=true, @assert entry.path == path end if repo != GitRepo() - @assert entry.repo.source == repo.source if repo.rev !== nothing @assert entry.repo.rev == repo.rev end @@ -1266,7 +1265,8 @@ function write_env(env::EnvCache; update_undo=true, if entry.path !== nothing env.project.sources[pkg] = Dict("path" => entry.path) elseif entry.repo != GitRepo() - d = Dict("url" => entry.repo.source) + d = Dict{String, String}() + entry.repo.source !== nothing && (d["url"] = entry.repo.source) entry.repo.rev !== nothing && (d["rev"] = entry.repo.rev) entry.repo.subdir !== nothing && (d["subdir"] = entry.repo.subdir) env.project.sources[pkg] = d diff --git a/test/new.jl b/test/new.jl index 57c8c32ff1..cf95f841de 100644 --- a/test/new.jl +++ b/test/new.jl @@ -1358,14 +1358,14 @@ end @test length(args) == 1 @test args[1].path == normpath("C:\\\\Users\\\\test\\\\project") @test args[1].subdir === nothing - + # Test with forward slashes too api, args, opts = first(Pkg.pkg"add C:/Users/test/project") @test api == Pkg.add @test length(args) == 1 @test args[1].path == normpath("C:/Users/test/project") @test args[1].subdir === nothing - + # Test that actual subdir syntax still works with Windows paths api, args, opts = first(Pkg.pkg"add C:\\Users\\test\\project:subdir") @test api == Pkg.add @@ -3516,6 +3516,23 @@ end end end +@testset "test instantiate with sources with only rev" begin + isolate() do + mktempdir() do dir + cp(joinpath(@__DIR__, "test_packages", "sources_only_rev", "Project.toml"), joinpath(dir, "Project.toml")) + cd(dir) do + with_current_env() do + @test !isfile("Manifest.toml") + Pkg.instantiate() + uuid, info = only(Pkg.dependencies()) + @test info.git_revision == "ba3d6704f09330ae973773496a4212f85e0ffe45" + @test info.git_source == "https://github.com/JuliaLang/Example.jl.git" + end + end + end + end +end + @testset "status showing incompatible loaded deps" begin cmd = addenv(`$(Base.julia_cmd()) --color=no --startup-file=no -e " using Pkg diff --git a/test/test_packages/sources_only_rev/Project.toml b/test/test_packages/sources_only_rev/Project.toml new file mode 100644 index 0000000000..73a01c5d00 --- /dev/null +++ b/test/test_packages/sources_only_rev/Project.toml @@ -0,0 +1,5 @@ +[deps] +Example = "7876af07-990d-54b4-ab0e-23690620f79a" + +[sources] +Example = {rev = "ba3d6704f09330ae973773496a4212f85e0ffe45"} From ca7d99786d28b32a2a99f5ea62c54bd50154640e Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 7 Jul 2025 13:13:15 +0200 Subject: [PATCH 122/154] fix assert from triggering with different path separators (#4305) Co-authored-by: KristofferC --- src/Types.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Types.jl b/src/Types.jl index f6f8a9249d..3607655d84 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -1251,7 +1251,7 @@ function write_env(env::EnvCache; update_undo=true, path, repo = get_path_repo(env.project, pkg) entry = manifest_info(env.manifest, uuid) if path !== nothing - @assert entry.path == path + @assert normpath(entry.path) == normpath(path) end if repo != GitRepo() if repo.rev !== nothing From 32921ac3a284e241cd14192c69ddf14c1648c1ae Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 7 Jul 2025 14:14:18 +0200 Subject: [PATCH 123/154] fix `status --diff` to work when package is in a non-root path (#4312) Co-authored-by: KristofferC --- src/Operations.jl | 7 ++++--- src/utils.jl | 16 ++++++++++++++++ test/new.jl | 15 +++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/Operations.jl b/src/Operations.jl index f85b8b6671..42951b011f 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -15,7 +15,7 @@ using Base.BinaryPlatforms import ...Pkg import ...Pkg: pkg_server, Registry, pathrepr, can_fancyprint, printpkgstyle, stderr_f, OFFLINE_MODE import ...Pkg: UPDATED_REGISTRY_THIS_SESSION, RESPECT_SYSIMAGE_VERSIONS, should_autoprecompile -import ...Pkg: usable_io +import ...Pkg: usable_io, discover_repo ######### # Utils # @@ -2988,10 +2988,11 @@ function status(env::EnvCache, registries::Vector{Registry.RegistryInstance}, pk old_env = nothing if git_diff project_dir = dirname(env.project_file) - if !ispath(joinpath(project_dir, ".git")) + git_repo_dir = discover_repo(project_dir) + if git_repo_dir == nothing @warn "diff option only available for environments in git repositories, ignoring." else - old_env = git_head_env(env, project_dir) + old_env = git_head_env(env, git_repo_dir) if old_env === nothing @warn "could not read project from HEAD, displaying absolute status instead." end diff --git a/src/utils.jl b/src/utils.jl index 2e40e98ab6..2b652d7f80 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -175,3 +175,19 @@ end if VERSION < v"1.2.0-DEV.269" # Defined in Base as of #30947 Base.isless(a::UUID, b::UUID) = a.value < b.value end + +function discover_repo(path::AbstractString) + dir = abspath(path) + stop_dir = homedir() + + while true + gitdir = joinpath(dir, ".git") + if isdir(gitdir) || isfile(gitdir) + return dir + end + dir == stop_dir && return nothing + parent = dirname(dir) + parent == dir && return nothing + dir = parent + end +end diff --git a/test/new.jl b/test/new.jl index cf95f841de..9a3c01e6d2 100644 --- a/test/new.jl +++ b/test/new.jl @@ -3516,6 +3516,21 @@ end end end +@testset "status diff non-root" begin + isolate(loaded_depot=true) do + cd_tempdir() do dir + Pkg.generate("A") + git_init_and_commit(".") + Pkg.activate("A") + Pkg.add("Example") + io = IOBuffer() + Pkg.status(; io, diff=true) + str = String(take!(io)) + @test occursin("+ Example", str) + end + end +end + @testset "test instantiate with sources with only rev" begin isolate() do mktempdir() do dir From 6ae649955a99dff7934f7dcd7ee57e919ba10923 Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Tue, 8 Jul 2025 00:33:09 +0200 Subject: [PATCH 124/154] Fix stack overflow in `safe_realpath` (#4313) --- src/utils.jl | 3 ++- test/misc.jl | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 2b652d7f80..45df5dc6c5 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -119,7 +119,6 @@ end # try to call realpath on as much as possible function safe_realpath(path) - isempty(path) && return path if ispath(path) try return realpath(path) @@ -128,6 +127,8 @@ function safe_realpath(path) end end a, b = splitdir(path) + # path cannot be reduced at the root or drive, avoid stack overflow + isempty(b) && return path return joinpath(safe_realpath(a), b) end diff --git a/test/misc.jl b/test/misc.jl index e9b3d00ff6..3a386d161a 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -19,8 +19,9 @@ end end @testset "safe_realpath" begin + realpath(Sys.BINDIR) == Pkg.safe_realpath(Sys.BINDIR) # issue #3085 - for p in ("", "some-non-existing-path") + for p in ("", "some-non-existing-path", "some-non-existing-drive:") @test p == Pkg.safe_realpath(p) end end From 4e49307f3a4f55d1718379d97c0558ac351fdd23 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Mon, 7 Jul 2025 20:39:29 -0400 Subject: [PATCH 125/154] Add error that explains PackageSpec.repo is a private field (#4170) --- src/API.jl | 5 ++++- src/Types.jl | 2 +- test/new.jl | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/API.jl b/src/API.jl index 52c4362bfe..6b746f9e0b 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1593,7 +1593,10 @@ end function handle_package_input!(pkg::PackageSpec) if pkg.path !== nothing && pkg.url !== nothing - pkgerror("`path` and `url` are conflicting specifications") + pkgerror("Conflicting `path` and `url` in PackageSpec") + end + if pkg.repo.source !== nothing || pkg.repo.rev !== nothing || pkg.repo.subdir !== nothing + pkgerror("`repo` is a private field of PackageSpec and should not be set directly") end pkg.repo = Types.GitRepo(rev = pkg.rev, source = pkg.url !== nothing ? pkg.url : pkg.path, subdir = pkg.subdir) diff --git a/src/Types.jl b/src/Types.jl index 3607655d84..d13105b8b1 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -93,7 +93,7 @@ mutable struct PackageSpec uuid::Union{Nothing,UUID} version::Union{Nothing,VersionTypes,String} tree_hash::Union{Nothing,SHA1} - repo::GitRepo + repo::GitRepo # private path::Union{Nothing,String} pinned::Bool # used for input only diff --git a/test/new.jl b/test/new.jl index 9a3c01e6d2..3b1a0faa53 100644 --- a/test/new.jl +++ b/test/new.jl @@ -2164,6 +2164,8 @@ end Pkg.dependencies(exuuid) do pkg @test pkg.version > v"0.3.0" end + + @test_throws PkgError("`repo` is a private field of PackageSpec and should not be set directly") Pkg.add([Pkg.PackageSpec(;repo=Pkg.Types.GitRepo(source="someurl"))]) end end From 4544cd97adb677c3165fe010e2adfe53c266c0c4 Mon Sep 17 00:00:00 2001 From: Mauro Date: Tue, 8 Jul 2025 02:41:23 +0200 Subject: [PATCH 126/154] Add a docs page about Depots (#2245) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Ian Butterworth --- docs/make.jl | 1 + docs/src/depots.md | 55 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 docs/src/depots.md diff --git a/docs/make.jl b/docs/make.jl index 00794f72e8..4b97e3ec23 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -44,6 +44,7 @@ makedocs( "repl.md", "api.md", "protocol.md", + "depots.md", ], ) diff --git a/docs/src/depots.md b/docs/src/depots.md new file mode 100644 index 0000000000..1d854b9367 --- /dev/null +++ b/docs/src/depots.md @@ -0,0 +1,55 @@ +# **15.** Depots + +The packages installed for a particular environment, defined in the +files `Project.toml` and `Manifest.toml` within the directory +structure, are not actually installed within that directory but into a +"depot". The location of the depots are set by the variable +[`DEPOT_PATH`](https://docs.julialang.org/en/v1/base/constants/#Base.DEPOT_PATH). + +For details on the default depot locations and how they vary by installation method, +see the [`DEPOT_PATH`](https://docs.julialang.org/en/v1/base/constants/#Base.DEPOT_PATH) documentation. + +Packages which are installed by a user go into the first depot and the Julia +standard library is in the last depot. + +You should not need to manage the user depot directly. Pkg will automatically clean up +the depots when packages are removed after a delay. However you may want to manually +remove old `.julia/compiled/` subdirectories if you have any that reside for older Julia +versions that you no longer use (hence have not been run to tidy themselves up). + +## Configuring the depot path with `JULIA_DEPOT_PATH` + +The depot path can be configured using the `JULIA_DEPOT_PATH` environment variable, +which is used to populate the global Julia [`DEPOT_PATH`](https://docs.julialang.org/en/v1/base/constants/#Base.DEPOT_PATH) variable +at startup. For complete details on the behavior of this environment variable, +see the [environment variables documentation](https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH). + +Unlike the shell `PATH` variable, empty entries in `JULIA_DEPOT_PATH` +have special behavior for easy overriding of the user depot while retaining access to system resources. +For example, to switch the user depot to `/custom/depot` while still accessing bundled +resources, use a trailing path separator: + +```bash +export JULIA_DEPOT_PATH="/custom/depot:" +``` + +!!! note + The trailing path separator (`:` on Unix, `;` on Windows) is crucial for including + the default system depots, which contain the standard library and other bundled + resources. Without it, Julia will only use the specified depot and will have to precompile + standard library packages, which can be time-consuming and inefficient. + +## Shared depots for distributed computing + +When using Julia in distributed computing environments, such as high-performance computing +(HPC) clusters, it's recommended to use a shared depot via `JULIA_DEPOT_PATH`. This allows +multiple Julia processes to share precompiled packages and reduces redundant compilation. + +Since Julia v1.10, multiple processes using the same depot coordinate via pidfile locks +to ensure only one process precompiles a package while others wait. However, due to +the caching of native code in pkgimages since v1.9, you may need to set the `JULIA_CPU_TARGET` +environment variable appropriately to ensure cache compatibility across different +worker nodes with varying CPU capabilities. + +For more details, see the [FAQ section on distributed computing](https://docs.julialang.org/en/v1/manual/faq/#Computing-cluster) +and the [environment variables documentation](https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_CPU_TARGET). From eb6320c068bc43398f22eb1412624dcdca7c4380 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 8 Jul 2025 04:49:25 -0400 Subject: [PATCH 127/154] Warn about yanked versions. Add docs about yanked versions. (#4310) --- CHANGELOG.md | 1 + docs/src/managing-packages.md | 24 ++++++++++ src/Operations.jl | 70 +++++++++++++++++++++++++--- src/Pkg.jl | 1 + src/REPLMode/command_declarations.jl | 2 + src/Types.jl | 4 +- test/api.jl | 27 +++++++++++ test/manifest/yanked/Manifest.toml | 62 ++++++++++++++++++++++++ test/manifest/yanked/Project.toml | 2 + 9 files changed, 185 insertions(+), 8 deletions(-) create mode 100644 test/manifest/yanked/Manifest.toml create mode 100644 test/manifest/yanked/Project.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index 5201b39403..77f81f2f22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ Pkg v1.13 Release Notes - `Pkg.build` now supports an `allow_reresolve` keyword argument to control whether the build process can re-resolve package versions, similar to the existing option for `Pkg.test`. ([#3329]) - Packages are now automatically added to `[sources]` when they are added by url or devved. - `update` now shows a helpful tip when trying to upgrade a specific package that can be upgraded but is held back because it's part of a less optimal resolver solution ([#4266]) +- `Pkg.status` now displays yanked packages with a `[yanked]` indicator and shows a warning when yanked packages are present. `Pkg.resolve` errors also display warnings about yanked packages that are not resolvable. ([#4310]) Pkg v1.12 Release Notes ======================= diff --git a/docs/src/managing-packages.md b/docs/src/managing-packages.md index ec6cf34be8..37805ae7e6 100644 --- a/docs/src/managing-packages.md +++ b/docs/src/managing-packages.md @@ -488,6 +488,30 @@ To fix such errors, you have a number of options: - try fixing the problem yourself. This becomes easier once you understand `Project.toml` files and how they declare their compatibility requirements. We'll return to this example in [Fixing conflicts](@ref Fixing-conflicts). +## Yanked packages + +Package registries can mark specific versions of packages as "yanked". A yanked package version +is one that should no longer be used, typically because it contains serious bugs, security +vulnerabilities, or other critical issues. When a package version is yanked, it becomes +unavailable for new installations but remains accessible (i.e. via `instantiate`) to maintain reproducibility +of existing environments. + +When you run `pkg> status`, yanked packages are clearly marked with a warning symbol: + +```julia-repl +(@v1.13) pkg> status + Status `~/.julia/environments/v1.13/Project.toml` + [682c06a0] JSON v0.21.3 + [f4259836] Example v1.2.0 [yanked] +``` + +The `[yanked]` annotation indicate that version `v1.2.0` of the `Example` package +has been yanked and should be updated or replaced. + +When resolving dependencies, Pkg will warn you if yanked packages are present and may provide +guidance on how to resolve the situation. It's important to address yanked packages promptly +to ensure the security and stability of your Julia environment. + ## Garbage collecting old, unused packages As packages are updated and projects are deleted, installed package versions and artifacts that were diff --git a/src/Operations.jl b/src/Operations.jl index 42951b011f..a2cd596b2a 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -21,6 +21,34 @@ import ...Pkg: usable_io, discover_repo # Utils # ######### +# Helper functions for yanked package checking +function is_pkgversion_yanked(uuid::UUID, version::VersionNumber, registries::Vector{Registry.RegistryInstance}=Registry.reachable_registries()) + for reg in registries + reg_pkg = get(reg, uuid, nothing) + if reg_pkg !== nothing + info = Registry.registry_info(reg_pkg) + if haskey(info.version_info, version) && Registry.isyanked(info, version) + return true + end + end + end + return false +end + +function is_pkgversion_yanked(pkg::PackageSpec, registries::Vector{Registry.RegistryInstance}=Registry.reachable_registries()) + if pkg.uuid === nothing || pkg.version === nothing || !(pkg.version isa VersionNumber) + return false + end + return is_pkgversion_yanked(pkg.uuid, pkg.version, registries) +end + +function is_pkgversion_yanked(entry::PackageEntry, registries::Vector{Registry.RegistryInstance}=Registry.reachable_registries()) + if entry.version === nothing || !(entry.version isa VersionNumber) + return false + end + return is_pkgversion_yanked(entry.uuid, entry.version, registries) +end + function default_preserve() if Base.get_bool_env("JULIA_PKG_PRESERVE_TIERED_INSTALLED", false) PRESERVE_TIERED_INSTALLED @@ -1716,12 +1744,26 @@ function _resolve(io::IO, env::EnvCache, registries::Vector{Registry.RegistryIns pkgs::Vector{PackageSpec}, preserve::PreserveLevel, julia_version) usingstrategy = preserve != PRESERVE_TIERED ? " using $preserve" : "" printpkgstyle(io, :Resolving, "package versions$(usingstrategy)...") - if preserve == PRESERVE_TIERED_INSTALLED - tiered_resolve(env, registries, pkgs, julia_version, true) - elseif preserve == PRESERVE_TIERED - tiered_resolve(env, registries, pkgs, julia_version, false) - else - targeted_resolve(env, registries, pkgs, preserve, julia_version) + try + if preserve == PRESERVE_TIERED_INSTALLED + tiered_resolve(env, registries, pkgs, julia_version, true) + elseif preserve == PRESERVE_TIERED + tiered_resolve(env, registries, pkgs, julia_version, false) + else + targeted_resolve(env, registries, pkgs, preserve, julia_version) + end + catch err + + if err isa Resolve.ResolverError + yanked_pkgs = filter(pkg -> is_pkgversion_yanked(pkg, registries), load_all_deps(env)) + if !isempty(yanked_pkgs) + indent = " "^(Pkg.pkgstyle_indent) + yanked_str = join(map(pkg -> indent * " - " * err_rep(pkg, quotes=false) * " " * string(pkg.version), yanked_pkgs), "\n") + printpkgstyle(io, :Warning, """The following package versions were yanked from their registry and \ + are not resolvable:\n$yanked_str""", color=Base.warn_color()) + end + end + rethrow() end end @@ -2868,6 +2910,12 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie diff ? print_diff(io, pkg.old, pkg.new) : print_single(io, pkg.new) + # show if package is yanked + pkg_spec = something(pkg.new, pkg.old) + if is_pkgversion_yanked(pkg_spec, registries) + printstyled(io, " [yanked]"; color=:yellow) + end + if outdated && !diff && pkg.compat_data !== nothing packages_holding_back, max_version, max_version_compat = pkg.compat_data if pkg.new.version !== max_version_compat && max_version_compat != max_version @@ -2948,6 +2996,16 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie end end + # Check if any packages are yanked for warning message + any_yanked_packages = any(pkg -> is_pkgversion_yanked(something(pkg.new, pkg.old), registries), package_statuses) + + # Add warning for yanked packages + if any_yanked_packages + yanked_str = sprint((io, args) -> printstyled(io, args...; color=:yellow), "[yanked]", context=io) + printpkgstyle(io, :Warning, """Package versions marked with $yanked_str have been pulled from their registry. \ + It is recommended to update them to resolve a valid version.""", color=Base.warn_color(), ignore_indent) + end + return nothing end diff --git a/src/Pkg.jl b/src/Pkg.jl index 488693073d..8ba2f4e107 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -565,6 +565,7 @@ Print out the status of the project/manifest. Packages marked with `⌃` have new versions that can be installed, e.g. via [`Pkg.update`](@ref). Those marked with `⌅` have new versions available, but cannot be installed due to compatibility conflicts with other packages. To see why, set the keyword argument `outdated=true`. +Packages marked with `[yanked]` are yanked versions that should be updated or replaced as they may contain bugs or security vulnerabilities. Setting `outdated=true` will only show packages that are not on the latest version, their maximum version and why they are not on the latest version (either due to other diff --git a/src/REPLMode/command_declarations.jl b/src/REPLMode/command_declarations.jl index 9e016c3f76..4abcdfe0e8 100644 --- a/src/REPLMode/command_declarations.jl +++ b/src/REPLMode/command_declarations.jl @@ -438,6 +438,8 @@ versions that may be installed, e.g. via `pkg> up`. Those marked with `⌅` have new versions available, but cannot be installed due to compatibility constraints. To see why use `pkg> status --outdated` which shows any packages that are not at their latest version and if any packages are holding them back. +Packages marked with `[yanked]` have been yanked from the registry and should be +updated or removed. Use `pkg> status --extensions` to show dependencies with extensions and what extension dependencies of those that are currently loaded. diff --git a/src/Types.jl b/src/Types.jl index d13105b8b1..187efa8816 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -130,12 +130,12 @@ function Base.hash(a::PackageSpec, h::UInt) return foldr(hash, [a.name, a.uuid, a.version, a.tree_hash, a.repo, a.path, a.pinned], init=h) end -function err_rep(pkg::PackageSpec) +function err_rep(pkg::PackageSpec; quotes::Bool=true) x = pkg.name !== nothing && pkg.uuid !== nothing ? x = "$(pkg.name) [$(string(pkg.uuid)[1:8])]" : pkg.name !== nothing ? pkg.name : pkg.uuid !== nothing ? string(pkg.uuid)[1:8] : pkg.repo.source - return "`$x`" + return quotes ? "`$x`" : x end has_name(pkg::PackageSpec) = pkg.name !== nothing diff --git a/test/api.jl b/test/api.jl index 0441f3f020..65c40ca37e 100644 --- a/test/api.jl +++ b/test/api.jl @@ -342,4 +342,31 @@ end end end end +@testset "Yanked package handling" begin + isolate() do; mktempdir() do tempdir + # Copy the yanked test environment + test_env_dir = joinpath(tempdir, "yanked_test") + cp(joinpath(@__DIR__, "manifest", "yanked"), test_env_dir) + Pkg.activate(test_env_dir) + + @testset "status shows yanked packages" begin + iob = IOBuffer() + Pkg.status(io=iob) + status_output = String(take!(iob)) + + @test occursin("Mocking v0.7.4 [yanked]", status_output) + @test occursin("Package versions marked with [yanked] have been pulled from their registry.", status_output) + end + @testset "resolve error shows yanked packages warning" begin + # Try to add a package that will cause resolve conflicts with yanked package + iob = IOBuffer() + @test_throws Pkg.Resolve.ResolverError Pkg.add("Example"; preserve=Pkg.PRESERVE_ALL, io=iob) + error_output = String(take!(iob)) + + @test occursin("The following package versions were yanked from their registry and are not resolvable:", error_output) + @test occursin("Mocking [78c3b35d] 0.7.4", error_output) + end + end end +end + end # module APITests diff --git a/test/manifest/yanked/Manifest.toml b/test/manifest/yanked/Manifest.toml new file mode 100644 index 0000000000..39261c8e24 --- /dev/null +++ b/test/manifest/yanked/Manifest.toml @@ -0,0 +1,62 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.13.0-DEV" +manifest_format = "2.0" +project_hash = "8a91c3bdaf7537df6f842463e0505fb7c623875c" + +[[deps.Compat]] +deps = ["TOML", "UUIDs"] +git-tree-sha1 = "3a3dfb30697e96a440e4149c8c51bf32f818c0f3" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.17.0" + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + + [deps.Compat.weakdeps] + Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" + +[[deps.ExprTools]] +git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" +uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" +version = "0.1.10" + +[[deps.Mocking]] +deps = ["Compat", "ExprTools"] +git-tree-sha1 = "d5ca7901d59738132d6f9be9a18da50bc85c5115" +uuid = "78c3b35d-d492-501b-9361-3d52fe80e533" +version = "0.7.4" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" diff --git a/test/manifest/yanked/Project.toml b/test/manifest/yanked/Project.toml new file mode 100644 index 0000000000..f61c7f288a --- /dev/null +++ b/test/manifest/yanked/Project.toml @@ -0,0 +1,2 @@ +[deps] +Mocking = "78c3b35d-d492-501b-9361-3d52fe80e533" From 6e84091f259a2f3830506a83f96c2c9599762b99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnar=20Farneb=C3=A4ck?= Date: Tue, 8 Jul 2025 11:16:05 +0200 Subject: [PATCH 128/154] More registry status (#4300) --- src/Registry/Registry.jl | 77 ++++++++++++++++++++++++++++++++++++++++ test/registry.jl | 17 ++++++++- 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/src/Registry/Registry.jl b/src/Registry/Registry.jl index fcaf05ae24..180b4855ac 100644 --- a/src/Registry/Registry.jl +++ b/src/Registry/Registry.jl @@ -176,6 +176,7 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depots::Union{S # Use the first depot as the target target_depot = depots1(depots) populate_known_registries_with_urls!(regs) + registry_update_log = get_registry_update_log() regdir = joinpath(target_depot, "registries") isdir(regdir) || mkpath(regdir) # only allow one julia process to download and install registries at a time @@ -214,6 +215,7 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depots::Union{S mv(tmp, joinpath(regdir, reg.name * ".tar.gz"); force=true) reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(_hash), "path" => reg.name * ".tar.gz") atomic_toml_write(joinpath(regdir, reg.name * ".toml"), reg_info) + registry_update_log[string(reg.uuid)] = now() printpkgstyle(io, :Added, "`$(reg.name)` registry to $(Base.contractuser(regdir))") else mktempdir() do tmp @@ -279,12 +281,14 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depots::Union{S elseif (url !== nothing && registry_use_pkg_server()) || reg.linked !== true # if the dir doesn't exist, or exists but doesn't contain a Registry.toml mv(tmp, regpath, force=true) + registry_update_log[string(reg.uuid)] = now() printpkgstyle(io, :Added, "registry `$(registry.name)` to `$(Base.contractuser(regpath))`") end end end end end # mkpidlock + save_registry_update_log(registry_update_log) return nothing end @@ -579,13 +583,86 @@ function status(io::IO=stderr_f()) if isempty(regs) println(io, " (no registries found)") else + registry_update_log = get_registry_update_log() + server_registry_info = Pkg.OFFLINE_MODE[] ? nothing : pkg_server_registry_info() + flavor = get(ENV, "JULIA_PKG_SERVER_REGISTRY_PREFERENCE", "") for reg in regs printstyled(io, " [$(string(reg.uuid)[1:8])]"; color = :light_black) print(io, " $(reg.name)") reg.repo === nothing || print(io, " ($(reg.repo))") println(io) + + registry_type = get_registry_type(reg) + if registry_type == :git + print(io, " git registry") + elseif registry_type == :unpacked + print(io, " unpacked registry with hash $(reg.tree_info)") + elseif registry_type == :packed + print(io, " packed registry with hash $(reg.tree_info)") + elseif registry_type == :bare + # We could try to detect a symlink but this is too + # rarely used to be worth the complexity. + print(io, " bare registry") + else + print(io, " unknown registry format") + end + update_time = get(registry_update_log, string(reg.uuid), nothing) + if !isnothing(update_time) + time_string = Dates.format(update_time, dateformat"yyyy-mm-dd HH:MM:SS") + print(io, ", last updated $(time_string)") + end + println(io) + + if registry_type != :git && !isnothing(server_registry_info) + server_url, registries = server_registry_info + if haskey(registries, reg.uuid) + print(io, " served by $(server_url)") + if flavor != "" + print(io, " ($flavor flavor)") + end + if registries[reg.uuid] != reg.tree_info + print(io, " - update available") + end + println(io) + end + end end end end +# The registry can be installed in a number of different ways, for +# evolutionary reasons. +# +# 1. A tarball that is not unpacked. In this case Pkg handles the +# registry in memory. The tarball is distributed by a package server. +# This is the preferred option, in particular for the General +# registry. +# +# 2. A tarball that is unpacked. This only differs from above by +# having the files on disk instead of in memory. In both cases Pkg +# keeps track of the tarball's tree hash to know if it can be updated. +# +# 3. A clone of a git repository. This is characterized by the +# presence of a .git directory. All updating is handled with git. +# This is not preferred for the General registry but may be the only +# practical option for private registries. +# +# 4. A bare registry with only the registry files and no metadata. +# This can be installed by adding or symlinking from a local path but +# there is no way to update it from Pkg. +# +# It is also possible for a packed/unpacked registry to coexist on +# disk with a git/bare registry, in which case a new Julia may use the +# former and a sufficiently old Julia the latter. +function get_registry_type(reg) + isnothing(reg.in_memory_registry) || return :packed + isnothing(reg.tree_info) || return :unpacked + isdir(joinpath(reg.path, ".git")) && return :git + isfile(joinpath(reg.path, "Registry.toml")) && return :bare + # Indicates either that the registry data is corrupt or that it + # has been handled by a future Julia version with non-backwards + # compatible conventions. + return :unknown +end + end # module diff --git a/test/registry.jl b/test/registry.jl index 96c4d1144a..4894de761c 100644 --- a/test/registry.jl +++ b/test/registry.jl @@ -5,6 +5,7 @@ using Pkg, UUIDs, LibGit2, Test using Pkg: depots1 using Pkg.REPLMode: pkgstr using Pkg.Types: PkgError, manifest_info, PackageSpec, EnvCache +using Dates: Second using ..Utils @@ -263,7 +264,7 @@ end # Test that `update()` with `depots` runs io = Base.BufferStream() - Registry.update(; depots=[depot_off_path], io) + Registry.update(; depots=[depot_off_path], io, update_cooldown = Second(0)) closewrite(io) output = read(io, String) @test occursin("registry at `$(depot_off_path)", output) @@ -275,6 +276,20 @@ end @test isinstalled((name = "Example", uuid = UUID("7876af07-990d-54b4-ab0e-23690620f79a"))) end end + # Registry status. Mostly verify that it runs without errors but + # also make some sanity checks on the output. We can't really know + # whether it was installed as a git clone or a tarball, so that + # limits how much information we are guaranteed to get from + # status. + temp_pkg_dir() do depot + Registry.add("General") + buf = IOBuffer() + Pkg.Registry.status(buf) + status = String(take!(buf)) + @test contains(status, "[23338594] General (https://github.com/JuliaRegistries/General.git)") + @test contains(status, "last updated") + end + # only clone default registry if there are no registries installed at all temp_pkg_dir() do depot1; mktempdir() do depot2 append!(empty!(DEPOT_PATH), [depot1, depot2]) From 21c4aa4329d7b43b368093aac7b8ad5df550f128 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 8 Jul 2025 08:40:56 -0400 Subject: [PATCH 129/154] add `pkg> compat --current` mode (#3266) --- CHANGELOG.md | 2 + ext/REPLExt/REPLExt.jl | 5 +- ext/REPLExt/compat.jl | 2 +- src/API.jl | 80 ++++++++++++++++++++++++++-- src/REPLMode/command_declarations.jl | 9 ++++ test/new.jl | 70 +++++++++++++++++++++++- 6 files changed, 160 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77f81f2f22..f0cc2664fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Pkg v1.13 Release Notes - Packages are now automatically added to `[sources]` when they are added by url or devved. - `update` now shows a helpful tip when trying to upgrade a specific package that can be upgraded but is held back because it's part of a less optimal resolver solution ([#4266]) - `Pkg.status` now displays yanked packages with a `[yanked]` indicator and shows a warning when yanked packages are present. `Pkg.resolve` errors also display warnings about yanked packages that are not resolvable. ([#4310]) +- Added `pkg> compat --current` command to automatically populate missing compat entries with the currently resolved package versions. Use `pkg> compat --current` for all packages or `pkg> compat Foo --current` for specific packages. ([#3266]) Pkg v1.12 Release Notes ======================= @@ -127,3 +128,4 @@ Pkg v1.7 Release Notes [#2995]: https://github.com/JuliaLang/Pkg.jl/issues/2995 [#3002]: https://github.com/JuliaLang/Pkg.jl/issues/3002 [#3021]: https://github.com/JuliaLang/Pkg.jl/issues/3021 +[#3266]: https://github.com/JuliaLang/Pkg.jl/pull/3266 diff --git a/ext/REPLExt/REPLExt.jl b/ext/REPLExt/REPLExt.jl index 980f8d7a24..8e08bad9e4 100644 --- a/ext/REPLExt/REPLExt.jl +++ b/ext/REPLExt/REPLExt.jl @@ -10,13 +10,16 @@ import REPL import .REPL: LineEdit, REPLCompletions, TerminalMenus import Pkg -import .Pkg: linewrap, pathrepr, compat, can_fancyprint, printpkgstyle, PKGMODE_PROJECT +import .Pkg: linewrap, pathrepr, can_fancyprint, printpkgstyle, PKGMODE_PROJECT using .Pkg: Types, Operations, API, Registry, Resolve, REPLMode, safe_realpath using .REPLMode: Statement, CommandSpec, Command, prepare_cmd, tokenize, core_parse, SPECS, api_options, parse_option, api_options, is_opt, wrap_option using .Types: Context, PkgError, pkgerror, EnvCache +using .API: set_current_compat +import .API: _compat + include("completions.jl") include("compat.jl") diff --git a/ext/REPLExt/compat.jl b/ext/REPLExt/compat.jl index a9a537cf0f..f2de1aaea0 100644 --- a/ext/REPLExt/compat.jl +++ b/ext/REPLExt/compat.jl @@ -1,5 +1,5 @@ # TODO: Overload -function compat(ctx::Context; io = nothing) +function _compat(ctx::Context; io = nothing) io = something(io, ctx.io) can_fancyprint(io) || pkgerror("Pkg.compat cannot be run interactively in this terminal") printpkgstyle(io, :Compat, pathrepr(ctx.env.project_file)) diff --git a/src/API.jl b/src/API.jl index 6b746f9e0b..da2cb0a23a 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1418,7 +1418,13 @@ function activate(f::Function, new_project::AbstractString) end end -function compat(ctx::Context, pkg::String, compat_str::Union{Nothing,String}; io = nothing, kwargs...) +function _compat(ctx::Context, pkg::String, compat_str::Union{Nothing,String}; current::Bool=false, io = nothing, kwargs...) + if current + if compat_str !== nothing + pkgerror("`current` is true, but `compat_str` is not nothing. This is not allowed.") + end + return set_current_compat(ctx, pkg; io=io) + end io = something(io, ctx.io) pkg = pkg == "Julia" ? "julia" : pkg isnothing(compat_str) || (compat_str = string(strip(compat_str, '"'))) @@ -1457,9 +1463,75 @@ function compat(ctx::Context, pkg::String, compat_str::Union{Nothing,String}; io pkgerror("No package named $pkg in current Project") end end -compat(pkg::String; kwargs...) = compat(pkg, nothing; kwargs...) -compat(pkg::String, compat_str::Union{Nothing,String}; kwargs...) = compat(Context(), pkg, compat_str; kwargs...) -compat(;kwargs...) = compat(Context(); kwargs...) +function compat(ctx::Context=Context(); current::Bool=false, kwargs...) + if current + return set_current_compat(ctx; kwargs...) + end + return _compat(ctx; kwargs...) +end +compat(pkg::String, compat_str::Union{Nothing,String}=nothing; kwargs...) = _compat(Context(), pkg, compat_str; kwargs...) + + +function set_current_compat(ctx::Context, target_pkg::Union{Nothing,String}=nothing; io = nothing) + io = something(io, ctx.io) + updated_deps = String[] + + deps_to_process = if target_pkg !== nothing + # Process only the specified package + if haskey(ctx.env.project.deps, target_pkg) + [(target_pkg, ctx.env.project.deps[target_pkg])] + else + pkgerror("Package $(target_pkg) not found in project dependencies") + end + else + # Process all packages (existing behavior) + collect(ctx.env.project.deps) + end + + # Process regular package dependencies + for (dep, uuid) in deps_to_process + compat_str = Operations.get_compat_str(ctx.env.project, dep) + if target_pkg !== nothing || isnothing(compat_str) + entry = get(ctx.env.manifest, uuid, nothing) + entry === nothing && continue + v = entry.version + v === nothing && continue + pkgversion = string(Base.thispatch(v)) + Operations.set_compat(ctx.env.project, dep, pkgversion) || + pkgerror("invalid compat version specifier \"$(pkgversion)\"") + push!(updated_deps, dep) + end + end + + # Also handle Julia compat entry when processing all packages (not when targeting a specific package) + if target_pkg === nothing + julia_compat_str = Operations.get_compat_str(ctx.env.project, "julia") + if isnothing(julia_compat_str) + # Set julia compat to current running version + julia_version = string(Base.thispatch(VERSION)) + Operations.set_compat(ctx.env.project, "julia", julia_version) || + pkgerror("invalid compat version specifier \"$(julia_version)\"") + push!(updated_deps, "julia") + end + end + + # Update messaging + if isempty(updated_deps) + if target_pkg !== nothing + printpkgstyle(io, :Info, "$(target_pkg) already has a compat entry or is not in manifest. No changes made.", color = Base.info_color()) + else + printpkgstyle(io, :Info, "no missing compat entries found. No changes made.", color = Base.info_color()) + end + elseif length(updated_deps) == 1 + printpkgstyle(io, :Info, "new entry set for $(only(updated_deps)) based on its current version", color = Base.info_color()) + else + printpkgstyle(io, :Info, "new entries set for $(join(updated_deps, ", ", " and ")) based on their current versions", color = Base.info_color()) + end + + write_env(ctx.env) + Operations.print_compat(ctx; io) +end +set_current_compat(;kwargs...) = set_current_compat(Context(); kwargs...) ####### # why # diff --git a/src/REPLMode/command_declarations.jl b/src/REPLMode/command_declarations.jl index 4abcdfe0e8..cb9d459d31 100644 --- a/src/REPLMode/command_declarations.jl +++ b/src/REPLMode/command_declarations.jl @@ -461,11 +461,20 @@ PSA[:name => "compat", :api => API.compat, :arg_count => 0 => 2, :completions => :complete_installed_packages_and_compat, + :option_spec => [ + PSA[:name => "current", :api => :current => true], + ], :description => "edit compat entries in the current Project and re-resolve", :help => md""" compat [pkg] [compat_string] + compat + compat --current + compat --current Edit project [compat] entries directly, or via an interactive menu by not specifying any arguments. +Use --current flag to automatically populate missing compat entries with currently resolved versions. +When used alone, applies to all packages missing compat entries. +When combined with a package name, applies only to that package. When directly editing use tab to complete the package name and any existing compat entry. Specifying a package with a blank compat entry will remove the entry. After changing compat entries a `resolve` will be attempted to check whether the current diff --git a/test/new.jl b/test/new.jl index 3b1a0faa53..e8ac1eb569 100644 --- a/test/new.jl +++ b/test/new.jl @@ -2897,7 +2897,8 @@ end # @testset "Pkg.compat" begin # State changes - isolate(loaded_depot=true) do + isolate(loaded_depot=true) do; mktempdir() do tempdir + Pkg.activate(tempdir) Pkg.add("Example") iob = IOBuffer() Pkg.status(compat=true, io = iob) @@ -2923,9 +2924,74 @@ end @test occursin(r"Compat `.+Project.toml`", output) @test occursin(r"\[7876af07\] *Example *none", output) @test occursin(r"julia *1.8", output) - end + end end + + # Test compat --current functionality + isolate(loaded_depot=true) do; mktempdir() do tempdir + path = copy_test_package(tempdir, "SimplePackage") + Pkg.activate(path) + # Add Example - this will automatically set a compat entry + Pkg.add("Example") + + # Example now has a compat entry from the add operation + @test Pkg.Operations.get_compat_str(Pkg.Types.Context().env.project, "Example") !== nothing + + # Use compat current to set compat entry for packages without compat entries (like Markdown) + iob = IOBuffer() + Pkg.compat("Markdown", current=true, io=iob) + output = String(take!(iob)) + @test occursin("new entry set for Markdown based on its current version", output) + + Pkg.compat(current=true, io=iob) + output = String(take!(iob)) + @test occursin("new entry set for julia based on its current version", output) + + # Check that all compat entries are set + @test Pkg.Operations.get_compat_str(Pkg.Types.Context().env.project, "julia") !== nothing + @test Pkg.Operations.get_compat_str(Pkg.Types.Context().env.project, "Example") !== nothing + @test Pkg.Operations.get_compat_str(Pkg.Types.Context().env.project, "Markdown") !== nothing + + # Test with no missing compat entries + iob = IOBuffer() + Pkg.compat(current=true, io=iob) + output = String(take!(iob)) + @test occursin("no missing compat entries found. No changes made.", output) + end end + + # Test compat current with multiple packages + isolate(loaded_depot=true) do; mktempdir() do tempdir + path = copy_test_package(tempdir, "SimplePackage") + Pkg.activate(path) + # Add both packages - this will automatically set compat entries for them + Pkg.add("Example") + Pkg.add("JSON") + Pkg.compat("Example", nothing) + Pkg.compat("JSON", nothing) + + # Both packages now have compat entries from the add operations + @test Pkg.Operations.get_compat_str(Pkg.Types.Context().env.project, "Example") === nothing + @test Pkg.Operations.get_compat_str(Pkg.Types.Context().env.project, "JSON") === nothing + + # Use compat current to set compat entries for packages without compat entries (like Markdown) + iob = IOBuffer() + Pkg.compat(current=true, io=iob) + output = String(take!(iob)) + @test occursin("new entries set for", output) + @test occursin("julia", output) + @test occursin("Markdown", output) + @test occursin("Example", output) + @test occursin("JSON", output) + + # Check that all compat entries are set + @test Pkg.Operations.get_compat_str(Pkg.Types.Context().env.project, "julia") !== nothing + @test Pkg.Operations.get_compat_str(Pkg.Types.Context().env.project, "Example") !== nothing + @test Pkg.Operations.get_compat_str(Pkg.Types.Context().env.project, "JSON") !== nothing + @test Pkg.Operations.get_compat_str(Pkg.Types.Context().env.project, "Markdown") !== nothing + end end end +Pkg.activate() + # # # Caching From 10ae3894064eab8be436e013cca482989001eaab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnar=20Farneb=C3=A4ck?= Date: Wed, 9 Jul 2025 10:09:01 +0200 Subject: [PATCH 130/154] Update registry update log also when adding a registry by copying or symlinking. (#4314) --- src/Registry/Registry.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Registry/Registry.jl b/src/Registry/Registry.jl index 180b4855ac..5af6b8d1b5 100644 --- a/src/Registry/Registry.jl +++ b/src/Registry/Registry.jl @@ -228,6 +228,8 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depots::Union{S isfile(joinpath(regpath, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in linked registry.") registry = Registry.RegistryInstance(regpath) printpkgstyle(io, :Symlinked, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`") + registry_update_log[string(reg.uuid)] = now() + save_registry_update_log(registry_update_log) return elseif reg.url !== nothing && reg.linked == true Pkg.Types.pkgerror(""" @@ -251,6 +253,8 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depots::Union{S regpath = joinpath(regdir, registry.name) cp(reg.path, regpath; force=true) # has to be cp given we're copying printpkgstyle(io, :Copied, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`") + registry_update_log[string(reg.uuid)] = now() + save_registry_update_log(registry_update_log) return elseif reg.url !== nothing # clone from url # retry to help spurious connection issues, particularly on CI From 4197dd78fd97095dbea5283328e9a05fbd6445f0 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 9 Jul 2025 10:09:31 +0200 Subject: [PATCH 131/154] make more descriptive errors if `first` fails on empty urls when installing a git repo or finding installed registries (#4282) Co-authored-by: KristofferC --- src/Operations.jl | 16 ++++++++++++++-- src/Registry/Registry.jl | 8 ++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/Operations.jl b/src/Operations.jl index a2cd596b2a..85f05b4b45 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -838,6 +838,17 @@ function install_git( urls::Set{String}, version_path::String )::Nothing + if isempty(urls) + pkgerror( + "Package $name [$uuid] has no repository URL available. This could happen if:\n" * + " - The package is not registered in any configured registry\n" * + " - The package exists in a registry but lacks repository information\n" * + " - Registry files are corrupted or incomplete\n" * + " - Network issues prevented registry updates\n" * + "Please check that the package name is correct and that your registries are up to date." + ) + end + repo = nothing tree = nothing # TODO: Consolidate this with some of the repo handling in Types.jl @@ -845,8 +856,9 @@ function install_git( clones_dir = joinpath(depots1(), "clones") ispath(clones_dir) || mkpath(clones_dir) repo_path = joinpath(clones_dir, string(uuid)) - repo = GitTools.ensure_clone(io, repo_path, first(urls); isbare=true, - header = "[$uuid] $name from $(first(urls))") + first_url = first(urls) + repo = GitTools.ensure_clone(io, repo_path, first_url; isbare=true, + header = "[$uuid] $name from $first_url") git_hash = LibGit2.GitHash(hash.bytes) for url in urls try LibGit2.with(LibGit2.GitObject, repo, git_hash) do g diff --git a/src/Registry/Registry.jl b/src/Registry/Registry.jl index 5af6b8d1b5..962c8ce958 100644 --- a/src/Registry/Registry.jl +++ b/src/Registry/Registry.jl @@ -138,7 +138,9 @@ function populate_known_registries_with_urls!(registries::Vector{RegistrySpec}) elseif reg.name !== nothing if reg.name == known.name named_regs = filter(r -> r.name == reg.name, known_registries) - if !all(r -> r.uuid == first(named_regs).uuid, named_regs) + if isempty(named_regs) + Pkg.Types.pkgerror("registry with name `$(reg.name)` not found in known registries.") + elseif !all(r -> r.uuid == first(named_regs).uuid, named_regs) Pkg.Types.pkgerror("multiple registries with name `$(reg.name)`, please specify with uuid.") end reg.uuid = known.uuid @@ -347,7 +349,9 @@ function find_installed_registries(io::IO, elseif needle.name !== nothing if needle.name == candidate.name named_regs = filter(r -> r.name == needle.name, haystack) - if !all(r -> r.uuid == first(named_regs).uuid, named_regs) + if isempty(named_regs) + Pkg.Types.pkgerror("registry with name `$(needle.name)` not found in reachable registries.") + elseif !all(r -> r.uuid == first(named_regs).uuid, named_regs) Pkg.Types.pkgerror("multiple registries with name `$(needle.name)`, please specify with uuid.") end push!(output, candidate) From 98098f245dbed23eb3d2ce84dc65cfc7e410c1aa Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 9 Jul 2025 11:24:40 +0200 Subject: [PATCH 132/154] only try download artifacts from pkg server if the corresponding package is in a registry from it (#4297) Co-authored-by: KristofferC --- src/Artifacts.jl | 15 +++++++------ src/Operations.jl | 47 ++++++++++++++++++++++++---------------- src/Registry/Registry.jl | 21 ++++++++++++++++++ 3 files changed, 57 insertions(+), 26 deletions(-) diff --git a/src/Artifacts.jl b/src/Artifacts.jl index 83b71737c4..0f77f4cc12 100644 --- a/src/Artifacts.jl +++ b/src/Artifacts.jl @@ -434,6 +434,7 @@ Ensures an artifact is installed, downloading it via the download information st function ensure_artifact_installed(name::String, artifacts_toml::String; platform::AbstractPlatform = HostPlatform(), pkg_uuid::Union{Base.UUID,Nothing}=nothing, + pkg_server_eligible::Bool=true, verbose::Bool = false, quiet_download::Bool = false, progress::Union{Function,Nothing} = nothing, @@ -444,23 +445,23 @@ function ensure_artifact_installed(name::String, artifacts_toml::String; end return ensure_artifact_installed(name, meta, artifacts_toml; - platform, verbose, quiet_download, progress, io) + pkg_server_eligible, platform, verbose, quiet_download, progress, io) end function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::String; + pkg_server_eligible::Bool=true, platform::AbstractPlatform = HostPlatform(), verbose::Bool = false, quiet_download::Bool = false, progress::Union{Function,Nothing} = nothing, io::IO=stderr_f()) - hash = SHA1(meta["git-tree-sha1"]) if !artifact_exists(hash) if isnothing(progress) || verbose == true - return try_artifact_download_sources(name, hash, meta, artifacts_toml; platform, verbose, quiet_download, io) + return try_artifact_download_sources(name, hash, meta, artifacts_toml; pkg_server_eligible, platform, verbose, quiet_download, io) else # if a custom progress handler is given it is taken to mean the caller wants to handle the download scheduling - return () -> try_artifact_download_sources(name, hash, meta, artifacts_toml; platform, quiet_download=true, io, progress) + return () -> try_artifact_download_sources(name, hash, meta, artifacts_toml; pkg_server_eligible, platform, quiet_download=true, io, progress) end else return artifact_path(hash) @@ -469,6 +470,7 @@ end function try_artifact_download_sources( name::String, hash::SHA1, meta::Dict, artifacts_toml::String; + pkg_server_eligible::Bool=true, platform::AbstractPlatform=HostPlatform(), verbose::Bool=false, quiet_download::Bool=false, @@ -476,9 +478,8 @@ function try_artifact_download_sources( progress::Union{Function,Nothing}=nothing) errors = Any[] - # first try downloading from Pkg server - # TODO: only do this if Pkg server knows about this package - if (server = pkg_server()) !== nothing + # first try downloading from Pkg server if the Pkg server knows about this package + if pkg_server_eligible && (server = pkg_server()) !== nothing url = "$server/artifact/$hash" download_success = let url = url @debug "Downloading artifact from Pkg server" name artifacts_toml platform url diff --git a/src/Operations.jl b/src/Operations.jl index 85f05b4b45..a7e626d863 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -936,15 +936,23 @@ function download_artifacts(ctx::Context; env = ctx.env io = ctx.io fancyprint = can_fancyprint(io) - pkg_roots = String[] + pkg_info = Tuple{String, Union{Base.UUID, Nothing}}[] for (uuid, pkg) in env.manifest pkg = manifest_info(env.manifest, uuid) pkg_root = source_path(env.manifest_file, pkg, julia_version) - pkg_root === nothing || push!(pkg_roots, pkg_root) + pkg_root === nothing || push!(pkg_info, (pkg_root, uuid)) end - push!(pkg_roots, dirname(env.project_file)) + push!(pkg_info, (dirname(env.project_file), env.pkg !== nothing ? env.pkg.uuid : nothing)) download_jobs = Dict{SHA1, Function}() + # Check what registries the current pkg server tracks + # Disable if precompiling to not access internet + server_registry_info = if Base.JLOptions().incremental == 0 + Registry.pkg_server_registry_info() + else + nothing + end + print_lock = Base.ReentrantLock() # for non-fancyprint printing download_states = Dict{SHA1, DownloadState}() @@ -958,12 +966,17 @@ function download_artifacts(ctx::Context; ansi_enablecursor = "\e[?25h" ansi_disablecursor = "\e[?25l" - all_collected_artifacts = reduce(vcat, map(pkg_root -> collect_artifacts(pkg_root; platform, include_lazy), pkg_roots)) - used_artifact_tomls = Set{String}(map(first, all_collected_artifacts)) - longest_name_length = maximum(all_collected_artifacts; init=0) do (artifacts_toml, artifacts) - maximum(textwidth, keys(artifacts); init=0) + all_collected_artifacts = reduce( + vcat, map( + ((pkg_root, pkg_uuid),) -> + map(ca -> (ca[1], ca[2], pkg_uuid), collect_artifacts(pkg_root; platform, include_lazy)), pkg_info + ) + ) + used_artifact_tomls = Set{String}(map(ca -> ca[1], all_collected_artifacts)) + longest_name_length = maximum(all_collected_artifacts; init = 0) do (artifacts_toml, artifacts, pkg_uuid) + maximum(textwidth, keys(artifacts); init = 0) end - for (artifacts_toml, artifacts) in all_collected_artifacts + for (artifacts_toml, artifacts, pkg_uuid) in all_collected_artifacts # For each Artifacts.toml, install each artifact we've collected from it for name in keys(artifacts) local rname = rpad(name, longest_name_length) @@ -981,9 +994,12 @@ function download_artifacts(ctx::Context; dstate.status_update_time = t end end + # Check if the current package is eligible for PkgServer artifact downloads + local pkg_server_eligible = pkg_uuid !== nothing && Registry.is_pkg_in_pkgserver_registry(pkg_uuid, server_registry_info, ctx.registries) + # returns a string if exists, or function that downloads the artifact if not local ret = ensure_artifact_installed(name, artifacts[name], artifacts_toml; - verbose, quiet_download=!(usable_io(io)), io, progress) + pkg_server_eligible, verbose, quiet_download=!(usable_io(io)), io, progress) if ret isa Function download_states[hash] = dstate download_jobs[hash] = @@ -1175,17 +1191,10 @@ function download_source(ctx::Context, pkgs; readonly=true) archive_urls = Pair{String,Bool}[] # Check if the current package is available in one of the registries being tracked by the pkg server # In that case, download from the package server - if server_registry_info !== nothing + if Registry.is_pkg_in_pkgserver_registry(pkg.uuid, server_registry_info, ctx.registries) server, registry_info = server_registry_info - for reg in ctx.registries - if reg.uuid in keys(registry_info) - if haskey(reg, pkg.uuid) - url = "$server/package/$(pkg.uuid)/$(pkg.tree_hash)" - push!(archive_urls, url => true) - break - end - end - end + url = "$server/package/$(pkg.uuid)/$(pkg.tree_hash)" + push!(archive_urls, url => true) end for repo_url in urls url = get_archive_url_for_version(repo_url, pkg.tree_hash) diff --git a/src/Registry/Registry.jl b/src/Registry/Registry.jl index 962c8ce958..44b39e892d 100644 --- a/src/Registry/Registry.jl +++ b/src/Registry/Registry.jl @@ -103,6 +103,27 @@ end pkg_server_url_hash(url::String) = Base.SHA1(split(url, '/')[end]) +""" + is_pkg_in_pkgserver_registry(pkg_uuid::Base.UUID, server_registry_info, registries) + +Check if a package UUID is tracked by the PkgServer by verifying it exists in +a registry that is known to the PkgServer. +""" +function is_pkg_in_pkgserver_registry(pkg_uuid::Base.UUID, server_registry_info, registries) + server_registry_info === nothing && return false + registries === nothing && return false + + server, registry_info = server_registry_info + for reg in registries + if reg.uuid in keys(registry_info) + if haskey(reg, pkg_uuid) + return true + end + end + end + return false +end + function download_default_registries(io::IO; only_if_empty::Bool = true, depots::Union{String, Vector{String}}=depots()) # Check the specified depots for installed registries installed_registries = reachable_registries(; depots) From 7f4e1700b1fd97e738f2c875e79f6e74589bfa3a Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 9 Jul 2025 12:19:49 +0200 Subject: [PATCH 133/154] Fix typos and inconsistencies in documentation (#4315) Co-authored-by: KristofferC Co-authored-by: Claude --- docs/src/artifacts.md | 2 +- docs/src/basedocs.md | 2 +- docs/src/compatibility.md | 2 +- docs/src/creating-packages.md | 2 +- docs/src/environments.md | 10 +++++----- docs/src/getting-started.md | 4 ++-- docs/src/glossary.md | 4 ++-- docs/src/managing-packages.md | 2 +- docs/src/toml-files.md | 4 ++-- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/src/artifacts.md b/docs/src/artifacts.md index d804dfeb10..d5fe5f38b7 100644 --- a/docs/src/artifacts.md +++ b/docs/src/artifacts.md @@ -230,7 +230,7 @@ This is deduced automatically by the `artifacts""` string macro, however, if you !!! compat "Julia 1.7" Pkg's extended platform selection requires at least Julia 1.7, and is considered experimental. -New in Julia 1.6, `Platform` objects can have extended attributes applied to them, allowing artifacts to be tagged with things such as CUDA driver version compatibility, microarchitectural compatibility, julia version compatibility and more! +New in Julia 1.7, `Platform` objects can have extended attributes applied to them, allowing artifacts to be tagged with things such as CUDA driver version compatibility, microarchitectural compatibility, julia version compatibility and more! Note that this feature is considered experimental and may change in the future. If you as a package developer find yourself needing this feature, please get in contact with us so it can evolve for the benefit of the whole ecosystem. In order to support artifact selection at `Pkg.add()` time, `Pkg` will run the specially-named file `/.pkg/select_artifacts.jl`, passing the current platform triplet as the first argument. diff --git a/docs/src/basedocs.md b/docs/src/basedocs.md index 7d51728ffe..9e07aa4ca9 100644 --- a/docs/src/basedocs.md +++ b/docs/src/basedocs.md @@ -4,7 +4,7 @@ EditURL = "https://github.com/JuliaLang/Pkg.jl/blob/master/docs/src/basedocs.md" # Pkg -Pkg is Julia's builtin package manager, and handles operations +Pkg is Julia's built-in package manager, and handles operations such as installing, updating and removing packages. !!! note diff --git a/docs/src/compatibility.md b/docs/src/compatibility.md index 8115eea833..dee8b05841 100644 --- a/docs/src/compatibility.md +++ b/docs/src/compatibility.md @@ -22,7 +22,7 @@ The format of the version specifier is described in detail below. The rules below apply to the `Project.toml` file; for registries, see [Registry Compat.toml](@ref). !!! info - Note that registration into Julia's General Registry requires each dependency to have a `[compat`] entry with an upper bound. + Note that registration into Julia's General Registry requires each dependency to have a `[compat]` entry with an upper bound. ## Version specifier format diff --git a/docs/src/creating-packages.md b/docs/src/creating-packages.md index 2cee5bf6cc..2850008925 100644 --- a/docs/src/creating-packages.md +++ b/docs/src/creating-packages.md @@ -127,7 +127,7 @@ just as public as those marked as public with the `export` keyword, but when fol `YourPackage.that_symbol`. Let's say we would like our `greet` function to be part of the public API, but not the -`greet_alien` function. We could the write the following and release it as version `1.0.0`. +`greet_alien` function. We could then write the following and release it as version `1.0.0`. ```julia module HelloWorld diff --git a/docs/src/environments.md b/docs/src/environments.md index d22ef8e58b..293dcccb0b 100644 --- a/docs/src/environments.md +++ b/docs/src/environments.md @@ -4,7 +4,7 @@ The following discusses Pkg's interaction with environments. For more on the rol ## Creating your own environments -So far we have added packages to the default environment at `~/.julia/environments/v1.9`. It is however easy to create other, independent, projects. +So far we have added packages to the default environment at `~/.julia/environments/v1.10`. It is however easy to create other, independent, projects. This approach has the benefit of allowing you to check in a `Project.toml`, and even a `Manifest.toml` if you wish, into version control (e.g. git) alongside your code. It should be pointed out that when two projects use the same package at the same version, the content of this package is not duplicated. In order to create a new project, create a directory for it and then activate that directory to make it the "active project", which package operations manipulate: @@ -117,12 +117,12 @@ between several incompatible packages. ## Shared environments -A "shared" environment is simply an environment that exists in `~/.julia/environments`. The default `v1.9` environment is +A "shared" environment is simply an environment that exists in `~/.julia/environments`. The default `v1.10` environment is therefore a shared environment: ```julia-repl (@v1.10) pkg> st -Status `~/.julia/environments/v1.9/Project.toml` +Status `~/.julia/environments/v1.10/Project.toml` ``` Shared environments can be activated with the `--shared` flag to `activate`: @@ -167,9 +167,9 @@ with its dependencies. ```julia-repl (@v1.10) pkg> add Images Resolving package versions... - Updating `~/.julia/environments/v1.9/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [916415d5] + Images v0.25.2 - Updating `~/.julia/environments/v1.9/Manifest.toml` + Updating `~/.julia/environments/v1.10/Manifest.toml` ... Precompiling environment... Progress [===================> ] 45/97 diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 93acb7c613..d822ebd3fa 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -101,8 +101,8 @@ For more information about managing packages, see the [Managing Packages](@ref M Up to this point, we have covered basic package management: adding, updating, and removing packages. -You may have noticed the `(@v1.9)` in the REPL prompt. -This lets us know that `v1.9` is the **active environment**. +You may have noticed the `(@v1.10)` in the REPL prompt. +This lets us know that `v1.10` is the **active environment**. Different environments can have totally different packages and versions installed from another environment. The active environment is the environment that will be modified by Pkg commands such as `add`, `rm` and `update`. diff --git a/docs/src/glossary.md b/docs/src/glossary.md index 54c00aa8ea..4914150ff1 100644 --- a/docs/src/glossary.md +++ b/docs/src/glossary.md @@ -14,8 +14,8 @@ may optionally have a manifest file: - **Manifest file:** a file in the root directory of a project, named `Manifest.toml` (or `JuliaManifest.toml`), describing a complete dependency graph and exact versions of each package and library used by a project. The file name may - also be suffixed by `-v{major}.{minor}.toml` which julia will prefer if the version - matches `VERSION`, allowing multiple environments to be maintained for different julia + also be suffixed by `-v{major}.{minor}.toml` which Julia will prefer if the version + matches `VERSION`, allowing multiple environments to be maintained for different Julia versions. **Package:** a project which provides reusable functionality that can be used by diff --git a/docs/src/managing-packages.md b/docs/src/managing-packages.md index 37805ae7e6..3295080ef7 100644 --- a/docs/src/managing-packages.md +++ b/docs/src/managing-packages.md @@ -28,7 +28,7 @@ Precompiling environment... 2 dependencies successfully precompiled in 2 seconds ``` -Here we added the package `JSON` to the current environment (which is the default `@v1.8` environment). +Here we added the package `JSON` to the current environment (which is the default `@v1.10` environment). In this example, we are using a fresh Julia installation, and this is our first time adding a package using Pkg. By default, Pkg installs the General registry and uses this registry to look up packages requested for inclusion in the current environment. diff --git a/docs/src/toml-files.md b/docs/src/toml-files.md index 4ad1bc569d..a6363d2dc3 100644 --- a/docs/src/toml-files.md +++ b/docs/src/toml-files.md @@ -24,7 +24,7 @@ are described below. For a package, the optional `authors` field is a TOML array describing the package authors. Entries in the array can either be a string in the form `"NAME"` or `"NAME "`, or a table keys following the [Citation File Format schema](https://github.com/citation-file-format/citation-file-format/blob/main/schema-guide.md) for either a -[`person`](https://github.com/citation-file-format/citation-file-format/blob/main/schema-guide.md#definitionsperson) or an[`entity`](https://github.com/citation-file-format/citation-file-format/blob/main/schema-guide.md#definitionsentity). +[`person`](https://github.com/citation-file-format/citation-file-format/blob/main/schema-guide.md#definitionsperson) or an [`entity`](https://github.com/citation-file-format/citation-file-format/blob/main/schema-guide.md#definitionsentity). For example: ```toml @@ -130,7 +130,7 @@ handled by Pkg operations such as `add`. ### The `[sources]` section -Specifiying a path or repo (+ branch) for a dependency is done in the `[sources]` section. +Specifying a path or repo (+ branch) for a dependency is done in the `[sources]` section. These are especially useful for controlling unregistered dependencies without having to bundle a corresponding manifest file. From 61a84472a5ddeac97afb1c19651febfbb8519207 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 12 Jul 2025 00:16:21 -0400 Subject: [PATCH 134/154] Add more autoprecompilation controls and `Pkg.precompile() do` to delay precompilation until after operations. (#4262) --- CHANGELOG.md | 2 ++ docs/src/api.md | 1 + docs/src/environments.md | 43 ++++++++++++++++++++++++++-- src/API.jl | 9 +++++- src/Pkg.jl | 61 +++++++++++++++++++++++++++++++++++++++- test/api.jl | 61 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 172 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0cc2664fc..eada20d58e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ Pkg v1.13 Release Notes - `update` now shows a helpful tip when trying to upgrade a specific package that can be upgraded but is held back because it's part of a less optimal resolver solution ([#4266]) - `Pkg.status` now displays yanked packages with a `[yanked]` indicator and shows a warning when yanked packages are present. `Pkg.resolve` errors also display warnings about yanked packages that are not resolvable. ([#4310]) - Added `pkg> compat --current` command to automatically populate missing compat entries with the currently resolved package versions. Use `pkg> compat --current` for all packages or `pkg> compat Foo --current` for specific packages. ([#3266]) +- Added `Pkg.precompile() do` block syntax to delay autoprecompilation until after multiple operations complete, improving efficiency when performing several environment changes. ([#4262]) +- Added `Pkg.autoprecompilation_enabled(state::Bool)` to globally enable or disable automatic precompilation for Pkg operations. ([#4262]) Pkg v1.12 Release Notes ======================= diff --git a/docs/src/api.md b/docs/src/api.md index afe34d8635..d87169077f 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -39,6 +39,7 @@ Pkg.gc Pkg.status Pkg.compat Pkg.precompile +Pkg.autoprecompilation_enabled Pkg.offline Pkg.why Pkg.dependencies diff --git a/docs/src/environments.md b/docs/src/environments.md index 293dcccb0b..1464f5db60 100644 --- a/docs/src/environments.md +++ b/docs/src/environments.md @@ -190,16 +190,53 @@ If a given package version errors during auto-precompilation, Pkg will remember automatically tries and will skip that package with a brief warning. Manual precompilation can be used to force these packages to be retried, as `pkg> precompile` will always retry all packages. -To disable the auto-precompilation, set `ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0`. - The indicators next to the package names displayed during precompilation -indicate the status of that package's precompilation. +indicate the status of that package's precompilation. - `[◐, ◓, ◑, ◒]` Animated "clock" characters indicate that the package is currently being precompiled. - `✓` A green checkmark indicates that the package has been successfully precompiled (after which that package will disappear from the list). If the checkmark is yellow it means that the package is currently loaded so the session will need to be restarted to access the version that was just precompiled. - `?` A question mark character indicates that a `PrecompilableError` was thrown, indicating that precompilation was disallowed, i.e. `__precompile__(false)` in that package. - `✗` A cross indicates that the package failed to precompile. +#### Controlling Auto-precompilation + +Auto-precompilation can be controlled in several ways: + +- **Environment variable**: Set `ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0` to disable auto-precompilation globally. +- **Programmatically**: Use `Pkg.autoprecompilation_enabled(false)` to disable auto-precompilation for the current session, or `Pkg.autoprecompilation_enabled(true)` to re-enable it. +- **Scoped control**: Use `Pkg.precompile(f, args...; kwargs...)` to execute a function `f` with auto-precompilation temporarily disabled, then automatically trigger precompilation afterward if any packages were modified during the execution. + +!!! compat "Julia 1.13" + The `Pkg.autoprecompilation_enabled()` function and `Pkg.precompile()` do-block syntax require at least Julia 1.13. + +For example, to add multiple packages without triggering precompilation after each one: + +```julia-repl +julia> Pkg.precompile() do + Pkg.add("Example") + Pkg.dev("JSON") + Pkg.update("HTTP") + end + Resolving package versions... + ... +Precompiling environment... + 14 dependencies successfully precompiled in 25 seconds +``` + +Or to temporarily disable auto-precompilation: + +```julia-repl +julia> Pkg.autoprecompilation_enabled(false) +false + +julia> Pkg.add("Example") # No precompilation happens + Resolving package versions... + ... + +julia> Pkg.autoprecompilation_enabled(true) +true +``` + ### Precompiling new versions of loaded packages If a package that has been updated is already loaded in the session, the precompilation process will go ahead and precompile diff --git a/src/API.jl b/src/API.jl index da2cb0a23a..bc913abad0 100644 --- a/src/API.jl +++ b/src/API.jl @@ -12,7 +12,7 @@ import FileWatching import Base: StaleCacheKey -import ..depots, ..depots1, ..logdir, ..devdir, ..printpkgstyle +import ..depots, ..depots1, ..logdir, ..devdir, ..printpkgstyle, .._autoprecompilation_enabled_scoped import ..Operations, ..GitTools, ..Pkg, ..Registry import ..can_fancyprint, ..pathrepr, ..isurl, ..PREV_ENV_PATH, ..atomic_toml_write using ..Types, ..TOML @@ -1185,6 +1185,13 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool end end +function precompile(f, args...; kwargs...) + Base.ScopedValues.@with _autoprecompilation_enabled_scoped => false begin + f() + Pkg.precompile(args...; kwargs...) + end +end + function tree_hash(repo::LibGit2.GitRepo, tree_hash::String) try return LibGit2.GitObject(repo, tree_hash) diff --git a/src/Pkg.jl b/src/Pkg.jl index 8ba2f4e107..2b0085c00b 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -70,7 +70,20 @@ const PREV_ENV_PATH = Ref{String}("") usable_io(io) = (io isa Base.TTY) || (io isa IOContext{IO} && io.io isa Base.TTY) can_fancyprint(io::IO) = (usable_io(io)) && (get(ENV, "CI", nothing) != "true") -should_autoprecompile() = Base.JLOptions().use_compiled_modules == 1 && Base.get_bool_env("JULIA_PKG_PRECOMPILE_AUTO", true) + +_autoprecompilation_enabled::Bool = true +const _autoprecompilation_enabled_scoped = Base.ScopedValues.ScopedValue{Bool}(true) +autoprecompilation_enabled(state::Bool) = (global _autoprecompilation_enabled = state) +function should_autoprecompile() + if Base.JLOptions().use_compiled_modules == 1 && + _autoprecompilation_enabled && + _autoprecompilation_enabled_scoped[] && + Base.get_bool_env("JULIA_PKG_PRECOMPILE_AUTO", true) + return true + else + return false + end +end """ in_repl_mode() @@ -206,11 +219,21 @@ const add = API.add Pkg.precompile(; strict::Bool=false, timing::Bool=false) Pkg.precompile(pkg; strict::Bool=false, timing::Bool=false) Pkg.precompile(pkgs; strict::Bool=false, timing::Bool=false) + Pkg.precompile(f, args...; kwargs...) Precompile all or specific dependencies of the project in parallel. Set `timing=true` to show the duration of the precompilation of each dependency. +To delay autoprecompilation of multiple Pkg actions until the end use. +This may be most efficient while manipulating the environment in various ways. + +```julia +Pkg.precompile() do + # Pkg actions here +end +``` + !!! note Errors will only throw when precompiling the top-level dependencies, given that not all manifest dependencies may be loaded by the top-level dependencies on the given system. @@ -228,6 +251,9 @@ Set `timing=true` to show the duration of the precompilation of each dependency. !!! compat "Julia 1.9" Timing mode requires at least Julia 1.9. +!!! compat "Julia 1.13" + The `Pkg.precompile(f, args...; kwargs...)` do-block syntax requires at least Julia 1.13. + # Examples ```julia Pkg.precompile() @@ -237,6 +263,39 @@ Pkg.precompile(["Foo", "Bar"]) """ const precompile = API.precompile +""" + Pkg.autoprecompilation_enabled(state::Bool) + +Enable or disable automatic precompilation for Pkg operations. + +When `state` is `true` (default), Pkg operations that modify the project environment +will automatically trigger precompilation of affected packages. When `state` is `false`, +automatic precompilation is disabled and packages will only be precompiled when +explicitly requested via [`Pkg.precompile`](@ref). + +This setting affects the global state and persists across Pkg operations in the same +Julia session. It can be used in combination with [`Pkg.precompile`](@ref) do-syntax +for more fine-grained control over when precompilation occurs. + +!!! compat "Julia 1.13" + This function requires at least Julia 1.13. + +# Examples +```julia +# Disable automatic precompilation +Pkg.autoprecompilation_enabled(false) +Pkg.add("Example") # Will not trigger auto-precompilation +Pkg.precompile() # Manual precompilation + +# Re-enable automatic precompilation +Pkg.autoprecompilation_enabled(true) +Pkg.add("AnotherPackage") # Will trigger auto-precompilation +``` + +See also [`Pkg.precompile`](@ref). +""" +autoprecompilation_enabled + """ Pkg.rm(pkg::Union{String, Vector{String}}; mode::PackageMode = PKGMODE_PROJECT) Pkg.rm(pkg::Union{PackageSpec, Vector{PackageSpec}}; mode::PackageMode = PKGMODE_PROJECT) diff --git a/test/api.jl b/test/api.jl index 65c40ca37e..a298cdce56 100644 --- a/test/api.jl +++ b/test/api.jl @@ -158,6 +158,67 @@ import .FakeTerminals.FakeTerminal @test !occursin("Precompiling", String(take!(iob))) # test that the previous precompile was a no-op end + dep8_path = git_init_package(tmp, joinpath("packages", "Dep8")) + function clear_dep8_cache() + rm(joinpath(Pkg.depots1(), "compiled", "v$(VERSION.major).$(VERSION.minor)", "Dep8"), force=true, recursive=true) + end + @testset "delayed precompilation with do-syntax" begin + iob = IOBuffer() + # Test that operations inside Pkg.precompile() do block don't trigger auto-precompilation + Pkg.precompile(io=iob) do + Pkg.add(Pkg.PackageSpec(path=dep8_path)) + Pkg.rm("Dep8") + clear_dep8_cache() + Pkg.add(Pkg.PackageSpec(path=dep8_path)) + end + + # The precompile should happen once at the end + @test count(r"Precompiling", String(take!(iob))) == 1 # should only precompile once + + # Verify it was precompiled by checking a second call is a no-op + Pkg.precompile(io=iob) + @test !occursin("Precompiling", String(take!(iob))) + end + + Pkg.rm("Dep8") + + @testset "autoprecompilation_enabled global control" begin + iob = IOBuffer() + withenv("JULIA_PKG_PRECOMPILE_AUTO" => nothing) do + original_state = Pkg._autoprecompilation_enabled + try + Pkg.autoprecompilation_enabled(false) + @test Pkg._autoprecompilation_enabled == false + + # Operations should not trigger autoprecompilation when globally disabled + clear_dep8_cache() + Pkg.add(Pkg.PackageSpec(path=dep8_path), io=iob) + @test !occursin("Precompiling", String(take!(iob))) + + # Manual precompile should still work + @test Base.isprecompiled(Base.identify_package("Dep8")) == false + Pkg.precompile(io=iob) + @test occursin("Precompiling", String(take!(iob))) + @test Base.isprecompiled(Base.identify_package("Dep8")) + + # Re-enable autoprecompilation + Pkg.autoprecompilation_enabled(true) + @test Pkg._autoprecompilation_enabled == true + + # Operations should now trigger autoprecompilation again + Pkg.rm("Dep8", io=iob) + clear_dep8_cache() + Pkg.add(Pkg.PackageSpec(path=dep8_path), io=iob) + @test Base.isprecompiled(Base.identify_package("Dep8")) + @test occursin("Precompiling", String(take!(iob))) + + finally + # Restore original state + Pkg.autoprecompilation_enabled(original_state) + end + end + end + @testset "instantiate" begin iob = IOBuffer() Pkg.activate("packages/Dep7") From 92440c651b4f181cba4078cff41e2e16797a0837 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 12 Jul 2025 00:55:36 -0400 Subject: [PATCH 135/154] Fix lack of color in CI and don't warn about loaded packages in test Pkg.precompile. (#4316) --- src/API.jl | 6 ++++-- src/Operations.jl | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/API.jl b/src/API.jl index bc913abad0..9fefecb380 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1172,10 +1172,12 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool end io = ctx.io - if io isa IOContext{IO} + if io isa IOContext{IO} && !isa(io.io, Base.PipeEndpoint) # precompile does quite a bit of output and using the IOContext{IO} can cause # some slowdowns, the important part here is to not specialize the whole - # precompile function on the io + # precompile function on the io. + # But don't unwrap the IOContext if it is a PipeEndpoint, as that would + # cause the output to lose color. io = io.io end diff --git a/src/Operations.jl b/src/Operations.jl index a7e626d863..1dbc82a6fc 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -2476,7 +2476,9 @@ function test(ctx::Context, pkgs::Vector{PackageSpec}; if should_autoprecompile() cacheflags = Base.CacheFlags(parse(UInt8, read(`$(Base.julia_cmd()) $(flags) --eval 'show(ccall(:jl_cache_flags, UInt8, ()))'`, String))) - Pkg.precompile(; io=ctx.io, configs = flags => cacheflags) + # Don't warn about already loaded packages, since we are going to run tests in a new + # subprocess anyway. + Pkg.precompile(; io=ctx.io, warn_loaded = false, configs = flags => cacheflags) end printpkgstyle(ctx.io, :Testing, "Running tests...") From 48d557705b0d9acc30487ea3211574cb2a0e8100 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 12 Jul 2025 16:38:49 -0400 Subject: [PATCH 136/154] Add missing recent changelogs (#4317) --- CHANGELOG.md | 101 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 87 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eada20d58e..6b9c934159 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,28 +1,69 @@ Pkg v1.13 Release Notes ======================= -- Project.toml environments now support a `readonly` field to mark environments as read-only, preventing modifications. ([#4284]) -- `Pkg.build` now supports an `allow_reresolve` keyword argument to control whether the build process can re-resolve package versions, similar to the existing option for `Pkg.test`. ([#3329]) -- Packages are now automatically added to `[sources]` when they are added by url or devved. -- `update` now shows a helpful tip when trying to upgrade a specific package that can be upgraded but is held back because it's part of a less optimal resolver solution ([#4266]) -- `Pkg.status` now displays yanked packages with a `[yanked]` indicator and shows a warning when yanked packages are present. `Pkg.resolve` errors also display warnings about yanked packages that are not resolvable. ([#4310]) -- Added `pkg> compat --current` command to automatically populate missing compat entries with the currently resolved package versions. Use `pkg> compat --current` for all packages or `pkg> compat Foo --current` for specific packages. ([#3266]) -- Added `Pkg.precompile() do` block syntax to delay autoprecompilation until after multiple operations complete, improving efficiency when performing several environment changes. ([#4262]) -- Added `Pkg.autoprecompilation_enabled(state::Bool)` to globally enable or disable automatic precompilation for Pkg operations. ([#4262]) +- Project.toml environments now support a `readonly` field to mark environments as read-only, preventing modifications. + ([#4284]) +- `Pkg.build` now supports an `allow_reresolve` keyword argument to control whether the build process can re-resolve + package versions, similar to the existing option for `Pkg.test`. ([#3329]) +- Packages are now automatically added to `[sources]` when they are added by url or devved. ([#4225]) +- `update` now shows a helpful tip when trying to upgrade a specific package that can be upgraded but is held back + because it's part of a less optimal resolver solution ([#4266]) +- `Pkg.status` now displays yanked packages with a `[yanked]` indicator and shows a warning when yanked packages are + present. `Pkg.resolve` errors also display warnings about yanked packages that are not resolvable. ([#4310]) +- Added `pkg> compat --current` command to automatically populate missing compat entries with the currently resolved + package versions. Use `pkg> compat --current` for all packages or `pkg> compat Foo --current` for specific packages. + ([#3266]) +- Added `Pkg.precompile() do` block syntax to delay autoprecompilation until after multiple operations complete, + improving efficiency when performing several environment changes. ([#4262]) +- Added `Pkg.autoprecompilation_enabled(state::Bool)` to globally enable or disable automatic precompilation for Pkg + operations. ([#4262]) +- Implemented atomic TOML writes to prevent data corruption when Pkg operations are interrupted or multiple processes + write simultaneously. All TOML files are now written atomically using temporary files and atomic moves. ([#4293]) +- Implemented lazy loading for RegistryInstance to significantly improve startup performance for operations that don't + require full registry data. This reduces `Pkg.instantiate()` time by approximately 60% in many cases. ([#4304]) +- Added support for directly adding git submodules via `Pkg.add(path="/path/to/git-submodule.jl")`. ([#3344]) +- Enhanced REPL user experience by automatically detecting and stripping accidental leading `]` characters in commands. + ([#3122]) +- Improved tip messages to show REPL mode syntax when operating in REPL mode. ([#3854]) +- Enhanced error handling with more descriptive error messages when operations fail on empty URLs during git repository + installation or registry discovery. ([#4282]) +- Improved error messages for invalid compat entries to provide better guidance for fixing them. ([#4302]) +- Added warnings when attempting to add local paths that contain dirty git repositories. ([#4309]) +- Enhanced package parsing to better handle complex URLs and paths with branch/tag/subdir specifiers. ([#4299]) +- Improved artifact download behavior to only attempt downloads from the Pkg server when the package is registered on + that server's registries. ([#4297]) +- Added comprehensive documentation page about depots, including depot layouts and configuration. ([#2245]) +- Enhanced error handling for packages missing from registries or manifests with more informative messages. ([#4303]) +- Added more robust error handling when packages have revisions but no source information. ([#4311]) +- Enhanced registry status reporting with more detailed information. ([#4300]) +- Fixed various edge cases in package resolution and manifest handling. ([#4307], [#4308], [#4312]) +- Improved handling of path separators across different operating systems. ([#4305]) +- Added better error messages when accessing private PackageSpec.repo field. ([#4170]) Pkg v1.12 Release Notes ======================= - Pkg now has support for "workspaces" which is a way to resolve multiple project files into a single manifest. - The functions `Pkg.status`, `Pkg.why`, `Pkg.instantiate`, `Pkg.precompile` (and their REPL variants) have been updated - to take a `workspace` option. Read more about this feature in the manual about the TOML-files. ([#3841]) + The functions `Pkg.status`, `Pkg.why`, `Pkg.instantiate`, `Pkg.precompile` (and their REPL variants) have been + updated to take a `workspace` option, with fixes for workspace path collection and package resolution in workspace + environments. Read more about this feature in the manual about the TOML-files. ([#3841], [#4229]) - Pkg now supports "apps" which are Julia packages that can be run directly from the terminal after installation. - Apps can be defined in a package's Project.toml and installed via Pkg. ([#3772]) -- `status` now shows when different versions/sources of dependencies are loaded than that which is expected by the manifest ([#4109]) + Apps can be defined in a package's Project.toml and installed via Pkg. Apps now support multiple apps per package + via submodules, allowing packages to define multiple command-line applications, with enhanced functionality including + update capabilities and better handling of already installed apps. ([#3772], [#4277], [#4263]) +- `status` now shows when different versions/sources of dependencies are loaded than that which is expected by the + manifest ([#4109]) - When adding or developing a package that exists in the `[weakdeps]` section, it is now automatically removed from weak dependencies and added as a regular dependency. ([#3865]) -- Enhanced fuzzy matching algorithm for package name suggestions. -- The Pkg REPL now supports GitHub pull request URLs, allowing direct package installation from PRs via `pkg> add https://github.com/Org/Package.jl/pull/123` ([#4295]) +- Enhanced fuzzy matching algorithm for package name suggestions with improved multi-factor scoring for better package + name suggestions. ([#4287]) +- The Pkg REPL now supports GitHub pull request URLs, allowing direct package installation from PRs via + `pkg> add https://github.com/Org/Package.jl/pull/123` ([#4295]) +- Improved git repository cloning performance by changing from `refs/*` to `refs/heads/*` to speed up operations on + repositories with many branches. ([#2330]) +- Improved REPL command parsing to handle leading whitespace with comma-separated packages. ([#4274]) +- Improved error messages when providing incorrect package UUIDs. ([#4270]) +- Added confirmation prompts before removing compat entries to prevent accidental deletions. ([#4254]) Pkg v1.11 Release Notes ======================= @@ -131,3 +172,35 @@ Pkg v1.7 Release Notes [#3002]: https://github.com/JuliaLang/Pkg.jl/issues/3002 [#3021]: https://github.com/JuliaLang/Pkg.jl/issues/3021 [#3266]: https://github.com/JuliaLang/Pkg.jl/pull/3266 +[#4266]: https://github.com/JuliaLang/Pkg.jl/pull/4266 +[#4310]: https://github.com/JuliaLang/Pkg.jl/pull/4310 +[#3329]: https://github.com/JuliaLang/Pkg.jl/pull/3329 +[#4262]: https://github.com/JuliaLang/Pkg.jl/pull/4262 +[#4293]: https://github.com/JuliaLang/Pkg.jl/pull/4293 +[#4304]: https://github.com/JuliaLang/Pkg.jl/pull/4304 +[#3344]: https://github.com/JuliaLang/Pkg.jl/pull/3344 +[#2330]: https://github.com/JuliaLang/Pkg.jl/pull/2330 +[#3122]: https://github.com/JuliaLang/Pkg.jl/pull/3122 +[#3854]: https://github.com/JuliaLang/Pkg.jl/pull/3854 +[#4282]: https://github.com/JuliaLang/Pkg.jl/pull/4282 +[#4302]: https://github.com/JuliaLang/Pkg.jl/pull/4302 +[#4309]: https://github.com/JuliaLang/Pkg.jl/pull/4309 +[#4299]: https://github.com/JuliaLang/Pkg.jl/pull/4299 +[#4295]: https://github.com/JuliaLang/Pkg.jl/pull/4295 +[#4277]: https://github.com/JuliaLang/Pkg.jl/pull/4277 +[#4297]: https://github.com/JuliaLang/Pkg.jl/pull/4297 +[#2245]: https://github.com/JuliaLang/Pkg.jl/pull/2245 +[#4303]: https://github.com/JuliaLang/Pkg.jl/pull/4303 +[#4254]: https://github.com/JuliaLang/Pkg.jl/pull/4254 +[#4270]: https://github.com/JuliaLang/Pkg.jl/pull/4270 +[#4263]: https://github.com/JuliaLang/Pkg.jl/pull/4263 +[#4229]: https://github.com/JuliaLang/Pkg.jl/pull/4229 +[#4274]: https://github.com/JuliaLang/Pkg.jl/pull/4274 +[#4311]: https://github.com/JuliaLang/Pkg.jl/pull/4311 +[#4300]: https://github.com/JuliaLang/Pkg.jl/pull/4300 +[#4307]: https://github.com/JuliaLang/Pkg.jl/pull/4307 +[#4308]: https://github.com/JuliaLang/Pkg.jl/pull/4308 +[#4312]: https://github.com/JuliaLang/Pkg.jl/pull/4312 +[#4305]: https://github.com/JuliaLang/Pkg.jl/pull/4305 +[#4170]: https://github.com/JuliaLang/Pkg.jl/pull/4170 +[#4287]: https://github.com/JuliaLang/Pkg.jl/pull/4287 From a84228360d6cff568a55911733e830cdf1c492da Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sun, 13 Jul 2025 08:41:40 -0400 Subject: [PATCH 137/154] Format code base with runic and set up runic CI (#4320) --- .github/workflows/check.yml | 30 + .pre-commit-config.yaml | 17 + contrib/list_missing_pkg_tags.jl | 6 +- docs/NEWS-update.jl | 4 +- docs/generate.jl | 50 +- docs/make.jl | 2 +- docs/src/creating-packages.md | 2 +- docs/src/protocol.md | 2 +- ext/REPLExt/REPLExt.jl | 53 +- ext/REPLExt/compat.jl | 12 +- ext/REPLExt/completions.jl | 58 +- ext/REPLExt/precompile.jl | 2 +- src/API.jl | 443 +-- src/Apps/Apps.jl | 134 +- src/Artifacts.jl | 1141 ++++---- src/BinaryPlatformsCompat.jl | 261 +- src/GitTools.jl | 60 +- src/HistoricalStdlibs.jl | 4 +- src/MiniProgressBars.jl | 34 +- src/Operations.jl | 929 +++--- src/Pkg.jl | 36 +- src/PlatformEngines.jl | 178 +- src/REPLMode/REPLMode.jl | 164 +- src/REPLMode/argument_parsers.jl | 70 +- src/REPLMode/command_declarations.jl | 1415 ++++----- src/Registry/Registry.jl | 393 +-- src/Registry/registry_instance.jl | 58 +- src/Resolve/Resolve.jl | 105 +- src/Resolve/fieldvalues.jl | 20 +- src/Resolve/graphtype.jl | 300 +- src/Resolve/maxsum.jl | 61 +- src/Resolve/versionweights.jl | 10 +- src/Types.jl | 280 +- src/Versions.jl | 106 +- src/fuzzysorting.jl | 50 +- src/generate.jl | 28 +- src/manifest.jl | 130 +- src/precompile.jl | 11 +- src/project.jl | 139 +- src/utils.jl | 18 +- test/FakeTerminals.jl | 4 +- test/NastyGenerator.jl | 35 +- test/api.jl | 678 ++--- test/apps.jl | 72 +- test/artifacts.jl | 252 +- test/binaryplatforms.jl | 84 +- test/extensions.jl | 62 +- test/force_latest_compatible_version.jl | 4 +- test/historical_stdlib_version.jl | 130 +- test/manifest/formats/v2.0/Manifest.toml | 1 - test/manifests.jl | 60 +- test/misc.jl | 2 +- test/new.jl | 2530 +++++++++-------- test/pkg.jl | 643 +++-- test/platformengines.jl | 60 +- test/project/bad/targets_not_a_table.toml | 1 - test/project_manifest.jl | 6 +- test/registry.jl | 556 ++-- test/repl.jl | 813 +++--- test/resolve.jl | 323 ++- test/resolve_utils.jl | 54 +- test/runtests.jl | 148 +- test/sandbox.jl | 238 +- test/sources.jl | 10 +- test/subdir.jl | 498 ++-- .../src/ArtifactInstallation.jl | 4 +- .../ArtifactOverrideLoading/Artifacts.toml | 1 - .../src/ArtifactOverrideLoading.jl | 10 +- .../julia_artifacts_test/pkg.jl | 2 +- test/test_packages/ArtifactTOMLSearch/pkg.jl | 2 +- .../ArtifactTOMLSearch/sub_module/pkg.jl | 2 +- .../ArtifactTOMLSearch/sub_package/pkg.jl | 2 +- .../BigProject/RecursiveDep/Project.toml | 2 +- .../RecursiveDep/src/RecursiveDep.jl | 2 +- .../BigProject/RecursiveDep2/Project.toml | 2 +- .../RecursiveDep2/src/RecursiveDep2.jl | 2 +- .../BuildProjectFixedDeps/.gitignore | 2 +- .../src/HasDepWithExtensions.jl | 2 +- .../HasExtensions.jl/ext/IndirectArraysExt.jl | 2 +- .../HasExtensions.jl/ext/OffsetArraysExt.jl | 2 +- test/test_packages/Rot13.jl/src/CLI.jl | 4 +- test/test_packages/Rot13.jl/src/Rot13.jl | 2 +- .../Rot13.jl/src/Rot13_edited.jl | 2 +- .../Sandbox_PreservePreferences/Project.toml | 2 +- .../dev/Foo/Project.toml | 2 +- .../dev/Foo/src/Foo.jl | 2 +- .../dev/Foo/test/Project.toml | 2 +- .../test/runtests.jl | 1 + .../TestDepTrackingPath/test/runtests.jl | 1 + test/test_packages/TestFailure/Project.toml | 2 +- .../TestThreads/test/runtests.jl | 1 - .../WorkspacePathResolution/Project.toml | 2 +- .../SubProjectA/src/SubProjectA.jl | 2 +- .../SubProjectB/Project.toml | 2 +- .../SubProjectB/src/SubProjectB.jl | 2 +- .../monorepo/packages/D/Project.toml | 2 +- test/test_packages/monorepo/test/runtests.jl | 2 +- test/utils.jl | 90 +- test/workspaces.jl | 254 +- 99 files changed, 7656 insertions(+), 6840 deletions(-) create mode 100644 .github/workflows/check.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 0000000000..8376d060fb --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,30 @@ +name: Code checks + +on: + pull_request: + push: + branches: ["master"] + +jobs: + + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 + env: + # Skip runic-pre-commit since we use runic-action below instead + SKIP: runic + + runic: + name: "Runic" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: '1.11' + - uses: julia-actions/cache@v2 + - uses: fredrikekre/runic-action@v1 + with: + version: "1.4" # Keep version in sync with .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..68066c2cc2 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,17 @@ +repos: + - repo: 'https://github.com/pre-commit/pre-commit-hooks' + rev: v5.0.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + # - id: check-toml # we have tomls with invalid syntax for tests + - id: check-yaml + - id: end-of-file-fixer + - id: mixed-line-ending + - id: trailing-whitespace + - repo: 'https://github.com/fredrikekre/runic-pre-commit' + rev: v2.0.1 + hooks: + - id: runic + additional_dependencies: + - 'Runic@1.4' # Keep version in sync with .github/workflows/Check.yml diff --git a/contrib/list_missing_pkg_tags.jl b/contrib/list_missing_pkg_tags.jl index ce95b9e517..93309889aa 100644 --- a/contrib/list_missing_pkg_tags.jl +++ b/contrib/list_missing_pkg_tags.jl @@ -7,7 +7,7 @@ const PKG_REPO_URL = "https://github.com/JuliaLang/Pkg.jl.git" const PKG_REPO_DIR = "Pkg.jl" function checkout_or_update_repo(url, dir) - if isdir(dir) + return if isdir(dir) println("Updating existing repository: $dir") repo = LibGit2.GitRepo(dir) LibGit2.fetch(repo) @@ -33,7 +33,7 @@ function extract_pkg_sha1(text::AbstractString) end function get_commit_hash_for_pkg_version(repo, tag) - try + return try tag_ref = LibGit2.GitReference(repo, "refs/tags/" * tag) LibGit2.checkout!(repo, string(LibGit2.GitHash(LibGit2.peel(tag_ref)))) version_file = joinpath(JULIA_REPO_DIR, PKG_VERSION_PATH) @@ -78,7 +78,7 @@ cd(tempdir) do missing_versions = filter(v -> v ∉ pkg_tags, collect(keys(version_commit_map))) # Sort versions numerically - sort!(missing_versions, by=VersionNumber) + sort!(missing_versions, by = VersionNumber) # Generate `git tag` commands println("\nGit tag commands for missing Pkg.jl versions:") diff --git a/docs/NEWS-update.jl b/docs/NEWS-update.jl index 3812e9e437..d0ca10d391 100644 --- a/docs/NEWS-update.jl +++ b/docs/NEWS-update.jl @@ -7,11 +7,11 @@ s = read(NEWS, String) m = match(r"\[#[0-9]+\]:", s) if m !== nothing - s = s[1:m.offset-1] + s = s[1:(m.offset - 1)] end footnote(n) = "[#$n]: https://github.com/JuliaLang/Pkg.jl/issues/$n" -N = map(m -> parse(Int,m.captures[1]), eachmatch(r"\[#([0-9]+)\]", s)) +N = map(m -> parse(Int, m.captures[1]), eachmatch(r"\[#([0-9]+)\]", s)) foots = join(map(footnote, sort!(unique(N))), "\n") open(NEWS, "w") do f diff --git a/docs/generate.jl b/docs/generate.jl index fa4af617ef..3d227f1374 100644 --- a/docs/generate.jl +++ b/docs/generate.jl @@ -4,38 +4,42 @@ function generate(io, command) cmd_nospace = replace(command, " " => "-") - println(io, """ - ```@raw html -
-
- - $(command) - - — - REPL command -
-
- ``` - ```@eval - using Pkg - Dict(Pkg.REPLMode.canonical_names())["$(command)"].help - ``` - ```@raw html -
-
- ``` - """) + return println( + io, """ + ```@raw html +
+
+ + $(command) + + — + REPL command +
+
+ ``` + ```@eval + using Pkg + Dict(Pkg.REPLMode.canonical_names())["$(command)"].help + ``` + ```@raw html +
+
+ ``` + """ + ) end function generate() io = IOBuffer() - println(io, """ + println( + io, """ # [**11.** REPL Mode Reference](@id REPL-Mode-Reference) This section describes available commands in the Pkg REPL. The Pkg REPL mode is mostly meant for interactive use, and for non-interactive use it is recommended to use the functional API, see [API Reference](@ref API-Reference). - """) + """ + ) # list commands println(io, "## `package` commands") foreach(command -> generate(io, command), ["add", "build", "compat", "develop", "free", "generate", "pin", "remove", "test", "update"]) diff --git a/docs/make.jl b/docs/make.jl index 4b97e3ec23..6b38dad0d7 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -17,7 +17,7 @@ if "pdf" in ARGS end # setup for doctesting -DocMeta.setdocmeta!(Pkg.BinaryPlatforms, :DocTestSetup, :(using Base.BinaryPlatforms); recursive=true) +DocMeta.setdocmeta!(Pkg.BinaryPlatforms, :DocTestSetup, :(using Base.BinaryPlatforms); recursive = true) # Run doctests first and disable them in makedocs Documenter.doctest(joinpath(@__DIR__, "src"), [Pkg]) diff --git a/docs/src/creating-packages.md b/docs/src/creating-packages.md index 2850008925..4e01f0c27a 100644 --- a/docs/src/creating-packages.md +++ b/docs/src/creating-packages.md @@ -627,7 +627,7 @@ may fit your package better. * The package name should begin with a capital letter and word separation is shown with upper camel case * Packages that provide the functionality of a project from another language should use the Julia convention * Packages that [provide pre-built libraries and executables](https://docs.binarybuilder.org/stable/jll/) can keep their original name, but should get `_jll`as a suffix. For example `pandoc_jll` wraps pandoc. However, note that the generation and release of most JLL packages is handled by the [Yggdrasil](https://github.com/JuliaPackaging/Yggdrasil) system. - + 10. For the complete list of rules for automatic merging into the General registry, see [these guidelines](https://juliaregistries.github.io/RegistryCI.jl/stable/guidelines/). diff --git a/docs/src/protocol.md b/docs/src/protocol.md index 15a6cbee1b..211b1caf46 100644 --- a/docs/src/protocol.md +++ b/docs/src/protocol.md @@ -13,7 +13,7 @@ The protocols also aim to address some of the limitations that existed prior to ## Protocols & Services -1. **Pkg Protocol:** what Julia Pkg Clients speak to Pkg Servers. The Pkg Server serves all resources that Pkg Clients need to install and use registered packages, including registry data, packages and artifacts. It is designed to be easily horizontally scalable and not to have any hard operational requirements: if service is slow, just start more servers; if a Pkg Server crashes, forget it and boot up a new one. +1. **Pkg Protocol:** what Julia Pkg Clients speak to Pkg Servers. The Pkg Server serves all resources that Pkg Clients need to install and use registered packages, including registry data, packages and artifacts. It is designed to be easily horizontally scalable and not to have any hard operational requirements: if service is slow, just start more servers; if a Pkg Server crashes, forget it and boot up a new one. 2. **Storage Protocol:** what Pkg Servers speak to get resources from Storage Services. Julia clients do not interact with Storage services directly and multiple independent Storage Services can symmetrically (all are treated equally) provide their service to a given Pkg Server. Since Pkg Servers cache what they serve to Clients and handle convenient content presentation, Storage Services can expose a much simpler protocol: all they do is serve up complete versions of registries, packages and artifacts, while guaranteeing persistence and completeness. Persistence means: once a version of a resource has been served, that version can be served forever. Completeness means: if the service serves a registry, it can serve all package versions referenced by that registry; if it serves a package version, it can serve all artifacts used by that package. Both protocols work over HTTPS, using only GET and HEAD requests. As is normal for HTTP, HEAD requests are used to get information about a resource, including whether it would be served, without actually downloading it. As described in what follows, the Pkg Protocol is client-to-server and may be unauthenticated, use basic auth, or OpenID; the Storage Protocol is server-to-server only and uses mutual authentication with TLS certificates. diff --git a/ext/REPLExt/REPLExt.jl b/ext/REPLExt/REPLExt.jl index 8e08bad9e4..31bbf66970 100644 --- a/ext/REPLExt/REPLExt.jl +++ b/ext/REPLExt/REPLExt.jl @@ -30,7 +30,7 @@ include("compat.jl") struct PkgCompletionProvider <: LineEdit.CompletionProvider end -function LineEdit.complete_line(c::PkgCompletionProvider, s; hint::Bool=false) +function LineEdit.complete_line(c::PkgCompletionProvider, s; hint::Bool = false) partial = REPL.beforecursor(s.input_buffer) full = LineEdit.input_string(s) ret, range, should_complete = completions(full, lastindex(partial); hint) @@ -120,16 +120,18 @@ function on_done(s, buf, ok, repl) do_cmds(repl, input) REPL.prepare_next(repl) REPL.reset_state(s) - s.current_mode.sticky || REPL.transition(s, main) + return s.current_mode.sticky || REPL.transition(s, main) end # Set up the repl Pkg REPLMode function create_mode(repl::REPL.AbstractREPL, main::LineEdit.Prompt) - pkg_mode = LineEdit.Prompt(promptf; + pkg_mode = LineEdit.Prompt( + promptf; prompt_prefix = repl.options.hascolor ? Base.text_colors[:blue] : "", prompt_suffix = "", complete = PkgCompletionProvider(), - sticky = true) + sticky = true + ) pkg_mode.repl = repl hp = main.hist @@ -152,9 +154,9 @@ function create_mode(repl::REPL.AbstractREPL, main::LineEdit.Prompt) repl_keymap = Dict() if shell_mode !== nothing - let shell_mode=shell_mode - repl_keymap[';'] = function (s,o...) - if isempty(s) || position(LineEdit.buffer(s)) == 0 + let shell_mode = shell_mode + repl_keymap[';'] = function (s, o...) + return if isempty(s) || position(LineEdit.buffer(s)) == 0 buf = copy(LineEdit.buffer(s)) LineEdit.transition(s, shell_mode) do LineEdit.state(s, shell_mode).input_buffer = buf @@ -167,9 +169,9 @@ function create_mode(repl::REPL.AbstractREPL, main::LineEdit.Prompt) end end - b = Dict{Any,Any}[ + b = Dict{Any, Any}[ skeymap, repl_keymap, mk, prefix_keymap, LineEdit.history_keymap, - LineEdit.default_keymap, LineEdit.escape_defaults + LineEdit.default_keymap, LineEdit.escape_defaults, ] pkg_mode.keymap_dict = LineEdit.keymap(b) return pkg_mode @@ -179,9 +181,9 @@ function repl_init(repl::REPL.LineEditREPL) main_mode = repl.interface.modes[1] pkg_mode = create_mode(repl, main_mode) push!(repl.interface.modes, pkg_mode) - keymap = Dict{Any,Any}( - ']' => function (s,args...) - if isempty(s) || position(LineEdit.buffer(s)) == 0 + keymap = Dict{Any, Any}( + ']' => function (s, args...) + return if isempty(s) || position(LineEdit.buffer(s)) == 0 buf = copy(LineEdit.buffer(s)) LineEdit.transition(s, pkg_mode) do LineEdit.state(s, pkg_mode).input_buffer = buf @@ -208,9 +210,9 @@ function try_prompt_pkg_add(pkgs::Vector{Symbol}) end if isempty(ctx.registries) if !REG_WARNED[] - printstyled(ctx.io, " │ "; color=:green) + printstyled(ctx.io, " │ "; color = :green) printstyled(ctx.io, "Attempted to find missing packages in package registries but no registries are installed.\n") - printstyled(ctx.io, " └ "; color=:green) + printstyled(ctx.io, " └ "; color = :green) printstyled(ctx.io, "Use package mode to install a registry. `pkg> registry add` will install the default registries.\n\n") REG_WARNED[] = true end @@ -230,22 +232,22 @@ function try_prompt_pkg_add(pkgs::Vector{Symbol}) available_pkg_list = length(available_pkgs) == 1 ? String(available_pkgs[1]) : "[$(join(available_pkgs, ", "))]" msg1 = "Package$(plural1) $(missing_pkg_list) not found, but $(plural2) named $(available_pkg_list) $(plural3) available from a registry." for line in linewrap(msg1, io = ctx.io, padding = length(" │ ")) - printstyled(ctx.io, " │ "; color=:green) + printstyled(ctx.io, " │ "; color = :green) println(ctx.io, line) end - printstyled(ctx.io, " │ "; color=:green) + printstyled(ctx.io, " │ "; color = :green) println(ctx.io, "Install package$(plural4)?") msg2 = string("add ", join(available_pkgs, ' ')) for (i, line) in pairs(linewrap(msg2; io = ctx.io, padding = length(string(" | ", promptf())))) - printstyled(ctx.io, " │ "; color=:green) + printstyled(ctx.io, " │ "; color = :green) if i == 1 - printstyled(ctx.io, promptf(); color=:blue) + printstyled(ctx.io, promptf(); color = :blue) else print(ctx.io, " "^length(promptf())) end println(ctx.io, line) end - printstyled(ctx.io, " └ "; color=:green) + printstyled(ctx.io, " └ "; color = :green) Base.prompt(stdin, ctx.io, "(y/n/o)", default = "y") catch err if err isa InterruptException # if ^C is entered @@ -261,7 +263,7 @@ function try_prompt_pkg_add(pkgs::Vector{Symbol}) resp = strip(resp) lower_resp = lowercase(resp) if lower_resp in ["y", "yes"] - API.add(string.(available_pkgs); allow_autoprecomp=false) + API.add(string.(available_pkgs); allow_autoprecomp = false) elseif lower_resp in ["o"] editable_envs = filter(v -> v != "@stdlib", LOAD_PATH) option_list = String[] @@ -280,16 +282,16 @@ function try_prompt_pkg_add(pkgs::Vector{Symbol}) push!(keybindings, only("$n")) push!(shown_envs, expanded_env) end - menu = TerminalMenus.RadioMenu(option_list, keybindings=keybindings, pagesize=length(option_list)) + menu = TerminalMenus.RadioMenu(option_list, keybindings = keybindings, pagesize = length(option_list)) default = something( # select the first non-default env by default, if possible findfirst(!=(Base.active_project()), shown_envs), 1 ) print(ctx.io, "\e[1A\e[1G\e[0J") # go up one line, to the start, and clear it - printstyled(ctx.io, " └ "; color=:green) + printstyled(ctx.io, " └ "; color = :green) choice = try - TerminalMenus.request("Select environment:", menu, cursor=default) + TerminalMenus.request("Select environment:", menu, cursor = default) catch err if err isa InterruptException # if ^C is entered println(ctx.io) @@ -299,7 +301,7 @@ function try_prompt_pkg_add(pkgs::Vector{Symbol}) end choice == -1 && return false API.activate(shown_envs[choice]) do - API.add(string.(available_pkgs); allow_autoprecomp=false) + API.add(string.(available_pkgs); allow_autoprecomp = false) end elseif (lower_resp in ["n"]) return false @@ -315,7 +317,6 @@ function try_prompt_pkg_add(pkgs::Vector{Symbol}) end - function __init__() if isdefined(Base, :active_repl) if Base.active_repl isa REPL.LineEditREPL @@ -333,7 +334,7 @@ function __init__() end end end - if !in(try_prompt_pkg_add, REPL.install_packages_hooks) + return if !in(try_prompt_pkg_add, REPL.install_packages_hooks) push!(REPL.install_packages_hooks, try_prompt_pkg_add) end end diff --git a/ext/REPLExt/compat.jl b/ext/REPLExt/compat.jl index f2de1aaea0..81d4af582e 100644 --- a/ext/REPLExt/compat.jl +++ b/ext/REPLExt/compat.jl @@ -9,12 +9,12 @@ function _compat(ctx::Context; io = nothing) compat_str = Operations.get_compat_str(ctx.env.project, "julia") push!(opt_strs, Operations.compat_line(io, "julia", nothing, compat_str, longest_dep_len, indent = "")) push!(opt_pkgs, "julia") - for (dep, uuid) in sort(collect(ctx.env.project.deps); by = x->x.first) + for (dep, uuid) in sort(collect(ctx.env.project.deps); by = x -> x.first) compat_str = Operations.get_compat_str(ctx.env.project, dep) push!(opt_strs, Operations.compat_line(io, dep, uuid, compat_str, longest_dep_len, indent = "")) push!(opt_pkgs, dep) end - menu = TerminalMenus.RadioMenu(opt_strs, pagesize=length(opt_strs)) + menu = TerminalMenus.RadioMenu(opt_strs, pagesize = length(opt_strs)) choice = try TerminalMenus.request(" Select an entry to edit:", menu) catch err @@ -35,7 +35,7 @@ function _compat(ctx::Context; io = nothing) start_pos = length(prompt) + 2 move_start = "\e[$(start_pos)G" clear_to_end = "\e[0J" - ccall(:jl_tty_set_mode, Int32, (Ptr{Cvoid},Int32), stdin.handle, true) + ccall(:jl_tty_set_mode, Int32, (Ptr{Cvoid}, Int32), stdin.handle, true) while true print(io, move_start, clear_to_end, buffer, "\e[$(start_pos + cursor)G") inp = TerminalMenus._readkey(stdin) @@ -65,9 +65,9 @@ function _compat(ctx::Context; io = nothing) if cursor == 1 buffer = buffer[2:end] elseif cursor == length(buffer) - buffer = buffer[1:end - 1] + buffer = buffer[1:(end - 1)] elseif cursor > 0 - buffer = buffer[1:(cursor-1)] * buffer[(cursor + 1):end] + buffer = buffer[1:(cursor - 1)] * buffer[(cursor + 1):end] else continue end @@ -85,7 +85,7 @@ function _compat(ctx::Context; io = nothing) end buffer finally - ccall(:jl_tty_set_mode, Int32, (Ptr{Cvoid},Int32), stdin.handle, false) + ccall(:jl_tty_set_mode, Int32, (Ptr{Cvoid}, Int32), stdin.handle, false) end new_entry = strip(resp) compat(ctx, dep, string(new_entry)) diff --git a/ext/REPLExt/completions.jl b/ext/REPLExt/completions.jl index 3a18512834..e9e0b3fcfb 100644 --- a/ext/REPLExt/completions.jl +++ b/ext/REPLExt/completions.jl @@ -34,11 +34,11 @@ function complete_local_dir(s, i1, i2) end function complete_expanded_local_dir(s, i1, i2, expanded_user, oldi2) - cmp = REPL.REPLCompletions.complete_path(s, i2, shell_escape=true) + cmp = REPL.REPLCompletions.complete_path(s, i2, shell_escape = true) cmp2 = cmp[2] completions = [REPL.REPLCompletions.completion_text(p) for p in cmp[1]] completions = filter!(completions) do x - Base.isaccessibledir(s[1:prevind(s, first(cmp2)-i1+1)]*x) + Base.isaccessibledir(s[1:prevind(s, first(cmp2) - i1 + 1)] * x) end if expanded_user if length(completions) == 1 && endswith(joinpath(homedir(), ""), first(completions)) @@ -98,13 +98,14 @@ end function complete_help(options, partial; hint::Bool) names = String[] for cmds in values(SPECS) - append!(names, [spec.canonical_name for spec in values(cmds)]) + append!(names, [spec.canonical_name for spec in values(cmds)]) end return sort!(unique!(append!(names, collect(keys(SPECS))))) end function complete_installed_packages(options, partial; hint::Bool) - env = try EnvCache() + env = try + EnvCache() catch err err isa PkgError || rethrow() return String[] @@ -116,7 +117,8 @@ function complete_installed_packages(options, partial; hint::Bool) end function complete_all_installed_packages(options, partial; hint::Bool) - env = try EnvCache() + env = try + EnvCache() catch err err isa PkgError || rethrow() return String[] @@ -125,7 +127,8 @@ function complete_all_installed_packages(options, partial; hint::Bool) end function complete_installed_packages_and_compat(options, partial; hint::Bool) - env = try EnvCache() + env = try + EnvCache() catch err err isa PkgError || rethrow() return String[] @@ -137,7 +140,8 @@ function complete_installed_packages_and_compat(options, partial; hint::Bool) end function complete_fixed_packages(options, partial; hint::Bool) - env = try EnvCache() + env = try + EnvCache() catch err err isa PkgError || rethrow() return String[] @@ -198,13 +202,23 @@ function complete_command(statement::Statement, final::Bool, on_sub::Bool) end complete_opt(opt_specs) = - unique(sort(map(wrap_option, - map(x -> getproperty(x, :name), - collect(values(opt_specs)))))) + unique( + sort( + map( + wrap_option, + map( + x -> getproperty(x, :name), + collect(values(opt_specs)) + ) + ) + ) +) -function complete_argument(spec::CommandSpec, options::Vector{String}, - partial::AbstractString, offset::Int, - index::Int; hint::Bool) +function complete_argument( + spec::CommandSpec, options::Vector{String}, + partial::AbstractString, offset::Int, + index::Int; hint::Bool + ) if spec.completions isa Symbol # if completions is a symbol, it is a function in REPLExt that needs to be forwarded # to REPLMode (couldn't be linked there because REPLExt is not a dependency of REPLMode) @@ -214,11 +228,11 @@ function complete_argument(spec::CommandSpec, options::Vector{String}, @error "REPLMode indicates a completion function called :$(spec.completions) that cannot be found in REPLExt" rethrow() end - spec.completions = function(opts, partial, offset, index; hint::Bool) - applicable(completions, opts, partial, offset, index) ? - completions(opts, partial, offset, index; hint) : - completions(opts, partial; hint) - end + spec.completions = function (opts, partial, offset, index; hint::Bool) + return applicable(completions, opts, partial, offset, index) ? + completions(opts, partial, offset, index; hint) : + completions(opts, partial; hint) + end end spec.completions === nothing && return String[] # finish parsing opts @@ -274,7 +288,7 @@ function _completions(input, final, offset, index; hint::Bool) end end -function completions(full, index; hint::Bool=false)::Tuple{Vector{String},UnitRange{Int},Bool} +function completions(full, index; hint::Bool = false)::Tuple{Vector{String}, UnitRange{Int}, Bool} pre = full[1:index] isempty(pre) && return default_commands(), 0:-1, false # empty input -> complete commands offset_adjust = 0 @@ -283,8 +297,8 @@ function completions(full, index; hint::Bool=false)::Tuple{Vector{String},UnitRa pre = string(pre[1], " ", pre[2:end]) offset_adjust = -1 end - last = split(pre, ' ', keepempty=true)[end] - offset = isempty(last) ? index+1+offset_adjust : last.offset+1+offset_adjust - final = isempty(last) # is the cursor still attached to the final token? + last = split(pre, ' ', keepempty = true)[end] + offset = isempty(last) ? index + 1 + offset_adjust : last.offset + 1 + offset_adjust + final = isempty(last) # is the cursor still attached to the final token? return _completions(pre, final, offset, index; hint) end diff --git a/ext/REPLExt/precompile.jl b/ext/REPLExt/precompile.jl index 2deb9b84f0..fbfdf14baf 100644 --- a/ext/REPLExt/precompile.jl +++ b/ext/REPLExt/precompile.jl @@ -32,7 +32,7 @@ let Base.precompile(Tuple{typeof(REPL.LineEdit.complete_line), REPLExt.PkgCompletionProvider, REPL.LineEdit.PromptState}) Base.precompile(Tuple{typeof(REPL.REPLCompletions.completion_text), REPL.REPLCompletions.PackageCompletion}) Base.precompile(Tuple{typeof(REPLExt.on_done), REPL.LineEdit.MIState, Base.GenericIOBuffer{Memory{UInt8}}, Bool, REPL.LineEditREPL}) - Base.precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:hint,), Tuple{Bool}}, typeof(REPL.LineEdit.complete_line), REPLExt.PkgCompletionProvider, REPL.LineEdit.PromptState}) + return Base.precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:hint,), Tuple{Bool}}, typeof(REPL.LineEdit.complete_line), REPLExt.PkgCompletionProvider, REPL.LineEdit.PromptState}) end if Base.generating_output() diff --git a/src/API.jl b/src/API.jl index 9fefecb380..1589e413b7 100644 --- a/src/API.jl +++ b/src/API.jl @@ -27,17 +27,17 @@ include("generate.jl") Base.@kwdef struct PackageInfo name::String - version::Union{Nothing,VersionNumber} - tree_hash::Union{Nothing,String} + version::Union{Nothing, VersionNumber} + tree_hash::Union{Nothing, String} is_direct_dep::Bool is_pinned::Bool is_tracking_path::Bool is_tracking_repo::Bool is_tracking_registry::Bool - git_revision::Union{Nothing,String} - git_source::Union{Nothing,String} + git_revision::Union{Nothing, String} + git_source::Union{Nothing, String} source::String - dependencies::Dict{String,UUID} + dependencies::Dict{String, UUID} end function Base.:(==)(a::PackageInfo, b::PackageInfo) @@ -53,34 +53,36 @@ end function package_info(env::EnvCache, pkg::PackageSpec)::PackageInfo entry = manifest_info(env.manifest, pkg.uuid) if entry === nothing - pkgerror("expected package $(err_rep(pkg)) to exist in the manifest", - " (use `resolve` to populate the manifest)") + pkgerror( + "expected package $(err_rep(pkg)) to exist in the manifest", + " (use `resolve` to populate the manifest)" + ) end - package_info(env, pkg, entry) + return package_info(env, pkg, entry) end function package_info(env::EnvCache, pkg::PackageSpec, entry::PackageEntry)::PackageInfo git_source = pkg.repo.source === nothing ? nothing : isurl(pkg.repo.source::String) ? pkg.repo.source::String : Operations.project_rel_path(env, pkg.repo.source::String) - _source_path = Operations.source_path(env.manifest_file, pkg) - if _source_path === nothing - @debug "Manifest file $(env.manifest_file) contents:\n$(read(env.manifest_file, String))" - pkgerror("could not find source path for package $(err_rep(pkg)) based on $(env.manifest_file)") - end + _source_path = Operations.source_path(env.manifest_file, pkg) + if _source_path === nothing + @debug "Manifest file $(env.manifest_file) contents:\n$(read(env.manifest_file, String))" + pkgerror("could not find source path for package $(err_rep(pkg)) based on $(env.manifest_file)") + end info = PackageInfo( - name = pkg.name, - version = pkg.version != VersionSpec() ? pkg.version : nothing, - tree_hash = pkg.tree_hash === nothing ? nothing : string(pkg.tree_hash), # TODO or should it just be a SHA? - is_direct_dep = pkg.uuid in values(env.project.deps), - is_pinned = pkg.pinned, - is_tracking_path = pkg.path !== nothing, - is_tracking_repo = pkg.repo.rev !== nothing || pkg.repo.source !== nothing, + name = pkg.name, + version = pkg.version != VersionSpec() ? pkg.version : nothing, + tree_hash = pkg.tree_hash === nothing ? nothing : string(pkg.tree_hash), # TODO or should it just be a SHA? + is_direct_dep = pkg.uuid in values(env.project.deps), + is_pinned = pkg.pinned, + is_tracking_path = pkg.path !== nothing, + is_tracking_repo = pkg.repo.rev !== nothing || pkg.repo.source !== nothing, is_tracking_registry = Operations.is_tracking_registry(pkg), - git_revision = pkg.repo.rev, - git_source = git_source, - source = Operations.project_rel_path(env, _source_path), - dependencies = copy(entry.deps), #TODO is copy needed? + git_revision = pkg.repo.rev, + git_source = git_source, + source = Operations.project_rel_path(env, _source_path), + dependencies = copy(entry.deps), #TODO is copy needed? ) return info end @@ -95,17 +97,17 @@ function dependencies(fn::Function, uuid::UUID) if dep === nothing pkgerror("dependency with UUID `$uuid` does not exist") end - fn(dep) + return fn(dep) end Base.@kwdef struct ProjectInfo - name::Union{Nothing,String} - uuid::Union{Nothing,UUID} - version::Union{Nothing,VersionNumber} + name::Union{Nothing, String} + uuid::Union{Nothing, UUID} + version::Union{Nothing, VersionNumber} ispackage::Bool - dependencies::Dict{String,UUID} - sources::Dict{String,Dict{String,String}} + dependencies::Dict{String, UUID} + sources::Dict{String, Dict{String, String}} path::String end @@ -113,26 +115,28 @@ project() = project(EnvCache()) function project(env::EnvCache)::ProjectInfo pkg = env.pkg return ProjectInfo( - name = pkg === nothing ? nothing : pkg.name, - uuid = pkg === nothing ? nothing : pkg.uuid, - version = pkg === nothing ? nothing : pkg.version::VersionNumber, - ispackage = pkg !== nothing, + name = pkg === nothing ? nothing : pkg.name, + uuid = pkg === nothing ? nothing : pkg.uuid, + version = pkg === nothing ? nothing : pkg.version::VersionNumber, + ispackage = pkg !== nothing, dependencies = env.project.deps, - sources = env.project.sources, - path = env.project_file + sources = env.project.sources, + path = env.project_file ) end -function check_package_name(x::AbstractString, mode::Union{Nothing,String,Symbol}=nothing) +function check_package_name(x::AbstractString, mode::Union{Nothing, String, Symbol} = nothing) if !Base.isidentifier(x) message = sprint() do iostr print(iostr, "`$x` is not a valid package name") if endswith(lowercase(x), ".jl") - print(iostr, ". Perhaps you meant `$(chop(x; tail=3))`") + print(iostr, ". Perhaps you meant `$(chop(x; tail = 3))`") end - if mode !== nothing && any(occursin.(['\\','/'], x)) # maybe a url or a path - print(iostr, "\nThe argument appears to be a URL or path, perhaps you meant ", - "`Pkg.$mode(url=\"...\")` or `Pkg.$mode(path=\"...\")`.") + if mode !== nothing && any(occursin.(['\\', '/'], x)) # maybe a url or a path + print( + iostr, "\nThe argument appears to be a URL or path, perhaps you meant ", + "`Pkg.$mode(url=\"...\")` or `Pkg.$mode(path=\"...\")`." + ) end end pkgerror(message) @@ -142,15 +146,15 @@ end check_package_name(::Nothing, ::Any) = nothing function require_not_empty(pkgs, f::Symbol) - isempty(pkgs) && pkgerror("$f requires at least one package") + return isempty(pkgs) && pkgerror("$f requires at least one package") end # Provide some convenience calls for f in (:develop, :add, :rm, :up, :pin, :free, :test, :build, :status, :why, :precompile) @eval begin $f(pkg::Union{AbstractString, PackageSpec}; kwargs...) = $f([pkg]; kwargs...) - $f(pkgs::Vector{<:AbstractString}; kwargs...) = $f([PackageSpec(pkg) for pkg in pkgs]; kwargs...) - function $f(pkgs::Vector{PackageSpec}; io::IO=$(f === :status ? :stdout_f : :stderr_f)(), kwargs...) + $f(pkgs::Vector{<:AbstractString}; kwargs...) = $f([PackageSpec(pkg) for pkg in pkgs]; kwargs...) + function $f(pkgs::Vector{PackageSpec}; io::IO = $(f === :status ? :stdout_f : :stderr_f)(), kwargs...) $(f != :precompile) && Registry.download_default_registries(io) ctx = Context() # Save initial environment for undo/redo functionality @@ -158,7 +162,7 @@ for f in (:develop, :add, :rm, :up, :pin, :free, :test, :build, :status, :why, : add_snapshot_to_undo(ctx.env) saved_initial_snapshot[] = true end - kwargs = merge((;kwargs...), (:io => io,)) + kwargs = merge((; kwargs...), (:io => io,)) pkgs = deepcopy(pkgs) # don't mutate input foreach(handle_package_input!, pkgs) ret = $f(ctx, pkgs; kwargs...) @@ -167,22 +171,24 @@ for f in (:develop, :add, :rm, :up, :pin, :free, :test, :build, :status, :why, : return ret end $f(ctx::Context; kwargs...) = $f(ctx, PackageSpec[]; kwargs...) - function $f(; name::Union{Nothing,AbstractString}=nothing, uuid::Union{Nothing,String,UUID}=nothing, - version::Union{VersionNumber, String, VersionSpec, Nothing}=nothing, - url=nothing, rev=nothing, path=nothing, mode=PKGMODE_PROJECT, subdir=nothing, kwargs...) + function $f(; + name::Union{Nothing, AbstractString} = nothing, uuid::Union{Nothing, String, UUID} = nothing, + version::Union{VersionNumber, String, VersionSpec, Nothing} = nothing, + url = nothing, rev = nothing, path = nothing, mode = PKGMODE_PROJECT, subdir = nothing, kwargs... + ) pkg = PackageSpec(; name, uuid, version, url, rev, path, subdir) if $f === status || $f === rm || $f === up - kwargs = merge((;kwargs...), (:mode => mode,)) + kwargs = merge((; kwargs...), (:mode => mode,)) end # Handle $f() case - if all(isnothing, [name,uuid,version,url,rev,path,subdir]) + return if all(isnothing, [name, uuid, version, url, rev, path, subdir]) $f(PackageSpec[]; kwargs...) else $f(pkg; kwargs...) end end function $f(pkgs::Vector{<:NamedTuple}; kwargs...) - $f([PackageSpec(;pkg...) for pkg in pkgs]; kwargs...) + return $f([PackageSpec(; pkg...) for pkg in pkgs]; kwargs...) end end end @@ -240,8 +246,10 @@ function update_source_if_set(env, pkg) return end -function develop(ctx::Context, pkgs::Vector{PackageSpec}; shared::Bool=true, - preserve::PreserveLevel=Operations.default_preserve(), platform::AbstractPlatform=HostPlatform(), kwargs...) +function develop( + ctx::Context, pkgs::Vector{PackageSpec}; shared::Bool = true, + preserve::PreserveLevel = Operations.default_preserve(), platform::AbstractPlatform = HostPlatform(), kwargs... + ) require_not_empty(pkgs, :develop) Context!(ctx; kwargs...) @@ -257,8 +265,10 @@ function develop(ctx::Context, pkgs::Vector{PackageSpec}; shared::Bool=true, pkgerror("rev argument not supported by `develop`; consider using `add` instead") end if pkg.version != VersionSpec() - pkgerror("version specification invalid when calling `develop`:", - " `$(pkg.version)` specified for package $(err_rep(pkg))") + pkgerror( + "version specification invalid when calling `develop`:", + " `$(pkg.version)` specified for package $(err_rep(pkg))" + ) end # not strictly necessary to check these fields early, but it is more efficient if pkg.name !== nothing && (length(findall(x -> x.name == pkg.name, pkgs)) > 1) @@ -282,12 +292,14 @@ function develop(ctx::Context, pkgs::Vector{PackageSpec}; shared::Bool=true, update_source_if_set(ctx.env, pkg) end - Operations.develop(ctx, pkgs, new_git; preserve=preserve, platform=platform) + Operations.develop(ctx, pkgs, new_git; preserve = preserve, platform = platform) return end -function add(ctx::Context, pkgs::Vector{PackageSpec}; preserve::PreserveLevel=Operations.default_preserve(), - platform::AbstractPlatform=HostPlatform(), target::Symbol=:deps, allow_autoprecomp::Bool=true, kwargs...) +function add( + ctx::Context, pkgs::Vector{PackageSpec}; preserve::PreserveLevel = Operations.default_preserve(), + platform::AbstractPlatform = HostPlatform(), target::Symbol = :deps, allow_autoprecomp::Bool = true, kwargs... + ) require_not_empty(pkgs, :add) Context!(ctx; kwargs...) @@ -301,8 +313,10 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}; preserve::PreserveLevel=Op end if pkg.repo.source !== nothing || pkg.repo.rev !== nothing if pkg.version != VersionSpec() - pkgerror("version specification invalid when tracking a repository:", - " `$(pkg.version)` specified for package $(err_rep(pkg))") + pkgerror( + "version specification invalid when tracking a repository:", + " `$(pkg.version)` specified for package $(err_rep(pkg))" + ) end end # not strictly necessary to check these fields early, but it is more efficient @@ -319,12 +333,12 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}; preserve::PreserveLevel=Op # repo + unpinned -> name, uuid, repo.rev, repo.source, tree_hash # repo + pinned -> name, uuid, tree_hash - Operations.update_registries(ctx; force=false, update_cooldown=Day(1)) + Operations.update_registries(ctx; force = false, update_cooldown = Day(1)) project_deps_resolve!(ctx.env, pkgs) registry_resolve!(ctx.registries, pkgs) stdlib_resolve!(pkgs) - ensure_resolved(ctx, ctx.env.manifest, pkgs, registry=true) + ensure_resolved(ctx, ctx.env.manifest, pkgs, registry = true) for pkg in pkgs if Types.collides_with_project(ctx.env, pkg) @@ -340,7 +354,7 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}; preserve::PreserveLevel=Op return end -function rm(ctx::Context, pkgs::Vector{PackageSpec}; mode=PKGMODE_PROJECT, all_pkgs::Bool=false, kwargs...) +function rm(ctx::Context, pkgs::Vector{PackageSpec}; mode = PKGMODE_PROJECT, all_pkgs::Bool = false, kwargs...) Context!(ctx; kwargs...) if all_pkgs !isempty(pkgs) && pkgerror("cannot specify packages when operating on all packages") @@ -353,9 +367,11 @@ function rm(ctx::Context, pkgs::Vector{PackageSpec}; mode=PKGMODE_PROJECT, all_p if pkg.name === nothing && pkg.uuid === nothing pkgerror("name or UUID specification required when calling `rm`") end - if !(pkg.version == VersionSpec() && pkg.pinned == false && - pkg.tree_hash === nothing && pkg.repo.source === nothing && - pkg.repo.rev === nothing && pkg.path === nothing) + if !( + pkg.version == VersionSpec() && pkg.pinned == false && + pkg.tree_hash === nothing && pkg.repo.source === nothing && + pkg.repo.rev === nothing && pkg.path === nothing + ) pkgerror("packages may only be specified by name or UUID when calling `rm`") end end @@ -374,24 +390,26 @@ function append_all_pkgs!(pkgs, ctx, mode) if mode == PKGMODE_PROJECT || mode == PKGMODE_COMBINED for (name::String, uuid::UUID) in ctx.env.project.deps path, repo = get_path_repo(ctx.env.project, name) - push!(pkgs, PackageSpec(name=name, uuid=uuid, path=path, repo=repo)) + push!(pkgs, PackageSpec(name = name, uuid = uuid, path = path, repo = repo)) end end if mode == PKGMODE_MANIFEST || mode == PKGMODE_COMBINED for (uuid, entry) in ctx.env.manifest path, repo = get_path_repo(ctx.env.project, entry.name) - push!(pkgs, PackageSpec(name=entry.name, uuid=uuid, path=path, repo=repo)) + push!(pkgs, PackageSpec(name = entry.name, uuid = uuid, path = path, repo = repo)) end end return end -function up(ctx::Context, pkgs::Vector{PackageSpec}; - level::UpgradeLevel=UPLEVEL_MAJOR, mode::PackageMode=PKGMODE_PROJECT, - preserve::Union{Nothing,PreserveLevel}= isempty(pkgs) ? nothing : PRESERVE_ALL, - update_registry::Bool=true, - skip_writing_project::Bool=false, - kwargs...) +function up( + ctx::Context, pkgs::Vector{PackageSpec}; + level::UpgradeLevel = UPLEVEL_MAJOR, mode::PackageMode = PKGMODE_PROJECT, + preserve::Union{Nothing, PreserveLevel} = isempty(pkgs) ? nothing : PRESERVE_ALL, + update_registry::Bool = true, + skip_writing_project::Bool = false, + kwargs... + ) Context!(ctx; kwargs...) if Operations.is_fully_pinned(ctx) printpkgstyle(ctx.io, :Update, "All dependencies are pinned - nothing to update.", color = Base.info_color()) @@ -399,7 +417,7 @@ function up(ctx::Context, pkgs::Vector{PackageSpec}; end if update_registry Registry.download_default_registries(ctx.io) - Operations.update_registries(ctx; force=true) + Operations.update_registries(ctx; force = true) end Operations.prune_manifest(ctx.env) if isempty(pkgs) @@ -418,13 +436,13 @@ function up(ctx::Context, pkgs::Vector{PackageSpec}; return end -resolve(; io::IO=stderr_f(), kwargs...) = resolve(Context(;io); kwargs...) -function resolve(ctx::Context; skip_writing_project::Bool=false, kwargs...) - up(ctx; level=UPLEVEL_FIXED, mode=PKGMODE_MANIFEST, update_registry=false, skip_writing_project, kwargs...) +resolve(; io::IO = stderr_f(), kwargs...) = resolve(Context(; io); kwargs...) +function resolve(ctx::Context; skip_writing_project::Bool = false, kwargs...) + up(ctx; level = UPLEVEL_FIXED, mode = PKGMODE_MANIFEST, update_registry = false, skip_writing_project, kwargs...) return nothing end -function pin(ctx::Context, pkgs::Vector{PackageSpec}; all_pkgs::Bool=false, kwargs...) +function pin(ctx::Context, pkgs::Vector{PackageSpec}; all_pkgs::Bool = false, kwargs...) Context!(ctx; kwargs...) if all_pkgs !isempty(pkgs) && pkgerror("cannot specify packages when operating on all packages") @@ -438,12 +456,16 @@ function pin(ctx::Context, pkgs::Vector{PackageSpec}; all_pkgs::Bool=false, kwar pkgerror("name or UUID specification required when calling `pin`") end if pkg.repo.source !== nothing - pkgerror("repository specification invalid when calling `pin`:", - " `$(pkg.repo.source)` specified for package $(err_rep(pkg))") + pkgerror( + "repository specification invalid when calling `pin`:", + " `$(pkg.repo.source)` specified for package $(err_rep(pkg))" + ) end if pkg.repo.rev !== nothing - pkgerror("git revision specification invalid when calling `pin`:", - " `$(pkg.repo.rev)` specified for package $(err_rep(pkg))") + pkgerror( + "git revision specification invalid when calling `pin`:", + " `$(pkg.repo.rev)` specified for package $(err_rep(pkg))" + ) end version = pkg.version if version isa VersionSpec @@ -460,7 +482,7 @@ function pin(ctx::Context, pkgs::Vector{PackageSpec}; all_pkgs::Bool=false, kwar return end -function free(ctx::Context, pkgs::Vector{PackageSpec}; all_pkgs::Bool=false, kwargs...) +function free(ctx::Context, pkgs::Vector{PackageSpec}; all_pkgs::Bool = false, kwargs...) Context!(ctx; kwargs...) if all_pkgs !isempty(pkgs) && pkgerror("cannot specify packages when operating on all packages") @@ -473,9 +495,11 @@ function free(ctx::Context, pkgs::Vector{PackageSpec}; all_pkgs::Bool=false, kwa if pkg.name === nothing && pkg.uuid === nothing pkgerror("name or UUID specification required when calling `free`") end - if !(pkg.version == VersionSpec() && pkg.pinned == false && - pkg.tree_hash === nothing && pkg.repo.source === nothing && - pkg.repo.rev === nothing && pkg.path === nothing) + if !( + pkg.version == VersionSpec() && pkg.pinned == false && + pkg.tree_hash === nothing && pkg.repo.source === nothing && + pkg.repo.rev === nothing && pkg.path === nothing + ) pkgerror("packages may only be specified by name or UUID when calling `free`") end end @@ -487,14 +511,16 @@ function free(ctx::Context, pkgs::Vector{PackageSpec}; all_pkgs::Bool=false, kwa return end -function test(ctx::Context, pkgs::Vector{PackageSpec}; - coverage=false, test_fn=nothing, - julia_args::Union{Cmd, AbstractVector{<:AbstractString}}=``, - test_args::Union{Cmd, AbstractVector{<:AbstractString}}=``, - force_latest_compatible_version::Bool=false, - allow_earlier_backwards_compatible_versions::Bool=true, - allow_reresolve::Bool=true, - kwargs...) +function test( + ctx::Context, pkgs::Vector{PackageSpec}; + coverage = false, test_fn = nothing, + julia_args::Union{Cmd, AbstractVector{<:AbstractString}} = ``, + test_args::Union{Cmd, AbstractVector{<:AbstractString}} = ``, + force_latest_compatible_version::Bool = false, + allow_earlier_backwards_compatible_versions::Bool = true, + allow_reresolve::Bool = true, + kwargs... + ) julia_args = Cmd(julia_args) test_args = Cmd(test_args) Context!(ctx; kwargs...) @@ -532,8 +558,8 @@ function is_manifest_current(path::AbstractString) return Operations.is_manifest_current(env) end -const UsageDict = Dict{String,DateTime} -const UsageByDepotDict = Dict{String,UsageDict} +const UsageDict = Dict{String, DateTime} +const UsageByDepotDict = Dict{String, UsageDict} """ gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, kwargs...) @@ -551,7 +577,7 @@ admin privileges depending on the setup). Use verbose mode (`verbose=true`) for detailed output. """ -function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, force=false, kwargs...) +function gc(ctx::Context = Context(); collect_delay::Period = Day(7), verbose = false, force = false, kwargs...) Context!(ctx; kwargs...) env = ctx.env @@ -585,6 +611,7 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, for (filename, infos) in parse_toml(usage_filepath) f.(Ref(filename), infos) end + return end # Extract usage data from this depot, (taking only the latest state for each @@ -592,7 +619,7 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, # into the overall list across depots to create a single, coherent view across # all depots. usage = UsageDict() - let usage=usage + let usage = usage reduce_usage!(joinpath(logdir(depot), "manifest_usage.toml")) do filename, info # For Manifest usage, store only the last DateTime for each filename found usage[filename] = max(get(usage, filename, DateTime(0)), DateTime(info["time"])::DateTime) @@ -601,7 +628,7 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, manifest_usage_by_depot[depot] = usage usage = UsageDict() - let usage=usage + let usage = usage reduce_usage!(joinpath(logdir(depot), "artifact_usage.toml")) do filename, info # For Artifact usage, store only the last DateTime for each filename found usage[filename] = max(get(usage, filename, DateTime(0)), DateTime(info["time"])::DateTime) @@ -612,7 +639,7 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, # track last-used usage = UsageDict() parents = Dict{String, Set{String}}() - let usage=usage + let usage = usage reduce_usage!(joinpath(logdir(depot), "scratch_usage.toml")) do filename, info # For Artifact usage, store only the last DateTime for each filename found usage[filename] = max(get(usage, filename, DateTime(0)), DateTime(info["time"])::DateTime) @@ -653,19 +680,20 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, # Write out the TOML file for this depot usage_path = joinpath(logdir(depot), fname) if !(isempty(usage)::Bool) || isfile(usage_path) - let usage=usage - atomic_toml_write(usage_path, usage, sorted=true) + let usage = usage + atomic_toml_write(usage_path, usage, sorted = true) end end end + return end # Write condensed Manifest usage - let all_manifest_tomls=all_manifest_tomls + let all_manifest_tomls = all_manifest_tomls write_condensed_toml(manifest_usage_by_depot, "manifest_usage.toml") do depot, usage # Keep only manifest usage markers that are still existent - let usage=usage - filter!(((k,v),) -> k in all_manifest_tomls, usage) + let usage = usage + filter!(((k, v),) -> k in all_manifest_tomls, usage) # Expand it back into a dict-of-dicts return Dict(k => [Dict("time" => v)] for (k, v) in usage) @@ -674,23 +702,23 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, end # Write condensed Artifact usage - let all_artifact_tomls=all_artifact_tomls + let all_artifact_tomls = all_artifact_tomls write_condensed_toml(artifact_usage_by_depot, "artifact_usage.toml") do depot, usage let usage = usage - filter!(((k,v),) -> k in all_artifact_tomls, usage) + filter!(((k, v),) -> k in all_artifact_tomls, usage) return Dict(k => [Dict("time" => v)] for (k, v) in usage) end end end # Write condensed scratch space usage - let all_scratch_parents=all_scratch_parents, all_scratch_dirs=all_scratch_dirs + let all_scratch_parents = all_scratch_parents, all_scratch_dirs = all_scratch_dirs write_condensed_toml(scratch_usage_by_depot, "scratch_usage.toml") do depot, usage # Keep only scratch directories that still exist - filter!(((k,v),) -> k in all_scratch_dirs, usage) + filter!(((k, v),) -> k in all_scratch_dirs, usage) # Expand it back into a dict-of-dicts - expanded_usage = Dict{String,Vector{Dict}}() + expanded_usage = Dict{String, Vector{Dict}}() for (k, v) in usage # Drop scratch spaces whose parents are all non-existent parents = scratch_parents_by_depot[depot][k] @@ -699,10 +727,12 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, continue end - expanded_usage[k] = [Dict( - "time" => v, - "parent_projects" => collect(parents), - )] + expanded_usage[k] = [ + Dict( + "time" => v, + "parent_projects" => collect(parents), + ), + ] end return expanded_usage end @@ -790,7 +820,7 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, end # Mark packages/artifacts as active or not by calling the appropriate user function - function mark(process_func::Function, index_files, ctx::Context; do_print=true, verbose=false, file_str=nothing) + function mark(process_func::Function, index_files, ctx::Context; do_print = true, verbose = false, file_str = nothing) marked_paths = String[] active_index_files = Set{String}() for index_file in index_files @@ -841,13 +871,16 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, push!(deletion_list, path) end end + return end # Scan manifests, parse them, read in all UUIDs listed and mark those as active # printpkgstyle(ctx.io, :Active, "manifests:") - packages_to_keep = mark(process_manifest_pkgs, all_manifest_tomls, ctx, - verbose=verbose, file_str="manifest files") + packages_to_keep = mark( + process_manifest_pkgs, all_manifest_tomls, ctx, + verbose = verbose, file_str = "manifest files" + ) # Do an initial scan of our depots to get a preliminary `packages_to_delete`. packages_to_delete = String[] @@ -876,15 +909,19 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, # `packages_to_delete`, as `process_artifacts_toml()` uses it internally to discount # `Artifacts.toml` files that will be deleted by the future culling operation. # printpkgstyle(ctx.io, :Active, "artifacts:") - artifacts_to_keep = let packages_to_delete=packages_to_delete - mark(x -> process_artifacts_toml(x, packages_to_delete), - all_artifact_tomls, ctx; verbose=verbose, file_str="artifact files") + artifacts_to_keep = let packages_to_delete = packages_to_delete + mark( + x -> process_artifacts_toml(x, packages_to_delete), + all_artifact_tomls, ctx; verbose = verbose, file_str = "artifact files" + ) end - repos_to_keep = mark(process_manifest_repos, all_manifest_tomls, ctx; do_print=false) + repos_to_keep = mark(process_manifest_repos, all_manifest_tomls, ctx; do_print = false) # printpkgstyle(ctx.io, :Active, "scratchspaces:") - spaces_to_keep = let packages_to_delete=packages_to_delete - mark(x -> process_scratchspace(x, packages_to_delete), - all_scratch_dirs, ctx; verbose=verbose, file_str="scratchspaces") + spaces_to_keep = let packages_to_delete = packages_to_delete + mark( + x -> process_scratchspace(x, packages_to_delete), + all_scratch_dirs, ctx; verbose = verbose, file_str = "scratchspaces" + ) end # Collect all orphaned paths (packages, artifacts and repos that are not reachable). These @@ -956,8 +993,8 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, end elseif uuid == Operations.PkgUUID && isfile(space_dir_or_file) # special cleanup for the precompile cache files that Pkg saves - if any(prefix->startswith(basename(space_dir_or_file), prefix), ("suspend_cache_", "pending_cache_")) - if mtime(space_dir_or_file) < (time() - (24*60*60)) + if any(prefix -> startswith(basename(space_dir_or_file), prefix), ("suspend_cache_", "pending_cache_")) + if mtime(space_dir_or_file) < (time() - (24 * 60 * 60)) push!(depot_orphaned_scratchspaces, space_dir_or_file) end end @@ -984,7 +1021,7 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, # Write out the `new_orphanage` for this depot mkpath(dirname(orphanage_file)) - atomic_toml_write(orphanage_file, new_orphanage, sorted=true) + atomic_toml_write(orphanage_file, new_orphanage, sorted = true) end function recursive_dir_size(path) @@ -996,12 +1033,12 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, try size += lstat(path).size catch ex - @error("Failed to calculate size of $path", exception=ex) + @error("Failed to calculate size of $path", exception = ex) end end end catch ex - @error("Failed to calculate size of $path", exception=ex) + @error("Failed to calculate size of $path", exception = ex) end return size end @@ -1012,7 +1049,7 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, try lstat(path).size catch ex - @error("Failed to calculate size of $path", exception=ex) + @error("Failed to calculate size of $path", exception = ex) 0 end else @@ -1020,14 +1057,16 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, end try Base.Filesystem.prepare_for_deletion(path) - Base.rm(path; recursive=true, force=true) + Base.rm(path; recursive = true, force = true) catch e - @warn("Failed to delete $path", exception=e) + @warn("Failed to delete $path", exception = e) return 0 end if verbose - printpkgstyle(ctx.io, :Deleted, pathrepr(path) * " (" * - Base.format_bytes(path_size) * ")") + printpkgstyle( + ctx.io, :Deleted, pathrepr(path) * " (" * + Base.format_bytes(path_size) * ")" + ) end return path_size end @@ -1081,12 +1120,12 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, # Do this silently because it's out of scope for Pkg.gc() but it's helpful to use this opportunity to do it if isdefined(Base.Filesystem, :delayed_delete_dir) if isdir(Base.Filesystem.delayed_delete_dir()) - for p in readdir(Base.Filesystem.delayed_delete_dir(), join=true) + for p in readdir(Base.Filesystem.delayed_delete_dir(), join = true) try Base.Filesystem.prepare_for_deletion(p) - Base.rm(p; recursive=true, force=true, allow_delayed_delete=false) + Base.rm(p; recursive = true, force = true, allow_delayed_delete = false) catch e - @debug "Failed to delete $p" exception=e + @debug "Failed to delete $p" exception = e end end end @@ -1104,7 +1143,7 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, s = ndel == 1 ? "" : "s" bytes_saved_string = Base.format_bytes(freed) - printpkgstyle(ctx.io, :Deleted, "$(ndel) $(name)$(s) ($bytes_saved_string)") + return printpkgstyle(ctx.io, :Deleted, "$(ndel) $(name)$(s) ($bytes_saved_string)") end print_deleted(ndel_pkg, package_space_freed, "package installation") print_deleted(ndel_repo, repo_space_freed, "repo") @@ -1118,7 +1157,7 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, return end -function build(ctx::Context, pkgs::Vector{PackageSpec}; verbose=false, allow_reresolve::Bool=true, kwargs...) +function build(ctx::Context, pkgs::Vector{PackageSpec}; verbose = false, allow_reresolve::Bool = true, kwargs...) Context!(ctx; kwargs...) if isempty(pkgs) @@ -1133,7 +1172,7 @@ function build(ctx::Context, pkgs::Vector{PackageSpec}; verbose=false, allow_rer project_resolve!(ctx.env, pkgs) manifest_resolve!(ctx.env.manifest, pkgs) ensure_resolved(ctx, ctx.env.manifest, pkgs) - Operations.build(ctx, Set{UUID}(pkg.uuid for pkg in pkgs), verbose; allow_reresolve) + return Operations.build(ctx, Set{UUID}(pkg.uuid for pkg in pkgs), verbose; allow_reresolve) end function get_or_make_pkgspec(pkgspecs::Vector{PackageSpec}, ctx::Context, uuid) @@ -1155,13 +1194,15 @@ function get_or_make_pkgspec(pkgspecs::Vector{PackageSpec}, ctx::Context, uuid) end end -function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool=false, - strict::Bool=false, warn_loaded = true, already_instantiated = false, timing::Bool = false, - _from_loading::Bool=false, configs::Union{Base.Precompilation.Config,Vector{Base.Precompilation.Config}}=(``=>Base.CacheFlags()), - workspace::Bool=false, kwargs...) +function precompile( + ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool = false, + strict::Bool = false, warn_loaded = true, already_instantiated = false, timing::Bool = false, + _from_loading::Bool = false, configs::Union{Base.Precompilation.Config, Vector{Base.Precompilation.Config}} = (`` => Base.CacheFlags()), + workspace::Bool = false, kwargs... + ) Context!(ctx; kwargs...) if !already_instantiated - instantiate(ctx; allow_autoprecomp=false, kwargs...) + instantiate(ctx; allow_autoprecomp = false, kwargs...) @debug "precompile: instantiated" end @@ -1181,14 +1222,14 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool io = io.io end - activate(dirname(ctx.env.project_file)) do + return activate(dirname(ctx.env.project_file)) do pkgs_name = String[pkg.name for pkg in pkgs] - return Base.Precompilation.precompilepkgs(pkgs_name; internal_call, strict, warn_loaded, timing, _from_loading, configs, manifest=workspace, io) + return Base.Precompilation.precompilepkgs(pkgs_name; internal_call, strict, warn_loaded, timing, _from_loading, configs, manifest = workspace, io) end end function precompile(f, args...; kwargs...) - Base.ScopedValues.@with _autoprecompilation_enabled_scoped => false begin + return Base.ScopedValues.@with _autoprecompilation_enabled_scoped => false begin f() Pkg.precompile(args...; kwargs...) end @@ -1204,10 +1245,12 @@ function tree_hash(repo::LibGit2.GitRepo, tree_hash::String) end instantiate(; kwargs...) = instantiate(Context(); kwargs...) -function instantiate(ctx::Context; manifest::Union{Bool, Nothing}=nothing, - update_registry::Bool=true, verbose::Bool=false, - platform::AbstractPlatform=HostPlatform(), allow_build::Bool=true, allow_autoprecomp::Bool=true, - workspace::Bool=false, julia_version_strict::Bool=false, kwargs...) +function instantiate( + ctx::Context; manifest::Union{Bool, Nothing} = nothing, + update_registry::Bool = true, verbose::Bool = false, + platform::AbstractPlatform = HostPlatform(), allow_build::Bool = true, allow_autoprecomp::Bool = true, + workspace::Bool = false, julia_version_strict::Bool = false, kwargs... + ) Context!(ctx; kwargs...) if Registry.download_default_registries(ctx.io) copy!(ctx.registries, Registry.reachable_registries()) @@ -1215,7 +1258,7 @@ function instantiate(ctx::Context; manifest::Union{Bool, Nothing}=nothing, if !isfile(ctx.env.project_file) && isfile(ctx.env.manifest_file) _manifest = Pkg.Types.read_manifest(ctx.env.manifest_file) Types.check_manifest_julia_version_compat(_manifest, ctx.env.manifest_file; julia_version_strict) - deps = Dict{String,String}() + deps = Dict{String, String}() for (uuid, pkg) in _manifest if pkg.name in keys(deps) # TODO, query what package to put in Project when in interactive mode? @@ -1224,7 +1267,7 @@ function instantiate(ctx::Context; manifest::Union{Bool, Nothing}=nothing, deps[pkg.name] = string(uuid) end Types.write_project(Dict("deps" => deps), ctx.env.project_file) - return instantiate(Context(); manifest=manifest, update_registry=update_registry, allow_autoprecomp=allow_autoprecomp, verbose=verbose, platform=platform, kwargs...) + return instantiate(Context(); manifest = manifest, update_registry = update_registry, allow_autoprecomp = allow_autoprecomp, verbose = verbose, platform = platform, kwargs...) end if (!isfile(ctx.env.manifest_file) && manifest === nothing) || manifest == false # given no manifest exists, only allow invoking a registry update if there are project deps @@ -1251,10 +1294,12 @@ function instantiate(ctx::Context; manifest::Union{Bool, Nothing}=nothing, resolve_cmd = Pkg.in_repl_mode() ? "pkg> resolve" : "Pkg.resolve()" rm_cmd = Pkg.in_repl_mode() ? "pkg> rm $name" : "Pkg.rm(\"$name\")" instantiate_cmd = Pkg.in_repl_mode() ? "pkg> instantiate" : "Pkg.instantiate()" - pkgerror("`$name` is a direct dependency, but does not appear in the manifest.", - " If you intend `$name` to be a direct dependency, run `$resolve_cmd` to populate the manifest.", - " Otherwise, remove `$name` with `$rm_cmd`.", - " Finally, run `$instantiate_cmd` again.") + pkgerror( + "`$name` is a direct dependency, but does not appear in the manifest.", + " If you intend `$name` to be a direct dependency, run `$resolve_cmd` to populate the manifest.", + " Otherwise, remove `$name` with `$rm_cmd`.", + " Finally, run `$instantiate_cmd` again." + ) end # check if all source code and artifacts are downloaded to exit early if Operations.is_instantiated(ctx.env, workspace; platform) @@ -1274,7 +1319,7 @@ function instantiate(ctx::Context; manifest::Union{Bool, Nothing}=nothing, if !(e isa PkgError) || update_registry == false rethrow(e) end - Operations.update_registries(ctx; force=false) + Operations.update_registries(ctx; force = false) Operations.check_registered(ctx.registries, pkgs) end new_git = UUID[] @@ -1293,12 +1338,12 @@ function instantiate(ctx::Context; manifest::Union{Bool, Nothing}=nothing, pkgerror("Did not find path `$(repo_source)` for $(err_rep(pkg))") end repo_path = Types.add_repo_cache_path(repo_source) - let repo_source=repo_source - LibGit2.with(GitTools.ensure_clone(ctx.io, repo_path, repo_source; isbare=true)) do repo + let repo_source = repo_source + LibGit2.with(GitTools.ensure_clone(ctx.io, repo_path, repo_source; isbare = true)) do repo # We only update the clone if the tree hash can't be found tree_hash_object = tree_hash(repo, string(pkg.tree_hash)) if tree_hash_object === nothing - GitTools.fetch(ctx.io, repo, repo_source; refspecs=Types.refspecs) + GitTools.fetch(ctx.io, repo, repo_source; refspecs = Types.refspecs) tree_hash_object = tree_hash(repo, string(pkg.tree_hash)) end if tree_hash_object === nothing @@ -1316,35 +1361,35 @@ function instantiate(ctx::Context; manifest::Union{Bool, Nothing}=nothing, # Install all artifacts Operations.download_artifacts(ctx; platform, verbose) # Run build scripts - allow_build && Operations.build_versions(ctx, union(new_apply, new_git); verbose=verbose) + allow_build && Operations.build_versions(ctx, union(new_apply, new_git); verbose = verbose) - allow_autoprecomp && Pkg._auto_precompile(ctx, already_instantiated = true) + return allow_autoprecomp && Pkg._auto_precompile(ctx, already_instantiated = true) end -@deprecate status(mode::PackageMode) status(mode=mode) +@deprecate status(mode::PackageMode) status(mode = mode) -function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool=false, mode=PKGMODE_PROJECT, workspace::Bool=false, outdated::Bool=false, compat::Bool=false, extensions::Bool=false, io::IO=stdout_f()) +function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool = false, mode = PKGMODE_PROJECT, workspace::Bool = false, outdated::Bool = false, compat::Bool = false, extensions::Bool = false, io::IO = stdout_f()) if compat diff && pkgerror("Compat status has no `diff` mode") outdated && pkgerror("Compat status has no `outdated` mode") extensions && pkgerror("Compat status has no `extensions` mode") Operations.print_compat(ctx, pkgs; io) else - Operations.status(ctx.env, ctx.registries, pkgs; mode, git_diff=diff, io, outdated, extensions, workspace) + Operations.status(ctx.env, ctx.registries, pkgs; mode, git_diff = diff, io, outdated, extensions, workspace) end return nothing end -function activate(;temp=false, shared=false, prev=false, io::IO=stderr_f()) +function activate(; temp = false, shared = false, prev = false, io::IO = stderr_f()) shared && pkgerror("Must give a name for a shared environment") - temp && return activate(mktempdir(); io=io) + temp && return activate(mktempdir(); io = io) if prev if isempty(PREV_ENV_PATH[]) pkgerror("No previously active environment found") else - return activate(PREV_ENV_PATH[]; io=io) + return activate(PREV_ENV_PATH[]; io = io) end end if !isnothing(Base.active_project()) @@ -1366,14 +1411,14 @@ function _activate_dep(dep_name::AbstractString) return end uuid = get(ctx.env.project.deps, dep_name, nothing) - if uuid !== nothing + return if uuid !== nothing entry = manifest_info(ctx.env.manifest, uuid) if entry.path !== nothing return joinpath(dirname(ctx.env.manifest_file), entry.path::String) end end end -function activate(path::AbstractString; shared::Bool=false, temp::Bool=false, io::IO=stderr_f()) +function activate(path::AbstractString; shared::Bool = false, temp::Bool = false, io::IO = stderr_f()) temp && pkgerror("Can not give `path` argument when creating a temporary environment") if !shared # `pkg> activate path`/`Pkg.activate(path)` does the following @@ -1420,19 +1465,19 @@ end function activate(f::Function, new_project::AbstractString) old = Base.ACTIVE_PROJECT[] Base.ACTIVE_PROJECT[] = new_project - try + return try f() finally Base.ACTIVE_PROJECT[] = old end end -function _compat(ctx::Context, pkg::String, compat_str::Union{Nothing,String}; current::Bool=false, io = nothing, kwargs...) +function _compat(ctx::Context, pkg::String, compat_str::Union{Nothing, String}; current::Bool = false, io = nothing, kwargs...) if current if compat_str !== nothing pkgerror("`current` is true, but `compat_str` is not nothing. This is not allowed.") end - return set_current_compat(ctx, pkg; io=io) + return set_current_compat(ctx, pkg; io = io) end io = something(io, ctx.io) pkg = pkg == "Julia" ? "julia" : pkg @@ -1472,16 +1517,16 @@ function _compat(ctx::Context, pkg::String, compat_str::Union{Nothing,String}; c pkgerror("No package named $pkg in current Project") end end -function compat(ctx::Context=Context(); current::Bool=false, kwargs...) +function compat(ctx::Context = Context(); current::Bool = false, kwargs...) if current return set_current_compat(ctx; kwargs...) end return _compat(ctx; kwargs...) end -compat(pkg::String, compat_str::Union{Nothing,String}=nothing; kwargs...) = _compat(Context(), pkg, compat_str; kwargs...) +compat(pkg::String, compat_str::Union{Nothing, String} = nothing; kwargs...) = _compat(Context(), pkg, compat_str; kwargs...) -function set_current_compat(ctx::Context, target_pkg::Union{Nothing,String}=nothing; io = nothing) +function set_current_compat(ctx::Context, target_pkg::Union{Nothing, String} = nothing; io = nothing) io = something(io, ctx.io) updated_deps = String[] @@ -1538,15 +1583,15 @@ function set_current_compat(ctx::Context, target_pkg::Union{Nothing,String}=noth end write_env(ctx.env) - Operations.print_compat(ctx; io) + return Operations.print_compat(ctx; io) end -set_current_compat(;kwargs...) = set_current_compat(Context(); kwargs...) +set_current_compat(; kwargs...) = set_current_compat(Context(); kwargs...) ####### # why # ####### -function why(ctx::Context, pkgs::Vector{PackageSpec}; io::IO, workspace::Bool=false, kwargs...) +function why(ctx::Context, pkgs::Vector{PackageSpec}; io::IO, workspace::Bool = false, kwargs...) require_not_empty(pkgs, :why) manifest_resolve!(ctx.env.manifest, pkgs) @@ -1584,6 +1629,7 @@ function why(ctx::Context, pkgs::Vector{PackageSpec}; io::IO, workspace::Bool=fa end find_paths!(final_paths, p, copy(path)) end + return end first = true @@ -1595,11 +1641,12 @@ function why(ctx::Context, pkgs::Vector{PackageSpec}; io::IO, workspace::Bool=fa foreach(reverse!, final_paths) final_paths_names = map(x -> [ctx.env.manifest[uuid].name for uuid in x], collect(final_paths)) sort!(final_paths_names, by = x -> (x, length(x))) - delimiter = sprint((io, args) -> printstyled(io, args...; color=:light_green), "→", context=io) + delimiter = sprint((io, args) -> printstyled(io, args...; color = :light_green), "→", context = io) for path in final_paths_names println(io, " ", join(path, " $delimiter ")) end end + return end @@ -1621,7 +1668,7 @@ const undo_entries = Dict{String, UndoState}() const max_undo_limit = 50 const saved_initial_snapshot = Ref(false) -function add_snapshot_to_undo(env=nothing) +function add_snapshot_to_undo(env = nothing) # only attempt to take a snapshot if there is # an active project to be found if env === nothing @@ -1639,14 +1686,14 @@ function add_snapshot_to_undo(env=nothing) return end snapshot = UndoSnapshot(now(), env.project, env.manifest) - deleteat!(state.entries, 1:(state.idx-1)) + deleteat!(state.entries, 1:(state.idx - 1)) pushfirst!(state.entries, snapshot) state.idx = 1 - resize!(state.entries, min(length(state.entries), max_undo_limit)) + return resize!(state.entries, min(length(state.entries), max_undo_limit)) end -undo(ctx = Context()) = redo_undo(ctx, :undo, 1) +undo(ctx = Context()) = redo_undo(ctx, :undo, 1) redo(ctx = Context()) = redo_undo(ctx, :redo, -1) function redo_undo(ctx, mode::Symbol, direction::Int) @assert direction == 1 || direction == -1 @@ -1657,16 +1704,16 @@ function redo_undo(ctx, mode::Symbol, direction::Int) state.idx += direction snapshot = state.entries[state.idx] ctx.env.manifest, ctx.env.project = snapshot.manifest, snapshot.project - write_env(ctx.env; update_undo=false) - Operations.show_update(ctx.env, ctx.registries; io=ctx.io) + write_env(ctx.env; update_undo = false) + return Operations.show_update(ctx.env, ctx.registries; io = ctx.io) end function setprotocol!(; - domain::AbstractString="github.com", - protocol::Union{Nothing, AbstractString}=nothing -) - GitTools.setprotocol!(domain=domain, protocol=protocol) + domain::AbstractString = "github.com", + protocol::Union{Nothing, AbstractString} = nothing + ) + GitTools.setprotocol!(domain = domain, protocol = protocol) return nothing end @@ -1679,8 +1726,10 @@ function handle_package_input!(pkg::PackageSpec) if pkg.repo.source !== nothing || pkg.repo.rev !== nothing || pkg.repo.subdir !== nothing pkgerror("`repo` is a private field of PackageSpec and should not be set directly") end - pkg.repo = Types.GitRepo(rev = pkg.rev, source = pkg.url !== nothing ? pkg.url : pkg.path, - subdir = pkg.subdir) + pkg.repo = Types.GitRepo( + rev = pkg.rev, source = pkg.url !== nothing ? pkg.url : pkg.path, + subdir = pkg.subdir + ) pkg.path = nothing pkg.tree_hash = nothing if pkg.version === nothing @@ -1689,7 +1738,7 @@ function handle_package_input!(pkg::PackageSpec) if !(pkg.version isa VersionNumber) pkg.version = VersionSpec(pkg.version) end - pkg.uuid = pkg.uuid isa String ? UUID(pkg.uuid) : pkg.uuid + return pkg.uuid = pkg.uuid isa String ? UUID(pkg.uuid) : pkg.uuid end function upgrade_manifest(man_path::String) @@ -1698,7 +1747,7 @@ function upgrade_manifest(man_path::String) Pkg.activate(dir) do Pkg.upgrade_manifest() end - mv(joinpath(dir, "Manifest.toml"), man_path, force = true) + return mv(joinpath(dir, "Manifest.toml"), man_path, force = true) end function upgrade_manifest(ctx::Context = Context()) before_format = ctx.env.manifest.manifest_format diff --git a/src/Apps/Apps.jl b/src/Apps/Apps.jl index 3126a65478..e5cfd8b10f 100644 --- a/src/Apps/Apps.jl +++ b/src/Apps/Apps.jl @@ -4,7 +4,7 @@ using Pkg using Pkg: atomic_toml_write using Pkg.Versions using Pkg.Types: AppInfo, PackageSpec, Context, EnvCache, PackageEntry, Manifest, handle_repo_add!, handle_repo_develop!, write_manifest, write_project, - pkgerror, projectfile_path, manifestfile_path + pkgerror, projectfile_path, manifestfile_path using Pkg.Operations: print_single, source_path, update_package_add using Pkg.API: handle_package_input! using TOML, UUIDs @@ -14,7 +14,7 @@ app_env_folder() = joinpath(first(DEPOT_PATH), "environments", "apps") app_manifest_file() = joinpath(app_env_folder(), "AppManifest.toml") julia_bin_path() = joinpath(first(DEPOT_PATH), "bin") -app_context() = Context(env=EnvCache(joinpath(app_env_folder(), "Project.toml"))) +app_context() = Context(env = EnvCache(joinpath(app_env_folder(), "Project.toml"))) function validate_app_name(name::AbstractString) if isempty(name) @@ -23,7 +23,7 @@ function validate_app_name(name::AbstractString) if !occursin(r"^[a-zA-Z][a-zA-Z0-9_-]*$", name) error("App name must start with a letter and contain only letters, numbers, underscores, and hyphens") end - if occursin(r"\.\.", name) || occursin(r"[/\\]", name) + return if occursin(r"\.\.", name) || occursin(r"[/\\]", name) error("App name cannot contain path traversal sequences or path separators") end end @@ -32,13 +32,13 @@ function validate_package_name(name::AbstractString) if isempty(name) error("Package name cannot be empty") end - if !occursin(r"^[a-zA-Z][a-zA-Z0-9_]*$", name) + return if !occursin(r"^[a-zA-Z][a-zA-Z0-9_]*$", name) error("Package name must start with a letter and contain only letters, numbers, and underscores") end end -function validate_submodule_name(name::Union{AbstractString,Nothing}) - if name !== nothing +function validate_submodule_name(name::Union{AbstractString, Nothing}) + return if name !== nothing if isempty(name) error("Submodule name cannot be empty") end @@ -51,7 +51,7 @@ end function rm_shim(name; kwargs...) validate_app_name(name) - Base.rm(joinpath(julia_bin_path(), name * (Sys.iswindows() ? ".bat" : "")); kwargs...) + return Base.rm(joinpath(julia_bin_path(), name * (Sys.iswindows() ? ".bat" : "")); kwargs...) end function get_project(sourcepath) @@ -66,7 +66,7 @@ end function overwrite_file_if_different(file, content) - if !isfile(file) || read(file, String) != content + return if !isfile(file) || read(file, String) != content mkpath(dirname(file)) write(file, content) end @@ -79,7 +79,7 @@ function check_apps_in_path(apps) @warn """ App '$app_name' was installed but is not available in PATH. Consider adding '$(julia_bin_path())' to your PATH environment variable. - """ maxlog=1 + """ maxlog = 1 break # Only show warning once per installation else # Check for collisions @@ -94,6 +94,7 @@ function check_apps_in_path(apps) end end end + return end function get_max_version_register(pkg::PackageSpec, regs) @@ -129,7 +130,7 @@ end # Main Functions # ################## -function _resolve(manifest::Manifest, pkgname=nothing) +function _resolve(manifest::Manifest, pkgname = nothing) for (uuid, pkg) in manifest.deps if pkgname !== nothing && pkg.name !== pkgname continue @@ -145,7 +146,7 @@ function _resolve(manifest::Manifest, pkgname=nothing) mkpath(dirname(projectfile)) if isfile(original_project_file) - cp(original_project_file, projectfile; force=true) + cp(original_project_file, projectfile; force = true) chmod(projectfile, 0o644) # Make the copied project file writable # Add entryfile stanza pointing to the package entry file @@ -167,7 +168,7 @@ function _resolve(manifest::Manifest, pkgname=nothing) # TODO: Julia path generate_shims_for_apps(pkg.name, pkg.apps, dirname(projectfile), joinpath(Sys.BINDIR, "julia")) end - write_manifest(manifest, app_manifest_file()) + return write_manifest(manifest, app_manifest_file()) end @@ -175,6 +176,7 @@ function add(pkg::Vector{PackageSpec}) for p in pkg add(p) end + return end @@ -193,7 +195,7 @@ function add(pkg::PackageSpec) else pkgs = [pkg] Pkg.Operations.registry_resolve!(ctx.registries, pkgs) - Pkg.Operations.ensure_resolved(ctx, manifest, pkgs, registry=true) + Pkg.Operations.ensure_resolved(ctx, manifest, pkgs, registry = true) pkg.version, pkg.tree_hash = get_max_version_register(pkg, ctx.registries) @@ -202,24 +204,25 @@ function add(pkg::PackageSpec) # Run Pkg.build()? - Base.rm(joinpath(app_env_folder(), pkg.name); force=true, recursive=true) + Base.rm(joinpath(app_env_folder(), pkg.name); force = true, recursive = true) sourcepath = source_path(ctx.env.manifest_file, pkg) project = get_project(sourcepath) # TODO: Wrong if package itself has a sourcepath? - entry = PackageEntry(;apps = project.apps, name = pkg.name, version = project.version, tree_hash = pkg.tree_hash, path = pkg.path, repo = pkg.repo, uuid=pkg.uuid) + entry = PackageEntry(; apps = project.apps, name = pkg.name, version = project.version, tree_hash = pkg.tree_hash, path = pkg.path, repo = pkg.repo, uuid = pkg.uuid) manifest.deps[pkg.uuid] = entry _resolve(manifest, pkg.name) precompile(pkg.name) @info "For package: $(pkg.name) installed apps $(join(keys(project.apps), ","))" - check_apps_in_path(project.apps) + return check_apps_in_path(project.apps) end function develop(pkg::Vector{PackageSpec}) for p in pkg develop(p) end + return end function develop(pkg::PackageSpec) @@ -229,7 +232,7 @@ function develop(pkg::PackageSpec) handle_package_input!(pkg) ctx = app_context() handle_repo_develop!(ctx, pkg, #=shared =# true) - Base.rm(joinpath(app_env_folder(), pkg.name); force=true, recursive=true) + Base.rm(joinpath(app_env_folder(), pkg.name); force = true, recursive = true) sourcepath = abspath(source_path(ctx.env.manifest_file, pkg)) project = get_project(sourcepath) @@ -242,14 +245,14 @@ function develop(pkg::PackageSpec) end - entry = PackageEntry(;apps = project.apps, name = pkg.name, version = project.version, tree_hash = pkg.tree_hash, path = sourcepath, repo = pkg.repo, uuid=pkg.uuid) + entry = PackageEntry(; apps = project.apps, name = pkg.name, version = project.version, tree_hash = pkg.tree_hash, path = sourcepath, repo = pkg.repo, uuid = pkg.uuid) manifest = ctx.env.manifest manifest.deps[pkg.uuid] = entry _resolve(manifest, pkg.name) precompile(pkg.name) @info "For package: $(pkg.name) installed apps: $(join(keys(project.apps), ","))" - check_apps_in_path(project.apps) + return check_apps_in_path(project.apps) end @@ -261,10 +264,11 @@ function update(pkgs_or_apps::Vector) end update(pkg_or_app) end + return end # XXX: Is updating an app ever different from rm-ing and adding it from scratch? -function update(pkg::Union{PackageSpec, Nothing}=nothing) +function update(pkg::Union{PackageSpec, Nothing} = nothing) ctx = app_context() manifest = ctx.env.manifest deps = Pkg.Operations.load_manifest_deps(manifest) @@ -288,8 +292,10 @@ function update(pkg::Union{PackageSpec, Nothing}=nothing) manifest_app = Pkg.Types.read_manifest(manifest_file) manifest_entry = manifest_app.deps[info.uuid] - entry = PackageEntry(;apps = project.apps, name = manifest_entry.name, version = manifest_entry.version, tree_hash = manifest_entry.tree_hash, - path = manifest_entry.path, repo = manifest_entry.repo, uuid = manifest_entry.uuid) + entry = PackageEntry(; + apps = project.apps, name = manifest_entry.name, version = manifest_entry.version, tree_hash = manifest_entry.tree_hash, + path = manifest_entry.path, repo = manifest_entry.repo, uuid = manifest_entry.uuid + ) manifest.deps[dep.uuid] = entry Pkg.Types.write_manifest(manifest, app_manifest_file()) @@ -298,7 +304,7 @@ function update(pkg::Union{PackageSpec, Nothing}=nothing) end function status(pkgs_or_apps::Vector) - if isempty(pkgs_or_apps) + return if isempty(pkgs_or_apps) status() else for pkg_or_app in pkgs_or_apps @@ -310,7 +316,7 @@ function status(pkgs_or_apps::Vector) end end -function status(pkg_or_app::Union{PackageSpec, Nothing}=nothing) +function status(pkg_or_app::Union{PackageSpec, Nothing} = nothing) # TODO: Sort. pkg_or_app = pkg_or_app === nothing ? nothing : pkg_or_app.name manifest = Pkg.Types.read_manifest(joinpath(app_env_folder(), "AppManifest.toml")) @@ -337,13 +343,14 @@ function status(pkg_or_app::Union{PackageSpec, Nothing}=nothing) continue end julia_cmd = contractuser(appinfo.julia_command) - printstyled(" $(appname)", color=:green) - printstyled(" $(julia_cmd) \n", color=:gray) + printstyled(" $(appname)", color = :green) + printstyled(" $(julia_cmd) \n", color = :gray) end end + return end -function precompile(pkg::Union{Nothing, String}=nothing) +function precompile(pkg::Union{Nothing, String} = nothing) manifest = Pkg.Types.read_manifest(joinpath(app_env_folder(), "AppManifest.toml")) deps = Pkg.Operations.load_manifest_deps(manifest) for dep in deps @@ -357,11 +364,12 @@ function precompile(pkg::Union{Nothing, String}=nothing) Pkg.precompile() end end + return end function require_not_empty(pkgs, f::Symbol) - if pkgs === nothing || isempty(pkgs) + return if pkgs === nothing || isempty(pkgs) pkgerror("app $f requires at least one package") end end @@ -374,9 +382,10 @@ function rm(pkgs_or_apps::Vector) end rm(pkg_or_app) end + return end -function rm(pkg_or_app::Union{PackageSpec, Nothing}=nothing) +function rm(pkg_or_app::Union{PackageSpec, Nothing} = nothing) pkg_or_app = pkg_or_app === nothing ? nothing : pkg_or_app.name require_not_empty(pkg_or_app, :rm) @@ -389,10 +398,10 @@ function rm(pkg_or_app::Union{PackageSpec, Nothing}=nothing) delete!(manifest.deps, dep.uuid) for (appname, appinfo) in dep.apps @info "Deleted $(appname)" - rm_shim(appname; force=true) + rm_shim(appname; force = true) end if dep.path === nothing - Base.rm(joinpath(app_env_folder(), dep.name); recursive=true) + Base.rm(joinpath(app_env_folder(), dep.name); recursive = true) end else for (uuid, pkg) in manifest.deps @@ -401,11 +410,11 @@ function rm(pkg_or_app::Union{PackageSpec, Nothing}=nothing) app = pkg.apps[app_idx] @info "Deleted app $(app.name)" delete!(pkg.apps, app.name) - rm_shim(app.name; force=true) + rm_shim(app.name; force = true) end if isempty(pkg.apps) delete!(manifest.deps, uuid) - Base.rm(joinpath(app_env_folder(), pkg.name); recursive=true) + Base.rm(joinpath(app_env_folder(), pkg.name); recursive = true) end end end @@ -417,19 +426,21 @@ end for f in (:develop, :add) @eval begin $f(pkg::Union{AbstractString, PackageSpec}; kwargs...) = $f([pkg]; kwargs...) - $f(pkgs::Vector{<:AbstractString}; kwargs...) = $f([PackageSpec(pkg) for pkg in pkgs]; kwargs...) - function $f(; name::Union{Nothing,AbstractString}=nothing, uuid::Union{Nothing,String,UUID}=nothing, - version::Union{VersionNumber, String, VersionSpec, Nothing}=nothing, - url=nothing, rev=nothing, path=nothing, subdir=nothing, kwargs...) + $f(pkgs::Vector{<:AbstractString}; kwargs...) = $f([PackageSpec(pkg) for pkg in pkgs]; kwargs...) + function $f(; + name::Union{Nothing, AbstractString} = nothing, uuid::Union{Nothing, String, UUID} = nothing, + version::Union{VersionNumber, String, VersionSpec, Nothing} = nothing, + url = nothing, rev = nothing, path = nothing, subdir = nothing, kwargs... + ) pkg = PackageSpec(; name, uuid, version, url, rev, path, subdir) - if all(isnothing, [name,uuid,version,url,rev,path,subdir]) + return if all(isnothing, [name, uuid, version, url, rev, path, subdir]) $f(PackageSpec[]; kwargs...) else $f(pkg; kwargs...) end end function $f(pkgs::Vector{<:NamedTuple}; kwargs...) - $f([PackageSpec(;pkg...) for pkg in pkgs]; kwargs...) + return $f([PackageSpec(; pkg...) for pkg in pkgs]; kwargs...) end end end @@ -442,12 +453,13 @@ end const SHIM_COMMENT = Sys.iswindows() ? "REM " : "#" const SHIM_VERSION = 1.0 const SHIM_HEADER = """$SHIM_COMMENT This file is generated by the Julia package manager. - $SHIM_COMMENT Shim version: $SHIM_VERSION""" +$SHIM_COMMENT Shim version: $SHIM_VERSION""" function generate_shims_for_apps(pkgname, apps, env, julia) for (_, app) in apps generate_shim(pkgname, app, env, julia) end + return end function generate_shim(pkgname, app::AppInfo, env, julia) @@ -456,7 +468,7 @@ function generate_shim(pkgname, app::AppInfo, env, julia) validate_submodule_name(app.submodule) module_spec = app.submodule === nothing ? pkgname : "$(pkgname).$(app.submodule)" - + filename = app.name * (Sys.iswindows() ? ".bat" : "") julia_bin_filename = joinpath(julia_bin_path(), filename) mkpath(dirname(julia_bin_filename)) @@ -470,7 +482,7 @@ function generate_shim(pkgname, app::AppInfo, env, julia) shell_shim(julia_escaped, module_spec_escaped, env) end overwrite_file_if_different(julia_bin_filename, content) - if Sys.isunix() + return if Sys.isunix() chmod(julia_bin_filename, 0o755) end end @@ -478,34 +490,34 @@ end function shell_shim(julia_escaped::String, module_spec_escaped::String, env) return """ - #!/bin/sh + #!/bin/sh - $SHIM_HEADER + $SHIM_HEADER - export JULIA_LOAD_PATH=$(repr(env)) - export JULIA_DEPOT_PATH=$(repr(join(DEPOT_PATH, ':'))) - exec $julia_escaped \\ - --startup-file=no \\ - -m $module_spec_escaped \\ - "\$@" - """ + export JULIA_LOAD_PATH=$(repr(env)) + export JULIA_DEPOT_PATH=$(repr(join(DEPOT_PATH, ':'))) + exec $julia_escaped \\ + --startup-file=no \\ + -m $module_spec_escaped \\ + "\$@" + """ end function windows_shim(julia_escaped::String, module_spec_escaped::String, env) return """ - @echo off + @echo off - $SHIM_HEADER + $SHIM_HEADER - setlocal - set JULIA_LOAD_PATH=$env - set JULIA_DEPOT_PATH=$(join(DEPOT_PATH, ';')) + setlocal + set JULIA_LOAD_PATH=$env + set JULIA_DEPOT_PATH=$(join(DEPOT_PATH, ';')) - $julia_escaped ^ - --startup-file=no ^ - -m $module_spec_escaped ^ - %* - """ + $julia_escaped ^ + --startup-file=no ^ + -m $module_spec_escaped ^ + %* + """ end end diff --git a/src/Artifacts.jl b/src/Artifacts.jl index 0f77f4cc12..d6f2c948f4 100644 --- a/src/Artifacts.jl +++ b/src/Artifacts.jl @@ -1,663 +1,684 @@ module PkgArtifacts -using Artifacts, Base.BinaryPlatforms, SHA -using ..MiniProgressBars, ..PlatformEngines -using Tar: can_symlink -using FileWatching: FileWatching - -import ..set_readonly, ..GitTools, ..TOML, ..pkg_server, ..can_fancyprint, - ..stderr_f, ..printpkgstyle, ..mv_temp_dir_retries, ..atomic_toml_write - -import Base: get, SHA1 -import Artifacts: artifact_names, ARTIFACTS_DIR_OVERRIDE, ARTIFACT_OVERRIDES, artifact_paths, - artifacts_dirs, pack_platform!, unpack_platform, load_artifacts_toml, - query_override, with_artifacts_directory, load_overrides -import ..Types: write_env_usage, parse_toml - -const Artifacts = PkgArtifacts # This is to preserve compatability for folks who depend on the internals of this module -export Artifacts, create_artifact, artifact_exists, artifact_path, remove_artifact, verify_artifact, - artifact_meta, artifact_hash, bind_artifact!, unbind_artifact!, download_artifact, - find_artifacts_toml, ensure_artifact_installed, @artifact_str, archive_artifact, - select_downloadable_artifacts, ArtifactDownloadInfo - -""" - create_artifact(f::Function) - -Creates a new artifact by running `f(artifact_path)`, hashing the result, and moving it -to the artifact store (`~/.julia/artifacts` on a typical installation). Returns the -identifying tree hash of this artifact. -""" -function create_artifact(f::Function) - # Ensure the `artifacts` directory exists in our default depot - artifacts_dir = first(artifacts_dirs()) - mkpath(artifacts_dir) - - # Temporary directory where we'll do our creation business - temp_dir = mktempdir(artifacts_dir) - - try - # allow the user to do their work inside the temporary directory - f(temp_dir) - - # Calculate the tree hash for this temporary directory - artifact_hash = SHA1(GitTools.tree_hash(temp_dir)) - - # If we created a dupe, just let the temp directory get destroyed. It's got the - # same contents as whatever already exists after all, so it doesn't matter. Only - # move its contents if it actually contains new contents. Note that we explicitly - # set `honor_overrides=false` here, as we wouldn't want to drop things into the - # system directory by accidentally creating something with the same content-hash - # as something that was foolishly overridden. This should be virtually impossible - # unless the user has been very unwise, but let's be cautious. - new_path = artifact_path(artifact_hash; honor_overrides=false) - mv_temp_dir_retries(temp_dir, new_path) - - # Give the people what they want - return artifact_hash - finally - # Always attempt to cleanup - rm(temp_dir; recursive=true, force=true) - end -end - -""" - remove_artifact(hash::SHA1; honor_overrides::Bool=false) - -Removes the given artifact (identified by its SHA1 git tree hash) from disk. Note that -if an artifact is installed in multiple depots, it will be removed from all of them. If -an overridden artifact is requested for removal, it will be silently ignored; this method -will never attempt to remove an overridden artifact. - -In general, we recommend that you use `Pkg.gc()` to manage artifact installations and do -not use `remove_artifact()` directly, as it can be difficult to know if an artifact is -being used by another package. -""" -function remove_artifact(hash::SHA1) - if query_override(hash) !== nothing - # We never remove overridden artifacts. - return - end + using Artifacts, Base.BinaryPlatforms, SHA + using ..MiniProgressBars, ..PlatformEngines + using Tar: can_symlink + using FileWatching: FileWatching + + import ..set_readonly, ..GitTools, ..TOML, ..pkg_server, ..can_fancyprint, + ..stderr_f, ..printpkgstyle, ..mv_temp_dir_retries, ..atomic_toml_write + + import Base: get, SHA1 + import Artifacts: artifact_names, ARTIFACTS_DIR_OVERRIDE, ARTIFACT_OVERRIDES, artifact_paths, + artifacts_dirs, pack_platform!, unpack_platform, load_artifacts_toml, + query_override, with_artifacts_directory, load_overrides + import ..Types: write_env_usage, parse_toml + + const Artifacts = PkgArtifacts # This is to preserve compatability for folks who depend on the internals of this module + export Artifacts, create_artifact, artifact_exists, artifact_path, remove_artifact, verify_artifact, + artifact_meta, artifact_hash, bind_artifact!, unbind_artifact!, download_artifact, + find_artifacts_toml, ensure_artifact_installed, @artifact_str, archive_artifact, + select_downloadable_artifacts, ArtifactDownloadInfo - # Get all possible paths (rooted in all depots) - possible_paths = artifacts_dirs(bytes2hex(hash.bytes)) - for path in possible_paths - if isdir(path) - rm(path; recursive=true, force=true) + """ + create_artifact(f::Function) + + Creates a new artifact by running `f(artifact_path)`, hashing the result, and moving it + to the artifact store (`~/.julia/artifacts` on a typical installation). Returns the + identifying tree hash of this artifact. + """ + function create_artifact(f::Function) + # Ensure the `artifacts` directory exists in our default depot + artifacts_dir = first(artifacts_dirs()) + mkpath(artifacts_dir) + + # Temporary directory where we'll do our creation business + temp_dir = mktempdir(artifacts_dir) + + try + # allow the user to do their work inside the temporary directory + f(temp_dir) + + # Calculate the tree hash for this temporary directory + artifact_hash = SHA1(GitTools.tree_hash(temp_dir)) + + # If we created a dupe, just let the temp directory get destroyed. It's got the + # same contents as whatever already exists after all, so it doesn't matter. Only + # move its contents if it actually contains new contents. Note that we explicitly + # set `honor_overrides=false` here, as we wouldn't want to drop things into the + # system directory by accidentally creating something with the same content-hash + # as something that was foolishly overridden. This should be virtually impossible + # unless the user has been very unwise, but let's be cautious. + new_path = artifact_path(artifact_hash; honor_overrides = false) + mv_temp_dir_retries(temp_dir, new_path) + + # Give the people what they want + return artifact_hash + finally + # Always attempt to cleanup + rm(temp_dir; recursive = true, force = true) end end -end - -""" - verify_artifact(hash::SHA1; honor_overrides::Bool=false) - -Verifies that the given artifact (identified by its SHA1 git tree hash) is installed on- -disk, and retains its integrity. If the given artifact is overridden, skips the -verification unless `honor_overrides` is set to `true`. -""" -function verify_artifact(hash::SHA1; honor_overrides::Bool=false) - # Silently skip overridden artifacts unless we really ask for it - if !honor_overrides + + """ + remove_artifact(hash::SHA1; honor_overrides::Bool=false) + + Removes the given artifact (identified by its SHA1 git tree hash) from disk. Note that + if an artifact is installed in multiple depots, it will be removed from all of them. If + an overridden artifact is requested for removal, it will be silently ignored; this method + will never attempt to remove an overridden artifact. + + In general, we recommend that you use `Pkg.gc()` to manage artifact installations and do + not use `remove_artifact()` directly, as it can be difficult to know if an artifact is + being used by another package. + """ + function remove_artifact(hash::SHA1) if query_override(hash) !== nothing - return true + # We never remove overridden artifacts. + return end - end - # If it doesn't even exist, then skip out - if !artifact_exists(hash) - return false + # Get all possible paths (rooted in all depots) + possible_paths = artifacts_dirs(bytes2hex(hash.bytes)) + for path in possible_paths + if isdir(path) + rm(path; recursive = true, force = true) + end + end + return end - # Otherwise actually run the verification - return all(hash.bytes .== GitTools.tree_hash(artifact_path(hash))) -end + """ + verify_artifact(hash::SHA1; honor_overrides::Bool=false) -""" - archive_artifact(hash::SHA1, tarball_path::String; honor_overrides::Bool=false) + Verifies that the given artifact (identified by its SHA1 git tree hash) is installed on- + disk, and retains its integrity. If the given artifact is overridden, skips the + verification unless `honor_overrides` is set to `true`. + """ + function verify_artifact(hash::SHA1; honor_overrides::Bool = false) + # Silently skip overridden artifacts unless we really ask for it + if !honor_overrides + if query_override(hash) !== nothing + return true + end + end -Archive an artifact into a tarball stored at `tarball_path`, returns the SHA256 of the -resultant tarball as a hexadecimal string. Throws an error if the artifact does not -exist. If the artifact is overridden, throws an error unless `honor_overrides` is set. -""" -function archive_artifact(hash::SHA1, tarball_path::String; honor_overrides::Bool=false) - if !honor_overrides - if query_override(hash) !== nothing - error("Will not archive an overridden artifact unless `honor_overrides` is set!") + # If it doesn't even exist, then skip out + if !artifact_exists(hash) + return false end - end - if !artifact_exists(hash) - error("Unable to archive artifact $(bytes2hex(hash.bytes)): does not exist!") + # Otherwise actually run the verification + return all(hash.bytes .== GitTools.tree_hash(artifact_path(hash))) end - # Package it up - package(artifact_path(hash), tarball_path) + """ + archive_artifact(hash::SHA1, tarball_path::String; honor_overrides::Bool=false) + + Archive an artifact into a tarball stored at `tarball_path`, returns the SHA256 of the + resultant tarball as a hexadecimal string. Throws an error if the artifact does not + exist. If the artifact is overridden, throws an error unless `honor_overrides` is set. + """ + function archive_artifact(hash::SHA1, tarball_path::String; honor_overrides::Bool = false) + if !honor_overrides + if query_override(hash) !== nothing + error("Will not archive an overridden artifact unless `honor_overrides` is set!") + end + end - # Calculate its sha256 and return that - return open(tarball_path, "r") do io - return bytes2hex(sha256(io)) + if !artifact_exists(hash) + error("Unable to archive artifact $(bytes2hex(hash.bytes)): does not exist!") + end + + # Package it up + package(artifact_path(hash), tarball_path) + + # Calculate its sha256 and return that + return open(tarball_path, "r") do io + return bytes2hex(sha256(io)) + end end -end -""" - ArtifactDownloadInfo + """ + ArtifactDownloadInfo -Auxilliary information about an artifact to be used with `bind_artifact!()` to give -a download location for that artifact, as well as the hash and size of that artifact. -""" -struct ArtifactDownloadInfo - # URL the artifact is available at as a gzip-compressed tarball - url::String + Auxilliary information about an artifact to be used with `bind_artifact!()` to give + a download location for that artifact, as well as the hash and size of that artifact. + """ + struct ArtifactDownloadInfo + # URL the artifact is available at as a gzip-compressed tarball + url::String - # SHA256 hash of the tarball - hash::Vector{UInt8} + # SHA256 hash of the tarball + hash::Vector{UInt8} - # Size in bytes of the tarball. `size <= 0` means unknown. - size::Int64 + # Size in bytes of the tarball. `size <= 0` means unknown. + size::Int64 - function ArtifactDownloadInfo(url, hash::AbstractVector, size = 0) - valid_hash_len = SHA.digestlen(SHA256_CTX) - hash_len = length(hash) - if hash_len != valid_hash_len - throw(ArgumentError("Invalid hash length '$(hash_len)', must be $(valid_hash_len)")) + function ArtifactDownloadInfo(url, hash::AbstractVector, size = 0) + valid_hash_len = SHA.digestlen(SHA256_CTX) + hash_len = length(hash) + if hash_len != valid_hash_len + throw(ArgumentError("Invalid hash length '$(hash_len)', must be $(valid_hash_len)")) + end + return new( + String(url), + Vector{UInt8}(hash), + Int64(size), + ) end - return new( - String(url), - Vector{UInt8}(hash), - Int64(size), - ) end -end -# Convenience constructor for string hashes -ArtifactDownloadInfo(url, hash::AbstractString, args...) = ArtifactDownloadInfo(url, hex2bytes(hash), args...) + # Convenience constructor for string hashes + ArtifactDownloadInfo(url, hash::AbstractString, args...) = ArtifactDownloadInfo(url, hex2bytes(hash), args...) -# Convenience constructor for legacy Tuple representation -ArtifactDownloadInfo(args::Tuple) = ArtifactDownloadInfo(args...) + # Convenience constructor for legacy Tuple representation + ArtifactDownloadInfo(args::Tuple) = ArtifactDownloadInfo(args...) -ArtifactDownloadInfo(adi::ArtifactDownloadInfo) = adi + ArtifactDownloadInfo(adi::ArtifactDownloadInfo) = adi -# Make the dict that will be embedded in the TOML -function make_dict(adi::ArtifactDownloadInfo) - ret = Dict{String,Any}( - "url" => adi.url, - "sha256" => bytes2hex(adi.hash), - ) - if adi.size > 0 - ret["size"] = adi.size + # Make the dict that will be embedded in the TOML + function make_dict(adi::ArtifactDownloadInfo) + ret = Dict{String, Any}( + "url" => adi.url, + "sha256" => bytes2hex(adi.hash), + ) + if adi.size > 0 + ret["size"] = adi.size + end + return ret end - return ret -end - -""" - bind_artifact!(artifacts_toml::String, name::String, hash::SHA1; - platform::Union{AbstractPlatform,Nothing} = nothing, - download_info::Union{Vector{Tuple},Nothing} = nothing, - lazy::Bool = false, - force::Bool = false) - -Writes a mapping of `name` -> `hash` within the given `(Julia)Artifacts.toml` file. If -`platform` is not `nothing`, this artifact is marked as platform-specific, and will be -a multi-mapping. It is valid to bind multiple artifacts with the same name, but -different `platform`s and `hash`'es within the same `artifacts_toml`. If `force` is set -to `true`, this will overwrite a pre-existant mapping, otherwise an error is raised. - -`download_info` is an optional vector that contains tuples of URLs and a hash. These -URLs will be listed as possible locations where this artifact can be obtained. If `lazy` -is set to `true`, even if download information is available, this artifact will not be -downloaded until it is accessed via the `artifact"name"` syntax, or -`ensure_artifact_installed()` is called upon it. -""" -function bind_artifact!(artifacts_toml::String, name::String, hash::SHA1; - platform::Union{AbstractPlatform,Nothing} = nothing, - download_info::Union{Vector{<:Tuple},Vector{<:ArtifactDownloadInfo},Nothing} = nothing, - lazy::Bool = false, - force::Bool = false) - # First, check to see if this artifact is already bound: - if isfile(artifacts_toml) - artifact_dict = parse_toml(artifacts_toml) - if !force && haskey(artifact_dict, name) - meta = artifact_dict[name] - if !isa(meta, Vector) - error("Mapping for '$name' within $(artifacts_toml) already exists!") - elseif any(p -> platforms_match(platform, p), unpack_platform(x, name, artifacts_toml) for x in meta) - error("Mapping for '$name'/$(triplet(platform)) within $(artifacts_toml) already exists!") + """ + bind_artifact!(artifacts_toml::String, name::String, hash::SHA1; + platform::Union{AbstractPlatform,Nothing} = nothing, + download_info::Union{Vector{Tuple},Nothing} = nothing, + lazy::Bool = false, + force::Bool = false) + + Writes a mapping of `name` -> `hash` within the given `(Julia)Artifacts.toml` file. If + `platform` is not `nothing`, this artifact is marked as platform-specific, and will be + a multi-mapping. It is valid to bind multiple artifacts with the same name, but + different `platform`s and `hash`'es within the same `artifacts_toml`. If `force` is set + to `true`, this will overwrite a pre-existant mapping, otherwise an error is raised. + + `download_info` is an optional vector that contains tuples of URLs and a hash. These + URLs will be listed as possible locations where this artifact can be obtained. If `lazy` + is set to `true`, even if download information is available, this artifact will not be + downloaded until it is accessed via the `artifact"name"` syntax, or + `ensure_artifact_installed()` is called upon it. + """ + function bind_artifact!( + artifacts_toml::String, name::String, hash::SHA1; + platform::Union{AbstractPlatform, Nothing} = nothing, + download_info::Union{Vector{<:Tuple}, Vector{<:ArtifactDownloadInfo}, Nothing} = nothing, + lazy::Bool = false, + force::Bool = false + ) + # First, check to see if this artifact is already bound: + if isfile(artifacts_toml) + artifact_dict = parse_toml(artifacts_toml) + + if !force && haskey(artifact_dict, name) + meta = artifact_dict[name] + if !isa(meta, Vector) + error("Mapping for '$name' within $(artifacts_toml) already exists!") + elseif any(p -> platforms_match(platform, p), unpack_platform(x, name, artifacts_toml) for x in meta) + error("Mapping for '$name'/$(triplet(platform)) within $(artifacts_toml) already exists!") + end end + else + artifact_dict = Dict{String, Any}() end - else - artifact_dict = Dict{String, Any}() - end - # Otherwise, the new piece of data we're going to write out is this dict: - meta = Dict{String,Any}( - "git-tree-sha1" => bytes2hex(hash.bytes), - ) + # Otherwise, the new piece of data we're going to write out is this dict: + meta = Dict{String, Any}( + "git-tree-sha1" => bytes2hex(hash.bytes), + ) - # If we're set to be lazy, then lazy we shall be - if lazy - meta["lazy"] = true - end + # If we're set to be lazy, then lazy we shall be + if lazy + meta["lazy"] = true + end + + # Integrate download info, if it is given. Note that there can be multiple + # download locations, each with its own tarball with its own hash, but which + # expands to the same content/treehash. + if download_info !== nothing + meta["download"] = make_dict.(ArtifactDownloadInfo.(download_info)) + end + + if platform === nothing + artifact_dict[name] = meta + else + # Add platform-specific keys to our `meta` dict + pack_platform!(meta, platform) + + # Insert this entry into the list of artifacts + if !haskey(artifact_dict, name) + artifact_dict[name] = [meta] + else + # Delete any entries that contain identical platforms + artifact_dict[name] = filter( + x -> unpack_platform(x, name, artifacts_toml) != platform, + artifact_dict[name] + ) + push!(artifact_dict[name], meta) + end + end - # Integrate download info, if it is given. Note that there can be multiple - # download locations, each with its own tarball with its own hash, but which - # expands to the same content/treehash. - if download_info !== nothing - meta["download"] = make_dict.(ArtifactDownloadInfo.(download_info)) + # Spit it out onto disk + let artifact_dict = artifact_dict + parent_dir = dirname(artifacts_toml) + atomic_toml_write(artifacts_toml, artifact_dict, sorted = true) + end + + # Mark that we have used this Artifact.toml + write_env_usage(artifacts_toml, "artifact_usage.toml") + return end - if platform === nothing - artifact_dict[name] = meta - else - # Add platform-specific keys to our `meta` dict - pack_platform!(meta, platform) - # Insert this entry into the list of artifacts + """ + unbind_artifact!(artifacts_toml::String, name::String; platform = nothing) + + Unbind the given `name` from an `(Julia)Artifacts.toml` file. + Silently fails if no such binding exists within the file. + """ + function unbind_artifact!( + artifacts_toml::String, name::String; + platform::Union{AbstractPlatform, Nothing} = nothing + ) + artifact_dict = parse_toml(artifacts_toml) if !haskey(artifact_dict, name) - artifact_dict[name] = [meta] + return + end + + if platform === nothing + delete!(artifact_dict, name) else - # Delete any entries that contain identical platforms artifact_dict[name] = filter( x -> unpack_platform(x, name, artifacts_toml) != platform, artifact_dict[name] ) - push!(artifact_dict[name], meta) end - end - # Spit it out onto disk - let artifact_dict = artifact_dict - parent_dir = dirname(artifacts_toml) - atomic_toml_write(artifacts_toml, artifact_dict, sorted=true) + atomic_toml_write(artifacts_toml, artifact_dict, sorted = true) + return end - # Mark that we have used this Artifact.toml - write_env_usage(artifacts_toml, "artifact_usage.toml") - return -end - + """ + download_artifact(tree_hash::SHA1, tarball_url::String, tarball_hash::String; + verbose::Bool = false, io::IO=stderr) -""" - unbind_artifact!(artifacts_toml::String, name::String; platform = nothing) + Download/install an artifact into the artifact store. Returns `true` on success, + returns an error object on failure. -Unbind the given `name` from an `(Julia)Artifacts.toml` file. -Silently fails if no such binding exists within the file. -""" -function unbind_artifact!(artifacts_toml::String, name::String; - platform::Union{AbstractPlatform,Nothing} = nothing) - artifact_dict = parse_toml(artifacts_toml) - if !haskey(artifact_dict, name) - return - end - - if platform === nothing - delete!(artifact_dict, name) - else - artifact_dict[name] = filter( - x -> unpack_platform(x, name, artifacts_toml) != platform, - artifact_dict[name] + !!! compat "Julia 1.8" + As of Julia 1.8 this function returns the error object rather than `false` when + failure occurs + """ + function download_artifact( + tree_hash::SHA1, + tarball_url::String, + tarball_hash::Union{String, Nothing} = nothing; + verbose::Bool = false, + quiet_download::Bool = false, + io::IO = stderr_f(), + progress::Union{Function, Nothing} = nothing, ) - end - - atomic_toml_write(artifacts_toml, artifact_dict, sorted=true) - return -end - -""" - download_artifact(tree_hash::SHA1, tarball_url::String, tarball_hash::String; - verbose::Bool = false, io::IO=stderr) - -Download/install an artifact into the artifact store. Returns `true` on success, -returns an error object on failure. - -!!! compat "Julia 1.8" - As of Julia 1.8 this function returns the error object rather than `false` when - failure occurs -""" -function download_artifact( - tree_hash::SHA1, - tarball_url::String, - tarball_hash::Union{String, Nothing} = nothing; - verbose::Bool = false, - quiet_download::Bool = false, - io::IO=stderr_f(), - progress::Union{Function, Nothing} = nothing, -) - _artifact_paths = artifact_paths(tree_hash) - pidfile = _artifact_paths[1] * ".pid" - mkpath(dirname(pidfile)) - t_wait_msg = Timer(2) do t - if progress === nothing - @info "downloading $tarball_url ($hex) in another process" - else - progress(0, 0; status="downloading in another process") - end - end - ret = FileWatching.mkpidlock(pidfile, stale_age = 20) do - close(t_wait_msg) - if artifact_exists(tree_hash) - return true + _artifact_paths = artifact_paths(tree_hash) + pidfile = _artifact_paths[1] * ".pid" + mkpath(dirname(pidfile)) + t_wait_msg = Timer(2) do t + if progress === nothing + @info "downloading $tarball_url ($hex) in another process" + else + progress(0, 0; status = "downloading in another process") + end end + ret = FileWatching.mkpidlock(pidfile, stale_age = 20) do + close(t_wait_msg) + if artifact_exists(tree_hash) + return true + end - # Ensure the `artifacts` directory exists in our default depot - artifacts_dir = first(artifacts_dirs()) - mkpath(artifacts_dir) - # expected artifact path - dst = joinpath(artifacts_dir, bytes2hex(tree_hash.bytes)) + # Ensure the `artifacts` directory exists in our default depot + artifacts_dir = first(artifacts_dirs()) + mkpath(artifacts_dir) + # expected artifact path + dst = joinpath(artifacts_dir, bytes2hex(tree_hash.bytes)) - # We download by using a temporary directory. We do this because the download may - # be corrupted or even malicious; we don't want to clobber someone else's artifact - # by trusting the tree hash that has been given to us; we will instead download it - # to a temporary directory, calculate the true tree hash, then move it to the proper - # location only after knowing what it is, and if something goes wrong in the process, - # everything should be cleaned up. + # We download by using a temporary directory. We do this because the download may + # be corrupted or even malicious; we don't want to clobber someone else's artifact + # by trusting the tree hash that has been given to us; we will instead download it + # to a temporary directory, calculate the true tree hash, then move it to the proper + # location only after knowing what it is, and if something goes wrong in the process, + # everything should be cleaned up. - # Temporary directory where we'll do our creation business - temp_dir = mktempdir(artifacts_dir) + # Temporary directory where we'll do our creation business + temp_dir = mktempdir(artifacts_dir) - try - download_verify_unpack(tarball_url, tarball_hash, temp_dir; - ignore_existence=true, verbose, quiet_download, io, progress) - isnothing(progress) || progress(10000, 10000; status="verifying") - calc_hash = SHA1(GitTools.tree_hash(temp_dir)) - - # Did we get what we expected? If not, freak out. - if calc_hash.bytes != tree_hash.bytes - msg = """ - Tree Hash Mismatch! - Expected git-tree-sha1: $(bytes2hex(tree_hash.bytes)) - Calculated git-tree-sha1: $(bytes2hex(calc_hash.bytes)) - """ - # Since tree hash calculation is rather fragile and file system dependent, - # we allow setting JULIA_PKG_IGNORE_HASHES=1 to ignore the error and move - # the artifact to the expected location and return true - ignore_hash_env_set = get(ENV, "JULIA_PKG_IGNORE_HASHES", "") != "" - if ignore_hash_env_set - ignore_hash = Base.get_bool_env("JULIA_PKG_IGNORE_HASHES", false) - ignore_hash === nothing && @error( - "Invalid ENV[\"JULIA_PKG_IGNORE_HASHES\"] value", - ENV["JULIA_PKG_IGNORE_HASHES"], - ) - ignore_hash = something(ignore_hash, false) - else - # default: false except Windows users who can't symlink - ignore_hash = Sys.iswindows() && - !mktempdir(can_symlink, artifacts_dir) - end - if ignore_hash - desc = ignore_hash_env_set ? - "Environment variable \$JULIA_PKG_IGNORE_HASHES is true" : - "System is Windows and user cannot create symlinks" - msg *= "\n$desc: \ + try + download_verify_unpack( + tarball_url, tarball_hash, temp_dir; + ignore_existence = true, verbose, quiet_download, io, progress + ) + isnothing(progress) || progress(10000, 10000; status = "verifying") + calc_hash = SHA1(GitTools.tree_hash(temp_dir)) + + # Did we get what we expected? If not, freak out. + if calc_hash.bytes != tree_hash.bytes + msg = """ + Tree Hash Mismatch! + Expected git-tree-sha1: $(bytes2hex(tree_hash.bytes)) + Calculated git-tree-sha1: $(bytes2hex(calc_hash.bytes)) + """ + # Since tree hash calculation is rather fragile and file system dependent, + # we allow setting JULIA_PKG_IGNORE_HASHES=1 to ignore the error and move + # the artifact to the expected location and return true + ignore_hash_env_set = get(ENV, "JULIA_PKG_IGNORE_HASHES", "") != "" + if ignore_hash_env_set + ignore_hash = Base.get_bool_env("JULIA_PKG_IGNORE_HASHES", false) + ignore_hash === nothing && @error( + "Invalid ENV[\"JULIA_PKG_IGNORE_HASHES\"] value", + ENV["JULIA_PKG_IGNORE_HASHES"], + ) + ignore_hash = something(ignore_hash, false) + else + # default: false except Windows users who can't symlink + ignore_hash = Sys.iswindows() && + !mktempdir(can_symlink, artifacts_dir) + end + if ignore_hash + desc = ignore_hash_env_set ? + "Environment variable \$JULIA_PKG_IGNORE_HASHES is true" : + "System is Windows and user cannot create symlinks" + msg *= "\n$desc: \ ignoring hash mismatch and moving \ artifact to the expected location" - @error(msg) - else - error(msg) + @error(msg) + else + error(msg) + end + end + # Move it to the location we expected + isnothing(progress) || progress(10000, 10000; status = "moving to artifact store") + mv_temp_dir_retries(temp_dir, dst) + catch err + @debug "download_artifact error" tree_hash tarball_url tarball_hash err + if isa(err, InterruptException) + rethrow(err) + end + # If something went wrong during download, return the error + return err + finally + # Always attempt to cleanup + try + rm(temp_dir; recursive = true, force = true) + catch e + e isa InterruptException && rethrow() + @warn("Failed to clean up temporary directory $(repr(temp_dir))", exception = e) end end - # Move it to the location we expected - isnothing(progress) || progress(10000, 10000; status="moving to artifact store") - mv_temp_dir_retries(temp_dir, dst) - catch err - @debug "download_artifact error" tree_hash tarball_url tarball_hash err - if isa(err, InterruptException) - rethrow(err) - end - # If something went wrong during download, return the error - return err - finally - # Always attempt to cleanup - try - rm(temp_dir; recursive=true, force=true) - catch e - e isa InterruptException && rethrow() - @warn("Failed to clean up temporary directory $(repr(temp_dir))", exception=e) - end + return true end - return true + + return ret end - return ret -end - -""" - ensure_artifact_installed(name::String, artifacts_toml::String; - platform::AbstractPlatform = HostPlatform(), - pkg_uuid::Union{Base.UUID,Nothing}=nothing, - verbose::Bool = false, - quiet_download::Bool = false, - io::IO=stderr) - -Ensures an artifact is installed, downloading it via the download information stored in -`artifacts_toml` if necessary. Throws an error if unable to install. -""" -function ensure_artifact_installed(name::String, artifacts_toml::String; - platform::AbstractPlatform = HostPlatform(), - pkg_uuid::Union{Base.UUID,Nothing}=nothing, - pkg_server_eligible::Bool=true, - verbose::Bool = false, - quiet_download::Bool = false, - progress::Union{Function,Nothing} = nothing, - io::IO=stderr_f()) - meta = artifact_meta(name, artifacts_toml; pkg_uuid=pkg_uuid, platform=platform) - if meta === nothing - error("Cannot locate artifact '$(name)' in '$(artifacts_toml)'") + """ + ensure_artifact_installed(name::String, artifacts_toml::String; + platform::AbstractPlatform = HostPlatform(), + pkg_uuid::Union{Base.UUID,Nothing}=nothing, + verbose::Bool = false, + quiet_download::Bool = false, + io::IO=stderr) + + Ensures an artifact is installed, downloading it via the download information stored in + `artifacts_toml` if necessary. Throws an error if unable to install. + """ + function ensure_artifact_installed( + name::String, artifacts_toml::String; + platform::AbstractPlatform = HostPlatform(), + pkg_uuid::Union{Base.UUID, Nothing} = nothing, + pkg_server_eligible::Bool = true, + verbose::Bool = false, + quiet_download::Bool = false, + progress::Union{Function, Nothing} = nothing, + io::IO = stderr_f() + ) + meta = artifact_meta(name, artifacts_toml; pkg_uuid = pkg_uuid, platform = platform) + if meta === nothing + error("Cannot locate artifact '$(name)' in '$(artifacts_toml)'") + end + + return ensure_artifact_installed( + name, meta, artifacts_toml; + pkg_server_eligible, platform, verbose, quiet_download, progress, io + ) end - return ensure_artifact_installed(name, meta, artifacts_toml; - pkg_server_eligible, platform, verbose, quiet_download, progress, io) -end - -function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::String; - pkg_server_eligible::Bool=true, - platform::AbstractPlatform = HostPlatform(), - verbose::Bool = false, - quiet_download::Bool = false, - progress::Union{Function,Nothing} = nothing, - io::IO=stderr_f()) - hash = SHA1(meta["git-tree-sha1"]) - if !artifact_exists(hash) - if isnothing(progress) || verbose == true - return try_artifact_download_sources(name, hash, meta, artifacts_toml; pkg_server_eligible, platform, verbose, quiet_download, io) + function ensure_artifact_installed( + name::String, meta::Dict, artifacts_toml::String; + pkg_server_eligible::Bool = true, + platform::AbstractPlatform = HostPlatform(), + verbose::Bool = false, + quiet_download::Bool = false, + progress::Union{Function, Nothing} = nothing, + io::IO = stderr_f() + ) + hash = SHA1(meta["git-tree-sha1"]) + if !artifact_exists(hash) + if isnothing(progress) || verbose == true + return try_artifact_download_sources(name, hash, meta, artifacts_toml; pkg_server_eligible, platform, verbose, quiet_download, io) + else + # if a custom progress handler is given it is taken to mean the caller wants to handle the download scheduling + return () -> try_artifact_download_sources(name, hash, meta, artifacts_toml; pkg_server_eligible, platform, quiet_download = true, io, progress) + end else - # if a custom progress handler is given it is taken to mean the caller wants to handle the download scheduling - return () -> try_artifact_download_sources(name, hash, meta, artifacts_toml; pkg_server_eligible, platform, quiet_download=true, io, progress) + return artifact_path(hash) end - else - return artifact_path(hash) end -end -function try_artifact_download_sources( + function try_artifact_download_sources( name::String, hash::SHA1, meta::Dict, artifacts_toml::String; - pkg_server_eligible::Bool=true, - platform::AbstractPlatform=HostPlatform(), - verbose::Bool=false, - quiet_download::Bool=false, - io::IO=stderr_f(), - progress::Union{Function,Nothing}=nothing) - - errors = Any[] - # first try downloading from Pkg server if the Pkg server knows about this package - if pkg_server_eligible && (server = pkg_server()) !== nothing - url = "$server/artifact/$hash" - download_success = let url = url - @debug "Downloading artifact from Pkg server" name artifacts_toml platform url - with_show_download_info(io, name, quiet_download) do - download_artifact(hash, url; verbose, quiet_download, io, progress) + pkg_server_eligible::Bool = true, + platform::AbstractPlatform = HostPlatform(), + verbose::Bool = false, + quiet_download::Bool = false, + io::IO = stderr_f(), + progress::Union{Function, Nothing} = nothing + ) + + errors = Any[] + # first try downloading from Pkg server if the Pkg server knows about this package + if pkg_server_eligible && (server = pkg_server()) !== nothing + url = "$server/artifact/$hash" + download_success = let url = url + @debug "Downloading artifact from Pkg server" name artifacts_toml platform url + with_show_download_info(io, name, quiet_download) do + download_artifact(hash, url; verbose, quiet_download, io, progress) + end + end + # download_success is either `true` or an error object + if download_success === true + return artifact_path(hash) + else + @debug "Failed to download artifact from Pkg server" download_success + push!(errors, (url, download_success)) end end - # download_success is either `true` or an error object - if download_success === true - return artifact_path(hash) - else - @debug "Failed to download artifact from Pkg server" download_success - push!(errors, (url, download_success)) - end - end - # If this artifact does not exist on-disk already, ensure it has download - # information, then download it! - if !haskey(meta, "download") - error("Cannot automatically install '$(name)'; no download section in '$(artifacts_toml)'") - end + # If this artifact does not exist on-disk already, ensure it has download + # information, then download it! + if !haskey(meta, "download") + error("Cannot automatically install '$(name)'; no download section in '$(artifacts_toml)'") + end - # Attempt to download from all sources - for entry in meta["download"] - url = entry["url"] - tarball_hash = entry["sha256"] - download_success = let url = url - @debug "Downloading artifact" name artifacts_toml platform url - with_show_download_info(io, name, quiet_download) do - download_artifact(hash, url, tarball_hash; verbose, quiet_download, io, progress) + # Attempt to download from all sources + for entry in meta["download"] + url = entry["url"] + tarball_hash = entry["sha256"] + download_success = let url = url + @debug "Downloading artifact" name artifacts_toml platform url + with_show_download_info(io, name, quiet_download) do + download_artifact(hash, url, tarball_hash; verbose, quiet_download, io, progress) + end + end + # download_success is either `true` or an error object + if download_success === true + return artifact_path(hash) + else + @debug "Failed to download artifact" download_success + push!(errors, (url, download_success)) end end - # download_success is either `true` or an error object - if download_success === true - return artifact_path(hash) - else - @debug "Failed to download artifact" download_success - push!(errors, (url, download_success)) + errmsg = """ + Unable to automatically download/install artifact '$(name)' from sources listed in '$(artifacts_toml)'. + Sources attempted: + """ + for (url, err) in errors + errmsg *= "- $(url)\n" + errmsg *= " Error: $(sprint(showerror, err))\n" end + error(errmsg) end - errmsg = """ - Unable to automatically download/install artifact '$(name)' from sources listed in '$(artifacts_toml)'. - Sources attempted: - """ - for (url, err) in errors - errmsg *= "- $(url)\n" - errmsg *= " Error: $(sprint(showerror, err))\n" - end - error(errmsg) -end -function with_show_download_info(f, io, name, quiet_download) - fancyprint = can_fancyprint(io) - if !quiet_download - fancyprint && print_progress_bottom(io) - printpkgstyle(io, :Downloading, "artifact: $name") - end - success = false - try - result = f() - success = result === true - return result - finally + function with_show_download_info(f, io, name, quiet_download) + fancyprint = can_fancyprint(io) if !quiet_download - fancyprint && print(io, "\033[1A") # move cursor up one line - fancyprint && print(io, "\033[2K") # clear line - if success - fancyprint && printpkgstyle(io, :Downloaded, "artifact: $name") - else - printpkgstyle(io, :Failure, "artifact: $name", color = :red) + fancyprint && print_progress_bottom(io) + printpkgstyle(io, :Downloading, "artifact: $name") + end + success = false + try + result = f() + success = result === true + return result + finally + if !quiet_download + fancyprint && print(io, "\033[1A") # move cursor up one line + fancyprint && print(io, "\033[2K") # clear line + if success + fancyprint && printpkgstyle(io, :Downloaded, "artifact: $name") + else + printpkgstyle(io, :Failure, "artifact: $name", color = :red) + end end end end -end -""" - ensure_all_artifacts_installed(artifacts_toml::String; - platform = HostPlatform(), - pkg_uuid = nothing, - include_lazy = false, - verbose = false, - quiet_download = false, - io::IO=stderr) + """ + ensure_all_artifacts_installed(artifacts_toml::String; + platform = HostPlatform(), + pkg_uuid = nothing, + include_lazy = false, + verbose = false, + quiet_download = false, + io::IO=stderr) -Installs all non-lazy artifacts from a given `(Julia)Artifacts.toml` file. `package_uuid` must -be provided to properly support overrides from `Overrides.toml` entries in depots. + Installs all non-lazy artifacts from a given `(Julia)Artifacts.toml` file. `package_uuid` must + be provided to properly support overrides from `Overrides.toml` entries in depots. -If `include_lazy` is set to `true`, then lazy packages will be installed as well. + If `include_lazy` is set to `true`, then lazy packages will be installed as well. -This function is deprecated and should be replaced with the following snippet: + This function is deprecated and should be replaced with the following snippet: - artifacts = select_downloadable_artifacts(artifacts_toml; platform, include_lazy) - for name in keys(artifacts) - ensure_artifact_installed(name, artifacts[name], artifacts_toml; platform=platform) - end + artifacts = select_downloadable_artifacts(artifacts_toml; platform, include_lazy) + for name in keys(artifacts) + ensure_artifact_installed(name, artifacts[name], artifacts_toml; platform=platform) + end -!!! warning - This function is deprecated in Julia 1.6 and will be removed in a future version. - Use `select_downloadable_artifacts()` and `ensure_artifact_installed()` instead. -""" -function ensure_all_artifacts_installed(artifacts_toml::String; - platform::AbstractPlatform = HostPlatform(), - pkg_uuid::Union{Nothing,Base.UUID} = nothing, - include_lazy::Bool = false, - verbose::Bool = false, - quiet_download::Bool = false, - io::IO=stderr_f()) - # This function should not be called anymore; use `select_downloadable_artifacts()` directly. - Base.depwarn("`ensure_all_artifacts_installed()` is deprecated; iterate over `select_downloadable_artifacts()` output with `ensure_artifact_installed()`.", :ensure_all_artifacts_installed) - # Collect all artifacts we're supposed to install - artifacts = select_downloadable_artifacts(artifacts_toml; platform, include_lazy, pkg_uuid) - for name in keys(artifacts) - # Otherwise, let's try and install it! - ensure_artifact_installed(name, artifacts[name], artifacts_toml; platform=platform, - verbose=verbose, quiet_download=quiet_download, io=io) - end -end - -""" - extract_all_hashes(artifacts_toml::String; - platform = HostPlatform(), - pkg_uuid = nothing, - include_lazy = false) - -Extract all hashes from a given `(Julia)Artifacts.toml` file. `package_uuid` must -be provided to properly support overrides from `Overrides.toml` entries in depots. - -If `include_lazy` is set to `true`, then lazy packages will be installed as well. -""" -function extract_all_hashes(artifacts_toml::String; - platform::AbstractPlatform = HostPlatform(), - pkg_uuid::Union{Nothing,Base.UUID} = nothing, - include_lazy::Bool = false) - hashes = Base.SHA1[] - if !isfile(artifacts_toml) - return hashes + !!! warning + This function is deprecated in Julia 1.6 and will be removed in a future version. + Use `select_downloadable_artifacts()` and `ensure_artifact_installed()` instead. + """ + function ensure_all_artifacts_installed( + artifacts_toml::String; + platform::AbstractPlatform = HostPlatform(), + pkg_uuid::Union{Nothing, Base.UUID} = nothing, + include_lazy::Bool = false, + verbose::Bool = false, + quiet_download::Bool = false, + io::IO = stderr_f() + ) + # This function should not be called anymore; use `select_downloadable_artifacts()` directly. + Base.depwarn("`ensure_all_artifacts_installed()` is deprecated; iterate over `select_downloadable_artifacts()` output with `ensure_artifact_installed()`.", :ensure_all_artifacts_installed) + # Collect all artifacts we're supposed to install + artifacts = select_downloadable_artifacts(artifacts_toml; platform, include_lazy, pkg_uuid) + for name in keys(artifacts) + # Otherwise, let's try and install it! + ensure_artifact_installed( + name, artifacts[name], artifacts_toml; platform = platform, + verbose = verbose, quiet_download = quiet_download, io = io + ) + end + return end - artifact_dict = load_artifacts_toml(artifacts_toml; pkg_uuid=pkg_uuid) + """ + extract_all_hashes(artifacts_toml::String; + platform = HostPlatform(), + pkg_uuid = nothing, + include_lazy = false) + + Extract all hashes from a given `(Julia)Artifacts.toml` file. `package_uuid` must + be provided to properly support overrides from `Overrides.toml` entries in depots. + + If `include_lazy` is set to `true`, then lazy packages will be installed as well. + """ + function extract_all_hashes( + artifacts_toml::String; + platform::AbstractPlatform = HostPlatform(), + pkg_uuid::Union{Nothing, Base.UUID} = nothing, + include_lazy::Bool = false + ) + hashes = Base.SHA1[] + if !isfile(artifacts_toml) + return hashes + end + + artifact_dict = load_artifacts_toml(artifacts_toml; pkg_uuid = pkg_uuid) - for name in keys(artifact_dict) - # Get the metadata about this name for the requested platform - meta = artifact_meta(name, artifact_dict, artifacts_toml; platform=platform) + for name in keys(artifact_dict) + # Get the metadata about this name for the requested platform + meta = artifact_meta(name, artifact_dict, artifacts_toml; platform = platform) - # If there are no instances of this name for the desired platform, skip it - meta === nothing && continue + # If there are no instances of this name for the desired platform, skip it + meta === nothing && continue - # If it's a lazy one and we aren't including lazy ones, skip - if get(meta, "lazy", false) && !include_lazy - continue + # If it's a lazy one and we aren't including lazy ones, skip + if get(meta, "lazy", false) && !include_lazy + continue + end + + # Otherwise, add it to the list! + push!(hashes, Base.SHA1(meta["git-tree-sha1"])) end - # Otherwise, add it to the list! - push!(hashes, Base.SHA1(meta["git-tree-sha1"])) + return hashes end - return hashes -end - -# Support `AbstractString`s, but avoid compilers needing to track backedges for callers -# of these functions in case a user defines a new type that is `<: AbstractString` -archive_artifact(hash::SHA1, tarball_path::AbstractString; kwargs...) = - archive_artifact(hash, string(tarball_path)::String; kwargs...) -bind_artifact!(artifacts_toml::AbstractString, name::AbstractString, hash::SHA1; kwargs...) = - bind_artifact!(string(artifacts_toml)::String, string(name)::String, hash; kwargs...) -unbind_artifact!(artifacts_toml::AbstractString, name::AbstractString) = - unbind_artifact!(string(artifacts_toml)::String, string(name)::String) -download_artifact(tree_hash::SHA1, tarball_url::AbstractString, args...; kwargs...) = - download_artifact(tree_hash, string(tarball_url)::String, args...; kwargs...) -ensure_artifact_installed(name::AbstractString, artifacts_toml::AbstractString; kwargs...) = - ensure_artifact_installed(string(name)::String, string(artifacts_toml)::String; kwargs...) -ensure_artifact_installed(name::AbstractString, meta::Dict, artifacts_toml::AbstractString; kwargs...) = - ensure_artifact_installed(string(name)::String, meta, string(artifacts_toml)::String; kwargs...) -ensure_all_artifacts_installed(artifacts_toml::AbstractString; kwargs...) = - ensure_all_artifacts_installed(string(artifacts_toml)::String; kwargs...) -extract_all_hashes(artifacts_toml::AbstractString; kwargs...) = - extract_all_hashes(string(artifacts_toml)::String; kwargs...) + # Support `AbstractString`s, but avoid compilers needing to track backedges for callers + # of these functions in case a user defines a new type that is `<: AbstractString` + archive_artifact(hash::SHA1, tarball_path::AbstractString; kwargs...) = + archive_artifact(hash, string(tarball_path)::String; kwargs...) + bind_artifact!(artifacts_toml::AbstractString, name::AbstractString, hash::SHA1; kwargs...) = + bind_artifact!(string(artifacts_toml)::String, string(name)::String, hash; kwargs...) + unbind_artifact!(artifacts_toml::AbstractString, name::AbstractString) = + unbind_artifact!(string(artifacts_toml)::String, string(name)::String) + download_artifact(tree_hash::SHA1, tarball_url::AbstractString, args...; kwargs...) = + download_artifact(tree_hash, string(tarball_url)::String, args...; kwargs...) + ensure_artifact_installed(name::AbstractString, artifacts_toml::AbstractString; kwargs...) = + ensure_artifact_installed(string(name)::String, string(artifacts_toml)::String; kwargs...) + ensure_artifact_installed(name::AbstractString, meta::Dict, artifacts_toml::AbstractString; kwargs...) = + ensure_artifact_installed(string(name)::String, meta, string(artifacts_toml)::String; kwargs...) + ensure_all_artifacts_installed(artifacts_toml::AbstractString; kwargs...) = + ensure_all_artifacts_installed(string(artifacts_toml)::String; kwargs...) + extract_all_hashes(artifacts_toml::AbstractString; kwargs...) = + extract_all_hashes(string(artifacts_toml)::String; kwargs...) end # module PkgArtifacts diff --git a/src/BinaryPlatformsCompat.jl b/src/BinaryPlatformsCompat.jl index 05b1a6ba2e..93403e05bd 100644 --- a/src/BinaryPlatformsCompat.jl +++ b/src/BinaryPlatformsCompat.jl @@ -1,149 +1,154 @@ module BinaryPlatformsCompat -export platform_key_abi, platform_dlext, valid_dl_path, arch, libc, - libgfortran_version, libstdcxx_version, cxxstring_abi, parse_dl_name_version, - detect_libgfortran_version, detect_libstdcxx_version, detect_cxxstring_abi, - call_abi, wordsize, triplet, select_platform, platforms_match, - CompilerABI, Platform, UnknownPlatform, Linux, MacOS, Windows, FreeBSD - -using Base.BinaryPlatforms: parse_dl_name_version, - detect_libgfortran_version, detect_libstdcxx_version, detect_cxxstring_abi, - os, call_abi, select_platform, platforms_match, - AbstractPlatform, Platform, HostPlatform - -import Base.BinaryPlatforms: libgfortran_version, libstdcxx_version, platform_name, - wordsize, platform_dlext, tags, arch, libc, call_abi, - cxxstring_abi - -struct UnknownPlatform <: AbstractPlatform - UnknownPlatform(args...; kwargs...) = new() -end -tags(::UnknownPlatform) = Dict{String,String}("os"=>"unknown") - - -struct CompilerABI - libgfortran_version::Union{Nothing,VersionNumber} - libstdcxx_version::Union{Nothing,VersionNumber} - cxxstring_abi::Union{Nothing,Symbol} - - function CompilerABI(;libgfortran_version::Union{Nothing, VersionNumber} = nothing, - libstdcxx_version::Union{Nothing, VersionNumber} = nothing, - cxxstring_abi::Union{Nothing, Symbol} = nothing) - return new(libgfortran_version, libstdcxx_version, cxxstring_abi) + export platform_key_abi, platform_dlext, valid_dl_path, arch, libc, + libgfortran_version, libstdcxx_version, cxxstring_abi, parse_dl_name_version, + detect_libgfortran_version, detect_libstdcxx_version, detect_cxxstring_abi, + call_abi, wordsize, triplet, select_platform, platforms_match, + CompilerABI, Platform, UnknownPlatform, Linux, MacOS, Windows, FreeBSD + + using Base.BinaryPlatforms: parse_dl_name_version, + detect_libgfortran_version, detect_libstdcxx_version, detect_cxxstring_abi, + os, call_abi, select_platform, platforms_match, + AbstractPlatform, Platform, HostPlatform + + import Base.BinaryPlatforms: libgfortran_version, libstdcxx_version, platform_name, + wordsize, platform_dlext, tags, arch, libc, call_abi, + cxxstring_abi + + struct UnknownPlatform <: AbstractPlatform + UnknownPlatform(args...; kwargs...) = new() end -end - -# Easy replacement constructor -function CompilerABI(cabi::CompilerABI; libgfortran_version=nothing, - libstdcxx_version=nothing, - cxxstring_abi=nothing) - return CompilerABI(; - libgfortran_version=something(libgfortran_version, Some(cabi.libgfortran_version)), - libstdcxx_version=something(libstdcxx_version, Some(cabi.libstdcxx_version)), - cxxstring_abi=something(cxxstring_abi, Some(cabi.cxxstring_abi)), - ) -end - -libgfortran_version(cabi::CompilerABI) = cabi.libgfortran_version -libstdcxx_version(cabi::CompilerABI) = cabi.libstdcxx_version -cxxstring_abi(cabi::CompilerABI) = cabi.cxxstring_abi - -for T in (:Linux, :Windows, :MacOS, :FreeBSD) - @eval begin - struct $(T) <: AbstractPlatform - p::Platform - function $(T)(arch::Symbol; compiler_abi=nothing, kwargs...) - if compiler_abi !== nothing - kwargs = (; kwargs..., - :libgfortran_version => libgfortran_version(compiler_abi), - :libstdcxx_version => libstdcxx_version(compiler_abi), - :cxxstring_abi => cxxstring_abi(compiler_abi) - ) + tags(::UnknownPlatform) = Dict{String, String}("os" => "unknown") + + + struct CompilerABI + libgfortran_version::Union{Nothing, VersionNumber} + libstdcxx_version::Union{Nothing, VersionNumber} + cxxstring_abi::Union{Nothing, Symbol} + + function CompilerABI(; + libgfortran_version::Union{Nothing, VersionNumber} = nothing, + libstdcxx_version::Union{Nothing, VersionNumber} = nothing, + cxxstring_abi::Union{Nothing, Symbol} = nothing + ) + return new(libgfortran_version, libstdcxx_version, cxxstring_abi) + end + end + + # Easy replacement constructor + function CompilerABI( + cabi::CompilerABI; libgfortran_version = nothing, + libstdcxx_version = nothing, + cxxstring_abi = nothing + ) + return CompilerABI(; + libgfortran_version = something(libgfortran_version, Some(cabi.libgfortran_version)), + libstdcxx_version = something(libstdcxx_version, Some(cabi.libstdcxx_version)), + cxxstring_abi = something(cxxstring_abi, Some(cabi.cxxstring_abi)), + ) + end + + libgfortran_version(cabi::CompilerABI) = cabi.libgfortran_version + libstdcxx_version(cabi::CompilerABI) = cabi.libstdcxx_version + cxxstring_abi(cabi::CompilerABI) = cabi.cxxstring_abi + + for T in (:Linux, :Windows, :MacOS, :FreeBSD) + @eval begin + struct $(T) <: AbstractPlatform + p::Platform + function $(T)(arch::Symbol; compiler_abi = nothing, kwargs...) + if compiler_abi !== nothing + kwargs = (; + kwargs..., + :libgfortran_version => libgfortran_version(compiler_abi), + :libstdcxx_version => libstdcxx_version(compiler_abi), + :cxxstring_abi => cxxstring_abi(compiler_abi), + ) + end + return new(Platform(string(arch), $(string(T)); kwargs..., validate_strict = true)) end - return new(Platform(string(arch), $(string(T)); kwargs..., validate_strict=true)) end end end -end -const PlatformUnion = Union{Linux,MacOS,Windows,FreeBSD} + const PlatformUnion = Union{Linux, MacOS, Windows, FreeBSD} -# First, methods we need to coerce to Symbol for backwards-compatibility -for f in (:arch, :libc, :call_abi, :cxxstring_abi) - @eval begin - function $(f)(p::PlatformUnion) - str = $(f)(p.p) - if str === nothing - return nothing + # First, methods we need to coerce to Symbol for backwards-compatibility + for f in (:arch, :libc, :call_abi, :cxxstring_abi) + @eval begin + function $(f)(p::PlatformUnion) + str = $(f)(p.p) + if str === nothing + return nothing + end + return Symbol(str) end - return Symbol(str) end end -end -# Next, things we don't need to coerce -for f in (:libgfortran_version, :libstdcxx_version, :platform_name, :wordsize, :platform_dlext, :tags, :triplet) + # Next, things we don't need to coerce + for f in (:libgfortran_version, :libstdcxx_version, :platform_name, :wordsize, :platform_dlext, :tags, :triplet) + @eval begin + $(f)(p::PlatformUnion) = $(f)(p.p) + end + end + + # Finally, add equality testing between these wrapper types and other AbstractPlatforms @eval begin - $(f)(p::PlatformUnion) = $(f)(p.p) + Base.:(==)(a::PlatformUnion, b::AbstractPlatform) = b == a.p end -end - -# Finally, add equality testing between these wrapper types and other AbstractPlatforms -@eval begin - Base.:(==)(a::PlatformUnion, b::AbstractPlatform) = b == a.p -end - -# Add one-off functions -MacOS(; kwargs...) = MacOS(:x86_64; kwargs...) -FreeBSD(; kwargs...) = FreeBSD(:x86_64; kwargs...) - -function triplet(p::AbstractPlatform) - # We are going to sub off to `Base.BinaryPlatforms.triplet()` here, - # with the important exception that we override `os_version` to better - # mimic the old behavior of `triplet()` - if Sys.isfreebsd(p) - p = deepcopy(p) - p["os_version"] = "11.1.0" - elseif Sys.isapple(p) - p = deepcopy(p) - p["os_version"] = "14.0.0" + + # Add one-off functions + MacOS(; kwargs...) = MacOS(:x86_64; kwargs...) + FreeBSD(; kwargs...) = FreeBSD(:x86_64; kwargs...) + + function triplet(p::AbstractPlatform) + # We are going to sub off to `Base.BinaryPlatforms.triplet()` here, + # with the important exception that we override `os_version` to better + # mimic the old behavior of `triplet()` + if Sys.isfreebsd(p) + p = deepcopy(p) + p["os_version"] = "11.1.0" + elseif Sys.isapple(p) + p = deepcopy(p) + p["os_version"] = "14.0.0" + end + return Base.BinaryPlatforms.triplet(p) end - return Base.BinaryPlatforms.triplet(p) -end - -""" - platform_key_abi(machine::AbstractString) - -Returns the platform key for the current platform, or any other though the -the use of the `machine` parameter. - -This method is deprecated, import `Base.BinaryPlatforms` and use either `HostPlatform()` -to get the current host platform, or `parse(Base.BinaryPlatforms.Platform, triplet)` -to parse the triplet for some other platform instead. -""" -platform_key_abi() = HostPlatform() -platform_key_abi(triplet::AbstractString) = parse(Platform, triplet) - -""" - valid_dl_path(path::AbstractString, platform::Platform) - -Return `true` if the given `path` ends in a valid dynamic library filename. -E.g. returns `true` for a path like `"usr/lib/libfoo.so.3.5"`, but returns -`false` for a path like `"libbar.so.f.a"`. - -This method is deprecated and will be removed in Julia 2.0. -""" -function valid_dl_path(path::AbstractString, platform::AbstractPlatform) - try - parse_dl_name_version(path, string(os(platform))::String) - return true - catch e - if isa(e, ArgumentError) - return false + + """ + platform_key_abi(machine::AbstractString) + + Returns the platform key for the current platform, or any other though the + the use of the `machine` parameter. + + This method is deprecated, import `Base.BinaryPlatforms` and use either `HostPlatform()` + to get the current host platform, or `parse(Base.BinaryPlatforms.Platform, triplet)` + to parse the triplet for some other platform instead. + """ + platform_key_abi() = HostPlatform() + platform_key_abi(triplet::AbstractString) = parse(Platform, triplet) + + """ + valid_dl_path(path::AbstractString, platform::Platform) + + Return `true` if the given `path` ends in a valid dynamic library filename. + E.g. returns `true` for a path like `"usr/lib/libfoo.so.3.5"`, but returns + `false` for a path like `"libbar.so.f.a"`. + + This method is deprecated and will be removed in Julia 2.0. + """ + function valid_dl_path(path::AbstractString, platform::AbstractPlatform) + try + parse_dl_name_version(path, string(os(platform))::String) + return true + catch e + if isa(e, ArgumentError) + return false + end + rethrow(e) end - rethrow(e) end -end end # module BinaryPlatformsCompat diff --git a/src/GitTools.jl b/src/GitTools.jl index 6977a8907b..ddce9aac46 100644 --- a/src/GitTools.jl +++ b/src/GitTools.jl @@ -41,13 +41,13 @@ const GIT_USERS = Dict{String, Union{Nothing, String}}() @deprecate setprotocol!(proto::Union{Nothing, AbstractString}) setprotocol!(protocol = proto) false function setprotocol!(; - domain::AbstractString="github.com", - protocol::Union{Nothing, AbstractString}=nothing, - user::Union{Nothing, AbstractString}=(protocol == "ssh" ? "git" : nothing) -) + domain::AbstractString = "github.com", + protocol::Union{Nothing, AbstractString} = nothing, + user::Union{Nothing, AbstractString} = (protocol == "ssh" ? "git" : nothing) + ) domain = lowercase(domain) GIT_PROTOCOLS[domain] = protocol - GIT_USERS[domain] = user + return GIT_USERS[domain] = user end function normalize_url(url::AbstractString) @@ -61,7 +61,7 @@ function normalize_url(url::AbstractString) proto = get(GIT_PROTOCOLS, lowercase(host), nothing) - if proto === nothing + return if proto === nothing url else user = get(GIT_USERS, lowercase(host), nothing) @@ -80,16 +80,16 @@ function ensure_clone(io::IO, target_path, url; kwargs...) end function checkout_tree_to_path(repo::LibGit2.GitRepo, tree::LibGit2.GitObject, path::String) - GC.@preserve path begin + return GC.@preserve path begin opts = LibGit2.CheckoutOptions( checkout_strategy = LibGit2.Consts.CHECKOUT_FORCE, target_directory = Base.unsafe_convert(Cstring, path) ) - LibGit2.checkout_tree(repo, tree, options=opts) + LibGit2.checkout_tree(repo, tree, options = opts) end end -function clone(io::IO, url, source_path; header=nothing, credentials=nothing, isbare=false, kwargs...) +function clone(io::IO, url, source_path; header = nothing, credentials = nothing, isbare = false, kwargs...) url = String(url)::String source_path = String(source_path)::String @assert !isdir(source_path) || isempty(readdir(source_path)) @@ -101,13 +101,13 @@ function clone(io::IO, url, source_path; header=nothing, credentials=nothing, is if credentials === nothing credentials = LibGit2.CachedCredentials() end - try + return try if use_cli_git() args = ["--quiet", url, source_path] isbare && pushfirst!(args, "--bare") cmd = `git clone $args` try - run(pipeline(cmd; stdout=devnull)) + run(pipeline(cmd; stdout = devnull)) catch err Pkg.Types.pkgerror("The command $(cmd) failed, error: $err") end @@ -127,12 +127,12 @@ function clone(io::IO, url, source_path; header=nothing, credentials=nothing, is return LibGit2.clone(url, source_path; callbacks, credentials, isbare, kwargs...) end catch err - rm(source_path; force=true, recursive=true) + rm(source_path; force = true, recursive = true) err isa LibGit2.GitError || err isa InterruptException || rethrow() if err isa InterruptException Pkg.Types.pkgerror("git clone of `$url` interrupted") elseif (err.class == LibGit2.Error.Net && err.code == LibGit2.Error.EINVALIDSPEC) || - (err.class == LibGit2.Error.Repository && err.code == LibGit2.Error.ENOTFOUND) + (err.class == LibGit2.Error.Repository && err.code == LibGit2.Error.ENOTFOUND) Pkg.Types.pkgerror("git repository not found at `$(url)`") else Pkg.Types.pkgerror("failed to clone from $(url), error: $err") @@ -149,7 +149,7 @@ function geturl(repo) end end -function fetch(io::IO, repo::LibGit2.GitRepo, remoteurl=nothing; header=nothing, credentials=nothing, refspecs=[""], kwargs...) +function fetch(io::IO, repo::LibGit2.GitRepo, remoteurl = nothing; header = nothing, credentials = nothing, refspecs = [""], kwargs...) if remoteurl === nothing remoteurl = geturl(repo) end @@ -171,12 +171,12 @@ function fetch(io::IO, repo::LibGit2.GitRepo, remoteurl=nothing; header=nothing, if credentials === nothing credentials = LibGit2.CachedCredentials() end - try + return try if use_cli_git() - let remoteurl=remoteurl + let remoteurl = remoteurl cmd = `git -C $(LibGit2.path(repo)) fetch -q $remoteurl $(only(refspecs))` try - run(pipeline(cmd; stdout=devnull)) + run(pipeline(cmd; stdout = devnull)) catch err Pkg.Types.pkgerror("The command $(cmd) failed, error: $err") end @@ -199,8 +199,8 @@ end # This code gratefully adapted from https://github.com/simonbyrne/GitX.jl -@enum GitMode mode_dir=0o040000 mode_normal=0o100644 mode_executable=0o100755 mode_symlink=0o120000 mode_submodule=0o160000 -Base.string(mode::GitMode) = string(UInt32(mode); base=8) +@enum GitMode mode_dir = 0o040000 mode_normal = 0o100644 mode_executable = 0o100755 mode_symlink = 0o120000 mode_submodule = 0o160000 +Base.string(mode::GitMode) = string(UInt32(mode); base = 8) Base.print(io::IO, mode::GitMode) = print(io, string(mode)) function gitmode(path::AbstractString) @@ -230,7 +230,7 @@ end Calculate the git blob hash of a given path. """ -function blob_hash(::Type{HashType}, path::AbstractString) where HashType +function blob_hash(::Type{HashType}, path::AbstractString) where {HashType} ctx = HashType() if islink(path) datalen = length(readlink(path)) @@ -242,7 +242,7 @@ function blob_hash(::Type{HashType}, path::AbstractString) where HashType SHA.update!(ctx, Vector{UInt8}("blob $(datalen)\0")) # Next, read data in in chunks of 4KB - buff = Vector{UInt8}(undef, 4*1024) + buff = Vector{UInt8}(undef, 4 * 1024) try if islink(path) @@ -290,9 +290,9 @@ end Calculate the git tree hash of a given path. """ -function tree_hash(::Type{HashType}, root::AbstractString; debug_out::Union{IO,Nothing} = nothing, indent::Int=0) where HashType +function tree_hash(::Type{HashType}, root::AbstractString; debug_out::Union{IO, Nothing} = nothing, indent::Int = 0) where {HashType} entries = Tuple{String, Vector{UInt8}, GitMode}[] - for f in sort(readdir(root; join=true); by = f -> gitmode(f) == mode_dir ? f*"/" : f) + for f in sort(readdir(root; join = true); by = f -> gitmode(f) == mode_dir ? f * "/" : f) # Skip `.git` directories if basename(f) == ".git" continue @@ -309,7 +309,7 @@ function tree_hash(::Type{HashType}, root::AbstractString; debug_out::Union{IO,N if debug_out !== nothing child_stream = IOBuffer() end - hash = tree_hash(HashType, filepath; debug_out=child_stream, indent=indent+1) + hash = tree_hash(HashType, filepath; debug_out = child_stream, indent = indent + 1) if debug_out !== nothing indent_str = "| "^indent println(debug_out, "$(indent_str)+ [D] $(basename(filepath)) - $(bytes2hex(hash))") @@ -329,7 +329,7 @@ function tree_hash(::Type{HashType}, root::AbstractString; debug_out::Union{IO,N content_size = 0 for (n, h, m) in entries - content_size += ndigits(UInt32(m); base=8) + 1 + sizeof(n) + 1 + sizeof(h) + content_size += ndigits(UInt32(m); base = 8) + 1 + sizeof(n) + 1 + sizeof(h) end # Return the hash of these entries @@ -341,10 +341,11 @@ function tree_hash(::Type{HashType}, root::AbstractString; debug_out::Union{IO,N end return SHA.digest!(ctx) end -tree_hash(root::AbstractString; debug_out::Union{IO,Nothing} = nothing) = tree_hash(SHA.SHA1_CTX, root; debug_out) +tree_hash(root::AbstractString; debug_out::Union{IO, Nothing} = nothing) = tree_hash(SHA.SHA1_CTX, root; debug_out) function check_valid_HEAD(repo) - try LibGit2.head(repo) + return try + LibGit2.head(repo) catch err url = try geturl(repo) @@ -355,8 +356,9 @@ function check_valid_HEAD(repo) end end -function git_file_stream(repo::LibGit2.GitRepo, spec::String; fakeit::Bool=false)::IO - blob = try LibGit2.GitBlob(repo, spec) +function git_file_stream(repo::LibGit2.GitRepo, spec::String; fakeit::Bool = false)::IO + blob = try + LibGit2.GitBlob(repo, spec) catch err err isa LibGit2.GitError && err.code == LibGit2.Error.ENOTFOUND || rethrow() fakeit && return devnull diff --git a/src/HistoricalStdlibs.jl b/src/HistoricalStdlibs.jl index d5b4ad5049..6867d1e832 100644 --- a/src/HistoricalStdlibs.jl +++ b/src/HistoricalStdlibs.jl @@ -5,13 +5,13 @@ struct StdlibInfo uuid::UUID # This can be `nothing` if it's an unregistered stdlib - version::Union{Nothing,VersionNumber} + version::Union{Nothing, VersionNumber} deps::Vector{UUID} weakdeps::Vector{UUID} end -const DictStdLibs = Dict{UUID,StdlibInfo} +const DictStdLibs = Dict{UUID, StdlibInfo} # Julia standard libraries with duplicate entries removed so as to store only the # first release in a set of releases that all contain the same set of stdlibs. diff --git a/src/MiniProgressBars.jl b/src/MiniProgressBars.jl index c0a487d6b6..5682fc04a4 100644 --- a/src/MiniProgressBars.jl +++ b/src/MiniProgressBars.jl @@ -5,12 +5,12 @@ export MiniProgressBar, start_progress, end_progress, show_progress, print_progr using Printf # Until Base.format_bytes supports sigdigits -function pkg_format_bytes(bytes; binary=true, sigdigits::Integer=3) +function pkg_format_bytes(bytes; binary = true, sigdigits::Integer = 3) units = binary ? Base._mem_units : Base._cnt_units factor = binary ? 1024 : 1000 bytes, mb = Base.prettyprint_getunits(bytes, length(units), Int64(factor)) if mb == 1 - return string(Int(bytes), " ", Base._mem_units[mb], bytes==1 ? "" : "s") + return string(Int(bytes), " ", Base._mem_units[mb], bytes == 1 ? "" : "s") else return string(Base.Ryu.writefixed(Float64(bytes), sigdigits), binary ? " $(units[mb])" : "$(units[mb])B") end @@ -37,10 +37,10 @@ const PROGRESS_BAR_PERCENTAGE_GRANULARITY = Ref(0.1) function start_progress(io::IO, _::MiniProgressBar) ansi_disablecursor = "\e[?25l" - print(io, ansi_disablecursor) + return print(io, ansi_disablecursor) end -function show_progress(io::IO, p::MiniProgressBar; termwidth=nothing, carriagereturn=true) +function show_progress(io::IO, p::MiniProgressBar; termwidth = nothing, carriagereturn = true) if p.max == 0 perc = 0.0 prev_perc = 0.0 @@ -64,22 +64,22 @@ function show_progress(io::IO, p::MiniProgressBar; termwidth=nothing, carriagere progress_text = if p.mode == :percentage @sprintf "%2.1f %%" perc elseif p.mode == :int - string(p.current, "/", p.max) + string(p.current, "/", p.max) elseif p.mode == :data - lpad(string(pkg_format_bytes(p.current; sigdigits=1), "/", pkg_format_bytes(p.max; sigdigits=1)), 20) + lpad(string(pkg_format_bytes(p.current; sigdigits = 1), "/", pkg_format_bytes(p.max; sigdigits = 1)), 20) else error("Unknown mode $(p.mode)") end termwidth = @something termwidth displaysize(io)[2] - max_progress_width = max(0, min(termwidth - textwidth(p.header) - textwidth(progress_text) - 10 , p.width)) + max_progress_width = max(0, min(termwidth - textwidth(p.header) - textwidth(progress_text) - 10, p.width)) n_filled = floor(Int, max_progress_width * perc / 100) partial_filled = (max_progress_width * perc / 100) - n_filled n_left = max_progress_width - n_filled headers = split(p.header) - to_print = sprint(; context=io) do io + to_print = sprint(; context = io) do io print(io, " "^p.indent) if p.main - printstyled(io, headers[1], " "; color=:green, bold=true) + printstyled(io, headers[1], " "; color = :green, bold = true) length(headers) > 1 && printstyled(io, join(headers[2:end], ' '), " ") else print(io, p.header, " ") @@ -88,31 +88,31 @@ function show_progress(io::IO, p::MiniProgressBar; termwidth=nothing, carriagere print(io, p.status) else hascolor = get(io, :color, false)::Bool - printstyled(io, "━"^n_filled; color=p.color) + printstyled(io, "━"^n_filled; color = p.color) if n_left > 0 if hascolor if partial_filled > 0.5 - printstyled(io, "╸"; color=p.color) # More filled, use ╸ + printstyled(io, "╸"; color = p.color) # More filled, use ╸ else - printstyled(io, "╺"; color=:light_black) # Less filled, use ╺ + printstyled(io, "╺"; color = :light_black) # Less filled, use ╺ end end c = hascolor ? "━" : " " - printstyled(io, c^(n_left-1+!hascolor); color=:light_black) + printstyled(io, c^(n_left - 1 + !hascolor); color = :light_black) end - printstyled(io, " "; color=:light_black) + printstyled(io, " "; color = :light_black) print(io, progress_text) end carriagereturn && print(io, "\r") end # Print everything in one call - print(io, to_print) + return print(io, to_print) end function end_progress(io, p::MiniProgressBar) ansi_enablecursor = "\e[?25h" ansi_clearline = "\e[2K" - print(io, ansi_enablecursor * ansi_clearline) + return print(io, ansi_enablecursor * ansi_clearline) end # Useful when writing a progress bar in the bottom @@ -130,7 +130,7 @@ function print_progress_bottom(io::IO) ansi_clearline = "\e[2K" ansi_movecol1 = "\e[1G" ansi_moveup(n::Int) = string("\e[", n, "A") - print(io, "\e[S" * ansi_moveup(1) * ansi_clearline * ansi_movecol1) + return print(io, "\e[S" * ansi_moveup(1) * ansi_clearline * ansi_movecol1) end end diff --git a/src/Operations.jl b/src/Operations.jl index 1dbc82a6fc..a1fa3fdd99 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -10,7 +10,7 @@ import LibGit2, Dates, TOML using ..Types, ..Resolve, ..PlatformEngines, ..GitTools, ..MiniProgressBars import ..depots, ..depots1, ..devdir, ..set_readonly, ..Types.PackageEntry import ..Artifacts: ensure_artifact_installed, artifact_names, extract_all_hashes, - artifact_exists, select_downloadable_artifacts, mv_temp_dir_retries + artifact_exists, select_downloadable_artifacts, mv_temp_dir_retries using Base.BinaryPlatforms import ...Pkg import ...Pkg: pkg_server, Registry, pathrepr, can_fancyprint, printpkgstyle, stderr_f, OFFLINE_MODE @@ -22,7 +22,7 @@ import ...Pkg: usable_io, discover_repo ######### # Helper functions for yanked package checking -function is_pkgversion_yanked(uuid::UUID, version::VersionNumber, registries::Vector{Registry.RegistryInstance}=Registry.reachable_registries()) +function is_pkgversion_yanked(uuid::UUID, version::VersionNumber, registries::Vector{Registry.RegistryInstance} = Registry.reachable_registries()) for reg in registries reg_pkg = get(reg, uuid, nothing) if reg_pkg !== nothing @@ -35,14 +35,14 @@ function is_pkgversion_yanked(uuid::UUID, version::VersionNumber, registries::Ve return false end -function is_pkgversion_yanked(pkg::PackageSpec, registries::Vector{Registry.RegistryInstance}=Registry.reachable_registries()) +function is_pkgversion_yanked(pkg::PackageSpec, registries::Vector{Registry.RegistryInstance} = Registry.reachable_registries()) if pkg.uuid === nothing || pkg.version === nothing || !(pkg.version isa VersionNumber) return false end return is_pkgversion_yanked(pkg.uuid, pkg.version, registries) end -function is_pkgversion_yanked(entry::PackageEntry, registries::Vector{Registry.RegistryInstance}=Registry.reachable_registries()) +function is_pkgversion_yanked(entry::PackageEntry, registries::Vector{Registry.RegistryInstance} = Registry.reachable_registries()) if entry.version === nothing || !(entry.version isa VersionNumber) return false end @@ -50,7 +50,7 @@ function is_pkgversion_yanked(entry::PackageEntry, registries::Vector{Registry.R end function default_preserve() - if Base.get_bool_env("JULIA_PKG_PRESERVE_TIERED_INSTALLED", false) + return if Base.get_bool_env("JULIA_PKG_PRESERVE_TIERED_INSTALLED", false) PRESERVE_TIERED_INSTALLED else PRESERVE_TIERED @@ -71,14 +71,14 @@ end # more accurate name is `should_be_tracking_registered_version` # the only way to know for sure is to key into the registries -tracking_registered_version(pkg::Union{PackageSpec, PackageEntry}, julia_version=VERSION) = +tracking_registered_version(pkg::Union{PackageSpec, PackageEntry}, julia_version = VERSION) = !is_stdlib(pkg.uuid, julia_version) && pkg.path === nothing && pkg.repo.source === nothing function source_path(manifest_file::String, pkg::Union{PackageSpec, PackageEntry}, julia_version = VERSION) - pkg.tree_hash !== nothing ? find_installed(pkg.name, pkg.uuid, pkg.tree_hash) : - pkg.path !== nothing ? joinpath(dirname(manifest_file), pkg.path) : - is_or_was_stdlib(pkg.uuid, julia_version) ? Types.stdlib_path(pkg.name) : - nothing + return pkg.tree_hash !== nothing ? find_installed(pkg.name, pkg.uuid, pkg.tree_hash) : + pkg.path !== nothing ? joinpath(dirname(manifest_file), pkg.path) : + is_or_was_stdlib(pkg.uuid, julia_version) ? Types.stdlib_path(pkg.name) : + nothing end #TODO rename @@ -96,8 +96,10 @@ function load_version(version, fixed, preserve::PreserveLevel) end end -function load_direct_deps(env::EnvCache, pkgs::Vector{PackageSpec}=PackageSpec[]; - preserve::PreserveLevel=PRESERVE_DIRECT) +function load_direct_deps( + env::EnvCache, pkgs::Vector{PackageSpec} = PackageSpec[]; + preserve::PreserveLevel = PRESERVE_DIRECT + ) pkgs_direct = load_project_deps(env.project, env.project_file, env.manifest, env.manifest_file, pkgs; preserve) for (path, project) in env.workspace @@ -132,12 +134,14 @@ function load_direct_deps(env::EnvCache, pkgs::Vector{PackageSpec}=PackageSpec[] return vcat(pkgs, pkgs_direct) end -function load_project_deps(project::Project, project_file::String, manifest::Manifest, manifest_file::String, pkgs::Vector{PackageSpec}=PackageSpec[]; - preserve::PreserveLevel=PRESERVE_DIRECT) +function load_project_deps( + project::Project, project_file::String, manifest::Manifest, manifest_file::String, pkgs::Vector{PackageSpec} = PackageSpec[]; + preserve::PreserveLevel = PRESERVE_DIRECT + ) pkgs_direct = PackageSpec[] if project.name !== nothing && project.uuid !== nothing && findfirst(pkg -> pkg.uuid == project.uuid, pkgs) === nothing path = Types.relative_project_path(manifest_file, dirname(project_file)) - pkg = PackageSpec(;name=project.name, uuid=project.uuid, version=project.version, path) + pkg = PackageSpec(; name = project.name, uuid = project.uuid, version = project.version, path) push!(pkgs_direct, pkg) end @@ -145,43 +149,51 @@ function load_project_deps(project::Project, project_file::String, manifest::Man findfirst(pkg -> pkg.uuid == uuid, pkgs) === nothing || continue # do not duplicate packages path, repo = get_path_repo(project, name) entry = manifest_info(manifest, uuid) - push!(pkgs_direct, entry === nothing ? - PackageSpec(;uuid, name, path, repo) : - PackageSpec(; - uuid = uuid, - name = name, - path = path === nothing ? entry.path : path, - repo = repo == GitRepo() ? entry.repo : repo, - pinned = entry.pinned, - tree_hash = entry.tree_hash, # TODO should tree_hash be changed too? - version = load_version(entry.version, isfixed(entry), preserve), - )) + push!( + pkgs_direct, entry === nothing ? + PackageSpec(; uuid, name, path, repo) : + PackageSpec(; + uuid = uuid, + name = name, + path = path === nothing ? entry.path : path, + repo = repo == GitRepo() ? entry.repo : repo, + pinned = entry.pinned, + tree_hash = entry.tree_hash, # TODO should tree_hash be changed too? + version = load_version(entry.version, isfixed(entry), preserve), + ) + ) end return pkgs_direct end -function load_manifest_deps(manifest::Manifest, pkgs::Vector{PackageSpec}=PackageSpec[]; - preserve::PreserveLevel=PRESERVE_ALL) +function load_manifest_deps( + manifest::Manifest, pkgs::Vector{PackageSpec} = PackageSpec[]; + preserve::PreserveLevel = PRESERVE_ALL + ) pkgs = copy(pkgs) for (uuid, entry) in manifest findfirst(pkg -> pkg.uuid == uuid, pkgs) === nothing || continue # do not duplicate packages - push!(pkgs, PackageSpec( - uuid = uuid, - name = entry.name, - path = entry.path, - pinned = entry.pinned, - repo = entry.repo, - tree_hash = entry.tree_hash, # TODO should tree_hash be changed too? - version = load_version(entry.version, isfixed(entry), preserve), - )) + push!( + pkgs, PackageSpec( + uuid = uuid, + name = entry.name, + path = entry.path, + pinned = entry.pinned, + repo = entry.repo, + tree_hash = entry.tree_hash, # TODO should tree_hash be changed too? + version = load_version(entry.version, isfixed(entry), preserve), + ) + ) end return pkgs end -function load_all_deps(env::EnvCache, pkgs::Vector{PackageSpec}=PackageSpec[]; - preserve::PreserveLevel=PRESERVE_ALL) - pkgs = load_manifest_deps(env.manifest, pkgs; preserve=preserve) +function load_all_deps( + env::EnvCache, pkgs::Vector{PackageSpec} = PackageSpec[]; + preserve::PreserveLevel = PRESERVE_ALL + ) + pkgs = load_manifest_deps(env.manifest, pkgs; preserve = preserve) # Sources takes presedence over the manifest... for pkg in pkgs path, repo = get_path_repo(env.project, pkg.name) @@ -195,7 +207,7 @@ function load_all_deps(env::EnvCache, pkgs::Vector{PackageSpec}=PackageSpec[]; pkg.repo.rev = repo.rev end end - return load_direct_deps(env, pkgs; preserve=preserve) + return load_direct_deps(env, pkgs; preserve = preserve) end function load_all_deps_loadable(env::EnvCache) @@ -207,7 +219,7 @@ function load_all_deps_loadable(env::EnvCache) end -function is_instantiated(env::EnvCache, workspace::Bool=false; platform = HostPlatform())::Bool +function is_instantiated(env::EnvCache, workspace::Bool = false; platform = HostPlatform())::Bool # Load everything if workspace pkgs = Operations.load_all_deps(env) @@ -220,7 +232,7 @@ function is_instantiated(env::EnvCache, workspace::Bool=false; platform = HostPl # so only add it if it isn't there idx = findfirst(x -> x.uuid == env.pkg.uuid, pkgs) if idx === nothing - push!(pkgs, Types.PackageSpec(name=env.pkg.name, uuid=env.pkg.uuid, version=env.pkg.version, path=dirname(env.project_file))) + push!(pkgs, Types.PackageSpec(name = env.pkg.name, uuid = env.pkg.uuid, version = env.pkg.version, path = dirname(env.project_file))) end else # Make sure artifacts for project exist even if it is not a package @@ -235,8 +247,10 @@ function update_manifest!(env::EnvCache, pkgs::Vector{PackageSpec}, deps_map, ju empty!(manifest) for pkg in pkgs - entry = PackageEntry(;name = pkg.name, version = pkg.version, pinned = pkg.pinned, - tree_hash = pkg.tree_hash, path = pkg.path, repo = pkg.repo, uuid=pkg.uuid) + entry = PackageEntry(; + name = pkg.name, version = pkg.version, pinned = pkg.pinned, + tree_hash = pkg.tree_hash, path = pkg.path, repo = pkg.repo, uuid = pkg.uuid + ) if is_stdlib(pkg.uuid, julia_version) # Only set stdlib versions for versioned (external) stdlibs entry.version = stdlib_version(pkg.uuid, julia_version) @@ -245,7 +259,7 @@ function update_manifest!(env::EnvCache, pkgs::Vector{PackageSpec}, deps_map, ju env.manifest[pkg.uuid] = entry end prune_manifest(env) - record_project_hash(env) + return record_project_hash(env) end # This has to be done after the packages have been downloaded @@ -289,7 +303,7 @@ function fixups_from_projectfile!(ctx::Context) end end end - prune_manifest(env) + return prune_manifest(env) end #################### @@ -351,8 +365,8 @@ end function collect_project(pkg::Union{PackageSpec, Nothing}, path::String) deps = PackageSpec[] weakdeps = Set{UUID}() - project_file = projectfile_path(path; strict=true) - project = project_file === nothing ? Project() : read_project(project_file) + project_file = projectfile_path(path; strict = true) + project = project_file === nothing ? Project() : read_project(project_file) julia_compat = get_compat(project, "julia") if !isnothing(julia_compat) && !(VERSION in julia_compat) pkgerror("julia version requirement from Project.toml's compat section not satisfied for package at `$path`") @@ -360,7 +374,7 @@ function collect_project(pkg::Union{PackageSpec, Nothing}, path::String) for (name, uuid) in project.deps path, repo = get_path_repo(project, name) vspec = get_compat(project, name) - push!(deps, PackageSpec(name=name, uuid=uuid, version=vspec, path=path, repo=repo)) + push!(deps, PackageSpec(name = name, uuid = uuid, version = vspec, path = path, repo = repo)) end for (name, uuid) in project.weakdeps vspec = get_compat(project, name) @@ -394,16 +408,21 @@ function collect_developed!(env::EnvCache, pkg::PackageSpec, developed::Vector{P if is_tracking_path(pkg) # normalize path # TODO: If path is collected from project, it is relative to the project file - # otherwise relative to manifest file.... - pkg.path = Types.relative_project_path(env.manifest_file, - project_rel_path(source_env, - source_path(source_env.manifest_file, pkg))) + # otherwise relative to manifest file.... + pkg.path = Types.relative_project_path( + env.manifest_file, + project_rel_path( + source_env, + source_path(source_env.manifest_file, pkg) + ) + ) push!(developed, pkg) collect_developed!(env, pkg, developed) elseif is_tracking_repo(pkg) push!(developed, pkg) end end + return end function collect_developed(env::EnvCache, pkgs::Vector{PackageSpec}) @@ -415,8 +434,8 @@ function collect_developed(env::EnvCache, pkgs::Vector{PackageSpec}) end function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UUID, String}) - deps_map = Dict{UUID,Vector{PackageSpec}}() - weak_map = Dict{UUID,Set{UUID}}() + deps_map = Dict{UUID, Vector{PackageSpec}}() + weak_map = Dict{UUID, Set{UUID}}() uuid = Types.project_uuid(env) deps, weakdeps = collect_project(env.pkg, dirname(env.project_file)) @@ -426,7 +445,7 @@ function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UU for (path, project) in env.workspace uuid = Types.project_uuid(project, path) - pkg = project.name === nothing ? nothing : PackageSpec(name=project.name, uuid=uuid) + pkg = project.name === nothing ? nothing : PackageSpec(name = project.name, uuid = uuid) deps, weakdeps = collect_project(pkg, path) deps_map[Types.project_uuid(env)] = deps weak_map[Types.project_uuid(env)] = weakdeps @@ -440,7 +459,7 @@ function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UU if (path === nothing || !isdir(path)) && (pkg.repo.rev !== nothing || pkg.repo.source !== nothing) # ensure revved package is installed # pkg.tree_hash is set in here - Types.handle_repo_add!(Types.Context(env=env), pkg) + Types.handle_repo_add!(Types.Context(env = env), pkg) # Recompute path path = project_rel_path(env, source_path(env.manifest_file, pkg)) end @@ -470,7 +489,7 @@ function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UU weak_map[pkg.uuid] = weakdeps end - fixed = Dict{UUID,Resolve.Fixed}() + fixed = Dict{UUID, Resolve.Fixed}() # Collect the dependencies for the fixed packages for (uuid, deps) in deps_map q = Dict{UUID, VersionSpec}() @@ -510,8 +529,10 @@ end # sets version to a VersionNumber # adds any other packages which may be in the dependency graph # all versioned packages should have a `tree_hash` -function resolve_versions!(env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, julia_version, - installed_only::Bool) +function resolve_versions!( + env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, julia_version, + installed_only::Bool + ) installed_only = installed_only || OFFLINE_MODE[] # compatibility if julia_version !== nothing @@ -519,7 +540,7 @@ function resolve_versions!(env::EnvCache, registries::Vector{Registry.RegistryIn env.manifest.julia_version = dropbuild(VERSION) v = intersect(julia_version, get_compat_workspace(env, "julia")) if isempty(v) - @warn "julia version requirement for project not satisfied" _module=nothing _file=nothing + @warn "julia version requirement for project not satisfied" _module = nothing _file = nothing end end @@ -552,8 +573,11 @@ function resolve_versions!(env::EnvCache, registries::Vector{Registry.RegistryIn compat = get_compat_workspace(env, pkg.name) v = intersect(pkg.version, compat) if isempty(v) - throw(Resolve.ResolverError( - "empty intersection between $(pkg.name)@$(pkg.version) and project compatibility $(compat)")) + throw( + Resolve.ResolverError( + "empty intersection between $(pkg.name)@$(pkg.version) and project compatibility $(compat)" + ) + ) end # Work around not clobbering 0.x.y+ for checked out old type of packages if !(pkg.version isa VersionNumber) @@ -600,7 +624,7 @@ function resolve_versions!(env::EnvCache, registries::Vector{Registry.RegistryIn pkg.version = vers[pkg.uuid] else name = is_stdlib(uuid) ? stdlib_infos()[uuid].name : registered_name(registries, uuid) - push!(pkgs, PackageSpec(;name=name, uuid=uuid, version=ver)) + push!(pkgs, PackageSpec(; name = name, uuid = uuid, version = ver)) end end final_deps_map = Dict{UUID, Dict{String, UUID}}() @@ -620,7 +644,7 @@ function resolve_versions!(env::EnvCache, registries::Vector{Registry.RegistryIn pkgerror("version $(pkg.version) of package $(pkg.name) is not available. Available versions: $(join(available_versions, ", "))") end for (uuid, _) in compat_map[pkg.uuid][pkg.version] - d[names[uuid]] = uuid + d[names[uuid]] = uuid end d end @@ -632,17 +656,21 @@ function resolve_versions!(env::EnvCache, registries::Vector{Registry.RegistryIn return final_deps_map end -get_or_make!(d::Dict{K,V}, k::K) where {K,V} = get!(d, k) do; V() end +get_or_make!(d::Dict{K, V}, k::K) where {K, V} = get!(d, k) do; + V() +end const JULIA_UUID = UUID("1222c4b2-2114-5bfd-aeef-88e4692bbb3e") const PKGORIGIN_HAVE_VERSION = :version in fieldnames(Base.PkgOrigin) -function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance}, uuid_to_name::Dict{UUID,String}, - reqs::Resolve.Requires, fixed::Dict{UUID,Resolve.Fixed}, julia_version, - installed_only::Bool) +function deps_graph( + env::EnvCache, registries::Vector{Registry.RegistryInstance}, uuid_to_name::Dict{UUID, String}, + reqs::Resolve.Requires, fixed::Dict{UUID, Resolve.Fixed}, julia_version, + installed_only::Bool + ) uuids = Set{UUID}() union!(uuids, keys(reqs)) union!(uuids, keys(fixed)) - for fixed_uuids in map(fx->keys(fx.requires), values(fixed)) + for fixed_uuids in map(fx -> keys(fx.requires), values(fixed)) union!(uuids, fixed_uuids) end @@ -650,11 +678,11 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} seen = Set{UUID}() # pkg -> version -> (dependency => compat): - all_compat = Dict{UUID,Dict{VersionNumber,Dict{UUID,VersionSpec}}}() - weak_compat = Dict{UUID,Dict{VersionNumber,Set{UUID}}}() + all_compat = Dict{UUID, Dict{VersionNumber, Dict{UUID, VersionSpec}}}() + weak_compat = Dict{UUID, Dict{VersionNumber, Set{UUID}}}() for (fp, fx) in fixed - all_compat[fp] = Dict(fx.version => Dict{UUID,VersionSpec}()) + all_compat[fp] = Dict(fx.version => Dict{UUID, VersionSpec}()) end while true @@ -702,7 +730,7 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} # TODO, pull this into a function Registry.isyanked(info, v) && continue if installed_only - pkg_spec = PackageSpec(name=pkg.name, uuid=pkg.uuid, version=v, tree_hash=Registry.treehash(info, v)) + pkg_spec = PackageSpec(name = pkg.name, uuid = pkg.uuid, version = v, tree_hash = Registry.treehash(info, v)) is_package_downloaded(env.manifest_file, pkg_spec) || continue end @@ -722,13 +750,14 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} merge!(dv, compat_info) union!(uuids, keys(compat_info)) end + return end add_compat!(all_compat_u, Registry.compat_info(info)) weak_compat_info = Registry.weak_compat_info(info) if weak_compat_info !== nothing add_compat!(all_compat_u, weak_compat_info) # Version to Set - for (v, compat_info) in weak_compat_info + for (v, compat_info) in weak_compat_info weak_compat_u[v] = keys(compat_info) end end @@ -750,7 +779,7 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} end return Resolve.Graph(all_compat, weak_compat, uuid_to_name, reqs, fixed, false, julia_version), - all_compat + all_compat end ######################## @@ -766,11 +795,11 @@ end # Returns if archive successfully installed function install_archive( - urls::Vector{Pair{String,Bool}}, - hash::SHA1, - version_path::String; - io::IO=stderr_f() -)::Bool + urls::Vector{Pair{String, Bool}}, + hash::SHA1, + version_path::String; + io::IO = stderr_f() + )::Bool # Because we use `mv_temp_dir_retries` which uses `rename` not `mv` it can fail if the temp # files are on a different fs. So use a temp dir in the same depot dir as some systems might # be serving different parts of the depot on different filesystems via links i.e. pkgeval does this. @@ -783,7 +812,7 @@ function install_archive( push!(tmp_objects, path) # for cleanup url_success = true try - PlatformEngines.download(url, path; verbose=false, io=io) + PlatformEngines.download(url, path; verbose = false, io = io) catch e e isa InterruptException && rethrow() url_success = false @@ -795,7 +824,7 @@ function install_archive( push!(tmp_objects, dir) # for cleanup # Might fail to extract an archive (https://github.com/JuliaPackaging/PkgServer.jl/issues/126) try - unpack(path, dir; verbose=false) + unpack(path, dir; verbose = false) catch e e isa ProcessFailedException || rethrow() @warn "failed to extract archive downloaded from $(url)" @@ -825,19 +854,19 @@ function install_archive( break # successful install end # Clean up and exit - foreach(x -> Base.rm(x; force=true, recursive=true), tmp_objects) + foreach(x -> Base.rm(x; force = true, recursive = true), tmp_objects) return url_success end const refspecs = ["+refs/*:refs/remotes/cache/*"] function install_git( - io::IO, - uuid::UUID, - name::String, - hash::SHA1, - urls::Set{String}, - version_path::String -)::Nothing + io::IO, + uuid::UUID, + name::String, + hash::SHA1, + urls::Set{String}, + version_path::String + )::Nothing if isempty(urls) pkgerror( "Package $name [$uuid] has no repository URL available. This could happen if:\n" * @@ -857,17 +886,20 @@ function install_git( ispath(clones_dir) || mkpath(clones_dir) repo_path = joinpath(clones_dir, string(uuid)) first_url = first(urls) - repo = GitTools.ensure_clone(io, repo_path, first_url; isbare=true, - header = "[$uuid] $name from $first_url") + repo = GitTools.ensure_clone( + io, repo_path, first_url; isbare = true, + header = "[$uuid] $name from $first_url" + ) git_hash = LibGit2.GitHash(hash.bytes) for url in urls - try LibGit2.with(LibGit2.GitObject, repo, git_hash) do g + try + LibGit2.with(LibGit2.GitObject, repo, git_hash) do g end break # object was found, we can stop catch err err isa LibGit2.GitError && err.code == LibGit2.Error.ENOTFOUND || rethrow() end - GitTools.fetch(io, repo, url, refspecs=refspecs) + GitTools.fetch(io, repo, url, refspecs = refspecs) end tree = try LibGit2.GitObject(repo, git_hash) @@ -886,9 +918,9 @@ function install_git( end end -function collect_artifacts(pkg_root::String; platform::AbstractPlatform=HostPlatform(), include_lazy::Bool=false) +function collect_artifacts(pkg_root::String; platform::AbstractPlatform = HostPlatform(), include_lazy::Bool = false) # Check to see if this package has an (Julia)Artifacts.toml - artifacts_tomls = Tuple{String,Base.TOML.TOMLDict}[] + artifacts_tomls = Tuple{String, Base.TOML.TOMLDict}[] for f in artifact_names artifacts_toml = joinpath(pkg_root, f) if isfile(artifacts_toml) @@ -903,7 +935,7 @@ function collect_artifacts(pkg_root::String; platform::AbstractPlatform=HostPlat meta_toml = String(read(select_cmd)) res = TOML.tryparse(meta_toml) if res isa TOML.ParserError - errstr = sprint(showerror, res; context=stderr) + errstr = sprint(showerror, res; context = stderr) pkgerror("failed to parse TOML output from running $(repr(selector_path)), got: \n$errstr") else push!(artifacts_tomls, (artifacts_toml, TOML.parse(meta_toml))) @@ -927,12 +959,14 @@ mutable struct DownloadState const bar::MiniProgressBar end -function download_artifacts(ctx::Context; - platform::AbstractPlatform=HostPlatform(), - julia_version = VERSION, - verbose::Bool=false, - io::IO=stderr_f(), - include_lazy::Bool=false) +function download_artifacts( + ctx::Context; + platform::AbstractPlatform = HostPlatform(), + julia_version = VERSION, + verbose::Bool = false, + io::IO = stderr_f(), + include_lazy::Bool = false + ) env = ctx.env io = ctx.io fancyprint = can_fancyprint(io) @@ -981,15 +1015,15 @@ function download_artifacts(ctx::Context; for name in keys(artifacts) local rname = rpad(name, longest_name_length) local hash = SHA1(artifacts[name]["git-tree-sha1"]::String) - local bar = MiniProgressBar(;header=rname, main=false, indent=2, color = Base.info_color()::Symbol, mode=:data, always_reprint=true) + local bar = MiniProgressBar(; header = rname, main = false, indent = 2, color = Base.info_color()::Symbol, mode = :data, always_reprint = true) local dstate = DownloadState(:ready, "", time_ns(), Base.ReentrantLock(), bar) - function progress(total, current; status="") + function progress(total, current; status = "") local t = time_ns() if isempty(status) dstate.bar.max = total dstate.bar.current = current end - lock(dstate.status_lock) do + return lock(dstate.status_lock) do dstate.status = status dstate.status_update_time = t end @@ -998,25 +1032,27 @@ function download_artifacts(ctx::Context; local pkg_server_eligible = pkg_uuid !== nothing && Registry.is_pkg_in_pkgserver_registry(pkg_uuid, server_registry_info, ctx.registries) # returns a string if exists, or function that downloads the artifact if not - local ret = ensure_artifact_installed(name, artifacts[name], artifacts_toml; - pkg_server_eligible, verbose, quiet_download=!(usable_io(io)), io, progress) + local ret = ensure_artifact_installed( + name, artifacts[name], artifacts_toml; + pkg_server_eligible, verbose, quiet_download = !(usable_io(io)), io, progress + ) if ret isa Function download_states[hash] = dstate download_jobs[hash] = () -> begin - try - dstate.state = :running - ret() - if !fancyprint && dstate.bar.max > 1 # if another process downloaded, then max is never set greater than 1 - @lock print_lock printpkgstyle(io, :Installed, "artifact $rname $(MiniProgressBars.pkg_format_bytes(dstate.bar.max; sigdigits=1))") - end - catch - dstate.state = :failed - rethrow() - else - dstate.state = :done + try + dstate.state = :running + ret() + if !fancyprint && dstate.bar.max > 1 # if another process downloaded, then max is never set greater than 1 + @lock print_lock printpkgstyle(io, :Installed, "artifact $rname $(MiniProgressBars.pkg_format_bytes(dstate.bar.max; sigdigits = 1))") end + catch + dstate.state = :failed + rethrow() + else + dstate.state = :done end + end end end end @@ -1027,26 +1063,26 @@ function download_artifacts(ctx::Context; try print(io, ansi_disablecursor) first = true - timer = Timer(0, interval=1/10) + timer = Timer(0, interval = 1 / 10) # TODO: Implement as a new MiniMultiProgressBar - main_bar = MiniProgressBar(; indent=2, header = "Installing artifacts", color = :green, mode = :int, always_reprint=true) + main_bar = MiniProgressBar(; indent = 2, header = "Installing artifacts", color = :green, mode = :int, always_reprint = true) main_bar.max = length(download_states) while !is_done[] main_bar.current = count(x -> x.state == :done, values(download_states)) - local str = sprint(context=io) do iostr + local str = sprint(context = io) do iostr first || print(iostr, ansi_cleartoend) n_printed = 1 - show_progress(iostr, main_bar; carriagereturn=false) + show_progress(iostr, main_bar; carriagereturn = false) println(iostr) - for dstate in sort!(collect(values(download_states)), by=v->v.bar.max, rev=true) - local status, status_update_time = lock(()->(dstate.status, dstate.status_update_time), dstate.status_lock) + for dstate in sort!(collect(values(download_states)), by = v -> v.bar.max, rev = true) + local status, status_update_time = lock(() -> (dstate.status, dstate.status_update_time), dstate.status_lock) # only update the bar's status message if it is stalled for at least 0.5 s. # If the new status message is empty, go back to showing the bar without waiting. if isempty(status) || time_ns() - status_update_time > UInt64(500_000_000) dstate.bar.status = status end dstate.state == :running && (dstate.bar.max > 1000 || !isempty(dstate.bar.status)) || continue - show_progress(iostr, dstate.bar; carriagereturn=false) + show_progress(iostr, dstate.bar; carriagereturn = false) println(iostr) n_printed += 1 end @@ -1058,7 +1094,7 @@ function download_artifacts(ctx::Context; end print(io, ansi_cleartoend) main_bar.current = count(x -> x[2].state == :done, download_states) - show_progress(io, main_bar; carriagereturn=false) + show_progress(io, main_bar; carriagereturn = false) println(io) catch e e isa InterruptException || rethrow() @@ -1090,7 +1126,7 @@ function download_artifacts(ctx::Context; if !isempty(errors) all_errors = collect(errors) - local str = sprint(context=io) do iostr + local str = sprint(context = io) do iostr for e in all_errors Base.showerror(iostr, e) length(all_errors) > 1 && println(iostr) @@ -1101,10 +1137,10 @@ function download_artifacts(ctx::Context; end - write_env_usage(used_artifact_tomls, "artifact_usage.toml") + return write_env_usage(used_artifact_tomls, "artifact_usage.toml") end -function check_artifacts_downloaded(pkg_root::String; platform::AbstractPlatform=HostPlatform()) +function check_artifacts_downloaded(pkg_root::String; platform::AbstractPlatform = HostPlatform()) for (artifacts_toml, artifacts) in collect_artifacts(pkg_root; platform) for name in keys(artifacts) if !artifact_exists(Base.SHA1(artifacts[name]["git-tree-sha1"])) @@ -1131,9 +1167,9 @@ function find_urls(registries::Vector{Registry.RegistryInstance}, uuid::UUID) end -download_source(ctx::Context; readonly=true) = download_source(ctx, values(ctx.env.manifest); readonly) +download_source(ctx::Context; readonly = true) = download_source(ctx, values(ctx.env.manifest); readonly) -function download_source(ctx::Context, pkgs; readonly=true) +function download_source(ctx::Context, pkgs; readonly = true) pidfile_stale_age = 10 # recommended value is about 3-5x an estimated normal download time (i.e. 2-3s) pkgs_to_install = NamedTuple{(:pkg, :urls, :path), Tuple{eltype(pkgs), Set{String}, String}}[] for pkg in pkgs @@ -1143,7 +1179,7 @@ function download_source(ctx::Context, pkgs; readonly=true) mkpath(dirname(path)) # the `packages/Package` dir needs to exist for the pidfile to be created FileWatching.mkpidlock(() -> ispath(path), path * ".pid", stale_age = pidfile_stale_age) && continue urls = find_urls(ctx.registries, pkg.uuid) - push!(pkgs_to_install, (;pkg, urls, path)) + push!(pkgs_to_install, (; pkg, urls, path)) end length(pkgs_to_install) == 0 && return Set{UUID}() @@ -1154,7 +1190,7 @@ function download_source(ctx::Context, pkgs; readonly=true) missed_packages = eltype(pkgs_to_install)[] widths = [textwidth(pkg.name) for (pkg, _) in pkgs_to_install] - max_name = maximum(widths; init=0) + max_name = maximum(widths; init = 0) # Check what registries the current pkg server tracks # Disable if precompiling to not access internet @@ -1188,7 +1224,7 @@ function download_source(ctx::Context, pkgs; readonly=true) put!(results, (pkg, false, (urls, path))) return end - archive_urls = Pair{String,Bool}[] + archive_urls = Pair{String, Bool}[] # Check if the current package is available in one of the registries being tracked by the pkg server # In that case, download from the package server if Registry.is_pkg_in_pkgserver_registry(pkg.uuid, server_registry_info, ctx.registries) @@ -1201,7 +1237,7 @@ function download_source(ctx::Context, pkgs; readonly=true) url !== nothing && push!(archive_urls, url => false) end try - success = install_archive(archive_urls, pkg.tree_hash, path, io=ctx.io) + success = install_archive(archive_urls, pkg.tree_hash, path, io = ctx.io) if success && readonly set_readonly(path) # In add mode, files should be read-only end @@ -1217,8 +1253,10 @@ function download_source(ctx::Context, pkgs; readonly=true) end end - bar = MiniProgressBar(; indent=1, header = "Downloading packages", color = Base.info_color(), - mode=:int, always_reprint=true) + bar = MiniProgressBar(; + indent = 1, header = "Downloading packages", color = Base.info_color(), + mode = :int, always_reprint = true + ) bar.max = length(pkgs_to_install) fancyprint = can_fancyprint(ctx.io) try @@ -1234,7 +1272,7 @@ function download_source(ctx::Context, pkgs; readonly=true) success, (urls, path) = exc_or_success_or_nothing, bt_or_pathurls success || push!(missed_packages, (; pkg, urls, path)) bar.current = i - str = sprint(; context=ctx.io) do io + str = sprint(; context = ctx.io) do io if success fancyprint && print_progress_bottom(io) vstr = if pkg.version !== nothing @@ -1321,10 +1359,11 @@ function prune_deps(iterator, keep::Set{UUID}) end clean && break end + return end function record_project_hash(env::EnvCache) - env.manifest.other["project_hash"] = Types.workspace_resolve_hash(env) + return env.manifest.other["project_hash"] = Types.workspace_resolve_hash(env) end ######### @@ -1361,16 +1400,16 @@ function any_package_not_installed(manifest::Manifest) return false end -function build(ctx::Context, uuids::Set{UUID}, verbose::Bool; allow_reresolve::Bool=true) +function build(ctx::Context, uuids::Set{UUID}, verbose::Bool; allow_reresolve::Bool = true) if any_package_not_installed(ctx.env.manifest) || !isfile(ctx.env.manifest_file) Pkg.instantiate(ctx, allow_build = false, allow_autoprecomp = false) end all_uuids = get_deps(ctx.env, uuids) - build_versions(ctx, all_uuids; verbose, allow_reresolve) + return build_versions(ctx, all_uuids; verbose, allow_reresolve) end -function dependency_order_uuids(env::EnvCache, uuids::Vector{UUID})::Dict{UUID,Int} - order = Dict{UUID,Int}() +function dependency_order_uuids(env::EnvCache, uuids::Vector{UUID})::Dict{UUID, Int} + order = Dict{UUID, Int}() seen = UUID[] k::Int = 0 function visit(uuid::UUID) @@ -1386,7 +1425,7 @@ function dependency_order_uuids(env::EnvCache, uuids::Vector{UUID})::Dict{UUID,I end foreach(visit, deps) pop!(seen) - order[uuid] = k += 1 + return order[uuid] = k += 1 end visit(uuid::String) = visit(UUID(uuid)) foreach(visit, uuids) @@ -1395,26 +1434,26 @@ end function gen_build_code(build_file::String; inherit_project::Bool = false) code = """ - $(Base.load_path_setup_code(false)) - cd($(repr(dirname(build_file)))) - include($(repr(build_file))) - """ + $(Base.load_path_setup_code(false)) + cd($(repr(dirname(build_file)))) + include($(repr(build_file))) + """ # This will make it so that running Pkg.build runs the build in a session with --startup=no # *unless* the parent julia session is started with --startup=yes explicitly. startup_flag = Base.JLOptions().startupfile == 1 ? "yes" : "no" return ``` - $(Base.julia_cmd()) -O0 --color=no --history-file=no - --startup-file=$startup_flag - $(inherit_project ? `--project=$(Base.active_project())` : ``) - --eval $code - ``` + $(Base.julia_cmd()) -O0 --color=no --history-file=no + --startup-file=$startup_flag + $(inherit_project ? `--project=$(Base.active_project())` : ``) + --eval $code + ``` end with_load_path(f::Function, new_load_path::String) = with_load_path(f, [new_load_path]) function with_load_path(f::Function, new_load_path::Vector{String}) old_load_path = copy(Base.LOAD_PATH) copy!(Base.LOAD_PATH, new_load_path) - try + return try f() finally copy!(LOAD_PATH, old_load_path) @@ -1426,9 +1465,9 @@ pkg_scratchpath() = joinpath(depots1(), "scratchspaces", PkgUUID) builddir(source_path::String) = joinpath(source_path, "deps") buildfile(source_path::String) = joinpath(builddir(source_path), "build.jl") -function build_versions(ctx::Context, uuids::Set{UUID}; verbose=false, allow_reresolve::Bool=true) +function build_versions(ctx::Context, uuids::Set{UUID}; verbose = false, allow_reresolve::Bool = true) # collect builds for UUIDs with `deps/build.jl` files - builds = Tuple{UUID,String,String,VersionNumber}[] + builds = Tuple{UUID, String, String, VersionNumber}[] for uuid in uuids is_stdlib(uuid) && continue if Types.is_project_uuid(ctx.env, uuid) @@ -1453,84 +1492,94 @@ function build_versions(ctx::Context, uuids::Set{UUID}; verbose=false, allow_rer # toposort builds by dependencies order = dependency_order_uuids(ctx.env, UUID[first(build) for build in builds]) sort!(builds, by = build -> order[first(build)]) - max_name = maximum(build->textwidth(build[2]), builds; init=0) + max_name = maximum(build -> textwidth(build[2]), builds; init = 0) - bar = MiniProgressBar(; indent=2, header = "Building packages", color = Base.info_color(), - mode=:int, always_reprint=true) + bar = MiniProgressBar(; + indent = 2, header = "Building packages", color = Base.info_color(), + mode = :int, always_reprint = true + ) bar.max = length(builds) fancyprint = can_fancyprint(ctx.io) fancyprint && start_progress(ctx.io, bar) # build each package versions in a child process try - for (n, (uuid, name, source_path, version)) in enumerate(builds) - pkg = PackageSpec(;uuid=uuid, name=name, version=version) - build_file = buildfile(source_path) - # compatibility shim - local build_project_override, build_project_preferences - if isfile(projectfile_path(builddir(source_path))) - build_project_override = nothing - with_load_path([builddir(source_path), Base.LOAD_PATH...]) do - build_project_preferences = Base.get_preferences() - end - else - build_project_override = gen_target_project(ctx, pkg, source_path, "build") - with_load_path([something(projectfile_path(source_path)), Base.LOAD_PATH...]) do - build_project_preferences = Base.get_preferences() + for (n, (uuid, name, source_path, version)) in enumerate(builds) + pkg = PackageSpec(; uuid = uuid, name = name, version = version) + build_file = buildfile(source_path) + # compatibility shim + local build_project_override, build_project_preferences + if isfile(projectfile_path(builddir(source_path))) + build_project_override = nothing + with_load_path([builddir(source_path), Base.LOAD_PATH...]) do + build_project_preferences = Base.get_preferences() + end + else + build_project_override = gen_target_project(ctx, pkg, source_path, "build") + with_load_path([something(projectfile_path(source_path)), Base.LOAD_PATH...]) do + build_project_preferences = Base.get_preferences() + end end - end - # Put log output in Pkg's scratchspace if the package is content addressed - # by tree sha and in the build directory if it is tracked by path etc. - entry = manifest_info(ctx.env.manifest, uuid) - if entry !== nothing && entry.tree_hash !== nothing - key = string(entry.tree_hash) - scratch = joinpath(pkg_scratchpath(), key) - mkpath(scratch) - log_file = joinpath(scratch, "build.log") - # Associate the logfile with the package being built - dict = Dict{String,Any}(scratch => [ - Dict{String,Any}("time" => Dates.now(), "parent_projects" => [projectfile_path(source_path)]) - ]) - open(joinpath(depots1(), "logs", "scratch_usage.toml"), "a") do io - TOML.print(io, dict) + # Put log output in Pkg's scratchspace if the package is content addressed + # by tree sha and in the build directory if it is tracked by path etc. + entry = manifest_info(ctx.env.manifest, uuid) + if entry !== nothing && entry.tree_hash !== nothing + key = string(entry.tree_hash) + scratch = joinpath(pkg_scratchpath(), key) + mkpath(scratch) + log_file = joinpath(scratch, "build.log") + # Associate the logfile with the package being built + dict = Dict{String, Any}( + scratch => [ + Dict{String, Any}("time" => Dates.now(), "parent_projects" => [projectfile_path(source_path)]), + ] + ) + open(joinpath(depots1(), "logs", "scratch_usage.toml"), "a") do io + TOML.print(io, dict) + end + else + log_file = splitext(build_file)[1] * ".log" end - else - log_file = splitext(build_file)[1] * ".log" - end - - fancyprint && print_progress_bottom(ctx.io) - printpkgstyle(ctx.io, :Building, - rpad(name * " ", max_name + 1, "─") * "→ " * pathrepr(log_file)) - bar.current = n-1 + fancyprint && print_progress_bottom(ctx.io) - fancyprint && show_progress(ctx.io, bar) - - let log_file=log_file - sandbox(ctx, pkg, builddir(source_path), build_project_override; preferences=build_project_preferences, allow_reresolve) do - flush(ctx.io) - ok = open(log_file, "w") do log - std = verbose ? ctx.io : log - success(pipeline(gen_build_code(buildfile(source_path)), - stdout=std, stderr=std)) - end - ok && return - n_lines = isinteractive() ? 100 : 5000 - # TODO: Extract last n lines more efficiently - log_lines = readlines(log_file) - log_show = join(log_lines[max(1, length(log_lines) - n_lines):end], '\n') - full_log_at, last_lines = - if length(log_lines) > n_lines - "\n\nFull log at $log_file", - ", showing the last $n_lines of log" - else - "", "" + printpkgstyle( + ctx.io, :Building, + rpad(name * " ", max_name + 1, "─") * "→ " * pathrepr(log_file) + ) + bar.current = n - 1 + + fancyprint && show_progress(ctx.io, bar) + + let log_file = log_file + sandbox(ctx, pkg, builddir(source_path), build_project_override; preferences = build_project_preferences, allow_reresolve) do + flush(ctx.io) + ok = open(log_file, "w") do log + std = verbose ? ctx.io : log + success( + pipeline( + gen_build_code(buildfile(source_path)), + stdout = std, stderr = std + ) + ) + end + ok && return + n_lines = isinteractive() ? 100 : 5000 + # TODO: Extract last n lines more efficiently + log_lines = readlines(log_file) + log_show = join(log_lines[max(1, length(log_lines) - n_lines):end], '\n') + full_log_at, last_lines = + if length(log_lines) > n_lines + "\n\nFull log at $log_file", + ", showing the last $n_lines of log" + else + "", "" + end + pkgerror("Error building `$(pkg.name)`$last_lines: \n$log_show$full_log_at") end - pkgerror("Error building `$(pkg.name)`$last_lines: \n$log_show$full_log_at") end end - end finally fancyprint && end_progress(ctx.io, bar) end @@ -1611,7 +1660,7 @@ function rm(ctx::Context, pkgs::Vector{PackageSpec}; mode::PackageMode) record_project_hash(ctx.env) # update project & manifest write_env(ctx.env) - show_update(ctx.env, ctx.registries; io=ctx.io) + return show_update(ctx.env, ctx.registries; io = ctx.io) end update_package_add(ctx::Context, pkg::PackageSpec, ::Nothing, is_dep::Bool) = pkg @@ -1620,38 +1669,44 @@ function update_package_add(ctx::Context, pkg::PackageSpec, entry::PackageEntry, if pkg.version == VersionSpec() println(ctx.io, "`$(pkg.name)` is pinned at `v$(entry.version)`: maintaining pinned version") end - return PackageSpec(; uuid=pkg.uuid, name=pkg.name, pinned=true, - version=entry.version, tree_hash=entry.tree_hash, - path=entry.path, repo=entry.repo) + return PackageSpec(; + uuid = pkg.uuid, name = pkg.name, pinned = true, + version = entry.version, tree_hash = entry.tree_hash, + path = entry.path, repo = entry.repo + ) end if entry.path !== nothing || entry.repo.source !== nothing || pkg.repo.source !== nothing return pkg # overwrite everything, nothing to copy over end if is_stdlib(pkg.uuid) return pkg # stdlibs are not versioned like other packages - elseif is_dep && ((isa(pkg.version, VersionNumber) && entry.version == pkg.version) || - (!isa(pkg.version, VersionNumber) && entry.version ∈ pkg.version)) + elseif is_dep && ( + (isa(pkg.version, VersionNumber) && entry.version == pkg.version) || + (!isa(pkg.version, VersionNumber) && entry.version ∈ pkg.version) + ) # leave the package as is at the installed version - return PackageSpec(; uuid=pkg.uuid, name=pkg.name, version=entry.version, - tree_hash=entry.tree_hash) + return PackageSpec(; + uuid = pkg.uuid, name = pkg.name, version = entry.version, + tree_hash = entry.tree_hash + ) end # adding a new version not compatible with the old version, so we just overwrite return pkg end # Update registries AND read them back in. -function update_registries(ctx::Context; force::Bool=true, kwargs...) +function update_registries(ctx::Context; force::Bool = true, kwargs...) OFFLINE_MODE[] && return !force && UPDATED_REGISTRY_THIS_SESSION[] && return - Registry.update(; io=ctx.io, kwargs...) + Registry.update(; io = ctx.io, kwargs...) copy!(ctx.registries, Registry.reachable_registries()) - UPDATED_REGISTRY_THIS_SESSION[] = true + return UPDATED_REGISTRY_THIS_SESSION[] = true end function is_all_registered(registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}) pkgs = filter(tracking_registered_version, pkgs) for pkg in pkgs - if !any(r->haskey(r, pkg.uuid), registries) + if !any(r -> haskey(r, pkg.uuid), registries) return pkg end end @@ -1696,29 +1751,38 @@ function assert_can_add(ctx::Context, pkgs::Vector{PackageSpec}) # package with the same name exist in the project: assert that they have the same uuid existing_uuid = get(ctx.env.project.deps, pkg.name, pkg.uuid) existing_uuid == pkg.uuid || - pkgerror("""Refusing to add package $(err_rep(pkg)). - Package `$(pkg.name)=$(existing_uuid)` with the same name already exists as a direct dependency. - To remove the existing package, use `$(Pkg.in_repl_mode() ? """pkg> rm $(pkg.name)""" : """import Pkg; Pkg.rm("$(pkg.name)")""")`. - """) + pkgerror( + """Refusing to add package $(err_rep(pkg)). + Package `$(pkg.name)=$(existing_uuid)` with the same name already exists as a direct dependency. + To remove the existing package, use `$(Pkg.in_repl_mode() ? """pkg> rm $(pkg.name)""" : """import Pkg; Pkg.rm("$(pkg.name)")""")`. + """ + ) # package with the same uuid exist in the project: assert they have the same name name = findfirst(==(pkg.uuid), ctx.env.project.deps) name === nothing || name == pkg.name || - pkgerror("""Refusing to add package $(err_rep(pkg)). - Package `$name=$(pkg.uuid)` with the same UUID already exists as a direct dependency. - To remove the existing package, use `$(Pkg.in_repl_mode() ? """pkg> rm $name""" : """import Pkg; Pkg.rm("$name")""")`. - """) + pkgerror( + """Refusing to add package $(err_rep(pkg)). + Package `$name=$(pkg.uuid)` with the same UUID already exists as a direct dependency. + To remove the existing package, use `$(Pkg.in_repl_mode() ? """pkg> rm $name""" : """import Pkg; Pkg.rm("$name")""")`. + """ + ) # package with the same uuid exist in the manifest: assert they have the same name entry = get(ctx.env.manifest, pkg.uuid, nothing) entry === nothing || entry.name == pkg.name || - pkgerror("""Refusing to add package $(err_rep(pkg)). - Package `$(entry.name)=$(pkg.uuid)` with the same UUID already exists in the manifest. - To remove the existing package, use `$(Pkg.in_repl_mode() ? """pkg> rm --manifest $(entry.name)=$(pkg.uuid)""" : """import Pkg; Pkg.rm(Pkg.PackageSpec(uuid="$(pkg.uuid)"); mode=Pkg.PKGMODE_MANIFEST)""")`. - """) + pkgerror( + """Refusing to add package $(err_rep(pkg)). + Package `$(entry.name)=$(pkg.uuid)` with the same UUID already exists in the manifest. + To remove the existing package, use `$(Pkg.in_repl_mode() ? """pkg> rm --manifest $(entry.name)=$(pkg.uuid)""" : """import Pkg; Pkg.rm(Pkg.PackageSpec(uuid="$(pkg.uuid)"); mode=Pkg.PKGMODE_MANIFEST)""")`. + """ + ) end + return end -function tiered_resolve(env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, julia_version, - try_all_installed::Bool) +function tiered_resolve( + env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, julia_version, + try_all_installed::Bool + ) if try_all_installed try # do not modify existing subgraph and only add installed versions of the new packages @debug "tiered_resolve: trying PRESERVE_ALL_INSTALLED" @@ -1761,11 +1825,13 @@ function targeted_resolve(env::EnvCache, registries::Vector{Registry.RegistryIns return pkgs, deps_map end -function _resolve(io::IO, env::EnvCache, registries::Vector{Registry.RegistryInstance}, - pkgs::Vector{PackageSpec}, preserve::PreserveLevel, julia_version) +function _resolve( + io::IO, env::EnvCache, registries::Vector{Registry.RegistryInstance}, + pkgs::Vector{PackageSpec}, preserve::PreserveLevel, julia_version + ) usingstrategy = preserve != PRESERVE_TIERED ? " using $preserve" : "" printpkgstyle(io, :Resolving, "package versions$(usingstrategy)...") - try + return try if preserve == PRESERVE_TIERED_INSTALLED tiered_resolve(env, registries, pkgs, julia_version, true) elseif preserve == PRESERVE_TIERED @@ -1779,18 +1845,20 @@ function _resolve(io::IO, env::EnvCache, registries::Vector{Registry.RegistryIns yanked_pkgs = filter(pkg -> is_pkgversion_yanked(pkg, registries), load_all_deps(env)) if !isempty(yanked_pkgs) indent = " "^(Pkg.pkgstyle_indent) - yanked_str = join(map(pkg -> indent * " - " * err_rep(pkg, quotes=false) * " " * string(pkg.version), yanked_pkgs), "\n") + yanked_str = join(map(pkg -> indent * " - " * err_rep(pkg, quotes = false) * " " * string(pkg.version), yanked_pkgs), "\n") printpkgstyle(io, :Warning, """The following package versions were yanked from their registry and \ - are not resolvable:\n$yanked_str""", color=Base.warn_color()) + are not resolvable:\n$yanked_str""", color = Base.warn_color()) end end rethrow() end end -function add(ctx::Context, pkgs::Vector{PackageSpec}, new_git=Set{UUID}(); - allow_autoprecomp::Bool=true, preserve::PreserveLevel=default_preserve(), platform::AbstractPlatform=HostPlatform(), - target::Symbol=:deps) +function add( + ctx::Context, pkgs::Vector{PackageSpec}, new_git = Set{UUID}(); + allow_autoprecomp::Bool = true, preserve::PreserveLevel = default_preserve(), platform::AbstractPlatform = HostPlatform(), + target::Symbol = :deps + ) assert_can_add(ctx, pkgs) # load manifest data for (i, pkg) in pairs(pkgs) @@ -1822,7 +1890,7 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}, new_git=Set{UUID}(); # After downloading resolutionary packages, search for (Julia)Artifacts.toml files # and ensure they are all downloaded and unpacked as well: - download_artifacts(ctx, platform=platform, julia_version=ctx.julia_version) + download_artifacts(ctx, platform = platform, julia_version = ctx.julia_version) # if env is a package add compat entries if ctx.env.project.name !== nothing && ctx.env.project.uuid !== nothing @@ -1840,7 +1908,7 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}, new_git=Set{UUID}(); record_project_hash(ctx.env) # compat entries changed the hash after it was last recorded in update_manifest! write_env(ctx.env) # write env before building - show_update(ctx.env, ctx.registries; io=ctx.io) + show_update(ctx.env, ctx.registries; io = ctx.io) build_versions(ctx, union(new_apply, new_git)) allow_autoprecomp && Pkg._auto_precompile(ctx) else @@ -1853,8 +1921,10 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}, new_git=Set{UUID}(); end # Input: name, uuid, and path -function develop(ctx::Context, pkgs::Vector{PackageSpec}, new_git::Set{UUID}; - preserve::PreserveLevel=default_preserve(), platform::AbstractPlatform=HostPlatform()) +function develop( + ctx::Context, pkgs::Vector{PackageSpec}, new_git::Set{UUID}; + preserve::PreserveLevel = default_preserve(), platform::AbstractPlatform = HostPlatform() + ) assert_can_add(ctx, pkgs) # no need to look at manifest.. dev will just nuke whatever is there before for pkg in pkgs @@ -1866,10 +1936,10 @@ function develop(ctx::Context, pkgs::Vector{PackageSpec}, new_git::Set{UUID}; update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version) new_apply = download_source(ctx) fixups_from_projectfile!(ctx) - download_artifacts(ctx; platform=platform, julia_version=ctx.julia_version) + download_artifacts(ctx; platform = platform, julia_version = ctx.julia_version) write_env(ctx.env) # write env before building - show_update(ctx.env, ctx.registries; io=ctx.io) - build_versions(ctx, union(new_apply, new_git)) + show_update(ctx.env, ctx.registries; io = ctx.io) + return build_versions(ctx, union(new_apply, new_git)) end # load version constraint @@ -1907,7 +1977,7 @@ function up_load_versions!(ctx::Context, pkg::PackageSpec, entry::PackageEntry, r = level == UPLEVEL_PATCH ? VersionRange(ver.major, ver.minor) : level == UPLEVEL_MINOR ? VersionRange(ver.major) : level == UPLEVEL_MAJOR ? VersionRange() : - error("unexpected upgrade level: $level") + error("unexpected upgrade level: $level") pkg.version = VersionSpec(r) end return false @@ -1922,13 +1992,15 @@ function up_load_manifest_info!(pkg::PackageSpec, entry::PackageEntry) if pkg.path === nothing pkg.path = entry.path end - pkg.pinned = entry.pinned + return pkg.pinned = entry.pinned # `pkg.version` and `pkg.tree_hash` is set by `up_load_versions!` end -function load_manifest_deps_up(env::EnvCache, pkgs::Vector{PackageSpec}=PackageSpec[]; - preserve::PreserveLevel=PRESERVE_ALL) +function load_manifest_deps_up( + env::EnvCache, pkgs::Vector{PackageSpec} = PackageSpec[]; + preserve::PreserveLevel = PRESERVE_ALL + ) manifest = env.manifest project = env.project explicit_upgraded = Set(pkg.uuid for pkg in pkgs) @@ -1963,28 +2035,32 @@ function load_manifest_deps_up(env::EnvCache, pkgs::Vector{PackageSpec}=PackageS end # The rest of the packages get fixed - push!(pkgs, PackageSpec( - uuid = uuid, - name = entry.name, - path = entry.path, - pinned = entry.pinned, - repo = entry.repo, - tree_hash = entry.tree_hash, # TODO should tree_hash be changed too? - version = something(entry.version, VersionSpec()) - )) + push!( + pkgs, PackageSpec( + uuid = uuid, + name = entry.name, + path = entry.path, + pinned = entry.pinned, + repo = entry.repo, + tree_hash = entry.tree_hash, # TODO should tree_hash be changed too? + version = something(entry.version, VersionSpec()) + ) + ) end return pkgs end function targeted_resolve_up(env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, preserve::PreserveLevel, julia_version) - pkgs = load_manifest_deps_up(env, pkgs; preserve=preserve) + pkgs = load_manifest_deps_up(env, pkgs; preserve = preserve) check_registered(registries, pkgs) deps_map = resolve_versions!(env, registries, pkgs, julia_version, preserve == PRESERVE_ALL_INSTALLED) return pkgs, deps_map end -function up(ctx::Context, pkgs::Vector{PackageSpec}, level::UpgradeLevel; - skip_writing_project::Bool=false, preserve::Union{Nothing,PreserveLevel}=nothing) +function up( + ctx::Context, pkgs::Vector{PackageSpec}, level::UpgradeLevel; + skip_writing_project::Bool = false, preserve::Union{Nothing, PreserveLevel} = nothing + ) requested_pkgs = pkgs @@ -2012,9 +2088,9 @@ function up(ctx::Context, pkgs::Vector{PackageSpec}, level::UpgradeLevel; update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version) new_apply = download_source(ctx) fixups_from_projectfile!(ctx) - download_artifacts(ctx, julia_version=ctx.julia_version) + download_artifacts(ctx, julia_version = ctx.julia_version) write_env(ctx.env; skip_writing_project) # write env before building - show_update(ctx.env, ctx.registries; io=ctx.io, hidden_upgrades_info = true) + show_update(ctx.env, ctx.registries; io = ctx.io, hidden_upgrades_info = true) if length(requested_pkgs) == 1 pkg = only(requested_pkgs) @@ -2027,20 +2103,22 @@ function up(ctx::Context, pkgs::Vector{PackageSpec}, level::UpgradeLevel; # Check if version didn't change and there's a newer version available if current_version == original_version && current_version !== nothing - temp_pkg = PackageSpec(name=pkg.name, uuid=pkg.uuid, version=current_version) + temp_pkg = PackageSpec(name = pkg.name, uuid = pkg.uuid, version = current_version) cinfo = status_compat_info(temp_pkg, ctx.env, ctx.registries) if cinfo !== nothing packages_holding_back, max_version, max_version_compat = cinfo if current_version < max_version - printpkgstyle(ctx.io, :Info, "$(pkg.name) can be updated but at the cost of downgrading other packages. " * - "To force upgrade to the latest version, try `add $(pkg.name)@$(max_version)`", color=Base.info_color()) + printpkgstyle( + ctx.io, :Info, "$(pkg.name) can be updated but at the cost of downgrading other packages. " * + "To force upgrade to the latest version, try `add $(pkg.name)@$(max_version)`", color = Base.info_color() + ) end end end end end - build_versions(ctx, union(new_apply, new_git)) + return build_versions(ctx, union(new_apply, new_git)) end function update_package_pin!(registries::Vector{Registry.RegistryInstance}, pkg::PackageSpec, entry::Union{Nothing, PackageEntry}) @@ -2084,10 +2162,10 @@ function pin(ctx::Context, pkgs::Vector{PackageSpec}) update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version) new = download_source(ctx) fixups_from_projectfile!(ctx) - download_artifacts(ctx; julia_version=ctx.julia_version) + download_artifacts(ctx; julia_version = ctx.julia_version) write_env(ctx.env) # write env before building - show_update(ctx.env, ctx.registries; io=ctx.io) - build_versions(ctx, new) + show_update(ctx.env, ctx.registries; io = ctx.io) + return build_versions(ctx, new) end function update_package_free!(registries::Vector{Registry.RegistryInstance}, pkg::PackageSpec, entry::PackageEntry, err_if_free::Bool) @@ -2107,22 +2185,24 @@ function update_package_free!(registries::Vector{Registry.RegistryInstance}, pkg return # -> name, uuid end if err_if_free - pkgerror("expected package $(err_rep(pkg)) to be pinned, tracking a path,", - " or tracking a repository") + pkgerror( + "expected package $(err_rep(pkg)) to be pinned, tracking a path,", + " or tracking a repository" + ) end return end # TODO: this is two technically different operations with the same name # split into two subfunctions ... -function free(ctx::Context, pkgs::Vector{PackageSpec}; err_if_free=true) +function free(ctx::Context, pkgs::Vector{PackageSpec}; err_if_free = true) for pkg in pkgs entry = manifest_info(ctx.env.manifest, pkg.uuid) delete!(ctx.env.project.sources, pkg.name) update_package_free!(ctx.registries, pkg, entry, err_if_free) end - if any(pkg -> pkg.version == VersionSpec(), pkgs) + return if any(pkg -> pkg.version == VersionSpec(), pkgs) pkgs = load_direct_deps(ctx.env, pkgs) check_registered(ctx.registries, pkgs) @@ -2134,28 +2214,28 @@ function free(ctx::Context, pkgs::Vector{PackageSpec}; err_if_free=true) fixups_from_projectfile!(ctx) download_artifacts(ctx) write_env(ctx.env) # write env before building - show_update(ctx.env, ctx.registries; io=ctx.io) + show_update(ctx.env, ctx.registries; io = ctx.io) build_versions(ctx, new) else foreach(pkg -> manifest_info(ctx.env.manifest, pkg.uuid).pinned = false, pkgs) write_env(ctx.env) - show_update(ctx.env, ctx.registries; io=ctx.io) + show_update(ctx.env, ctx.registries; io = ctx.io) end end function gen_test_code(source_path::String; test_args::Cmd) test_file = testfile(source_path) return """ - $(Base.load_path_setup_code(false)) - cd($(repr(dirname(test_file)))) - append!(empty!(ARGS), $(repr(test_args.exec))) - include($(repr(test_file))) - """ + $(Base.load_path_setup_code(false)) + cd($(repr(dirname(test_file)))) + append!(empty!(ARGS), $(repr(test_args.exec))) + include($(repr(test_file))) + """ end function get_threads_spec() - if haskey(ENV, "JULIA_NUM_THREADS") + return if haskey(ENV, "JULIA_NUM_THREADS") if isempty(ENV["JULIA_NUM_THREADS"]) throw(ArgumentError("JULIA_NUM_THREADS is set to an empty string. It is not clear what Pkg.test should set for `-t` on the test worker.")) end @@ -2194,7 +2274,7 @@ end function with_temp_env(fn::Function, temp_env::String) load_path = copy(LOAD_PATH) active_project = Base.ACTIVE_PROJECT[] - try + return try push!(empty!(LOAD_PATH), "@", temp_env) Base.ACTIVE_PROJECT[] = nothing fn() @@ -2209,8 +2289,10 @@ function sandbox_preserve(env::EnvCache, target::PackageSpec, test_project::Stri env = deepcopy(env) # include root in manifest (in case any dependencies point back to it) if env.pkg !== nothing - env.manifest[env.pkg.uuid] = PackageEntry(;name=env.pkg.name, path=dirname(env.project_file), - deps=env.project.deps) + env.manifest[env.pkg.uuid] = PackageEntry(; + name = env.pkg.name, path = dirname(env.project_file), + deps = env.project.deps + ) end # if the source manifest is an old format, upgrade the manifest_format so # that warnings aren't thrown for the temp sandbox manifest @@ -2245,16 +2327,18 @@ function abspath!(env::EnvCache, project::Project) end # ctx + pkg used to compute parent dep graph -function sandbox(fn::Function, ctx::Context, target::PackageSpec, - sandbox_path::String, sandbox_project_override; - preferences::Union{Nothing,Dict{String,Any}} = nothing, - force_latest_compatible_version::Bool=false, - allow_earlier_backwards_compatible_versions::Bool=true, - allow_reresolve::Bool=true) +function sandbox( + fn::Function, ctx::Context, target::PackageSpec, + sandbox_path::String, sandbox_project_override; + preferences::Union{Nothing, Dict{String, Any}} = nothing, + force_latest_compatible_version::Bool = false, + allow_earlier_backwards_compatible_versions::Bool = true, + allow_reresolve::Bool = true + ) sandbox_project = projectfile_path(sandbox_path) - mktempdir() do tmp - tmp_project = projectfile_path(tmp) + return mktempdir() do tmp + tmp_project = projectfile_path(tmp) tmp_manifest = manifestfile_path(tmp) tmp_preferences = joinpath(tmp, first(Base.preferences_names)) @@ -2322,9 +2406,9 @@ function sandbox(fn::Function, ctx::Context, target::PackageSpec, end try - Pkg.resolve(temp_ctx; io=devnull, skip_writing_project=true) + Pkg.resolve(temp_ctx; io = devnull, skip_writing_project = true) @debug "Using _parent_ dep graph" - catch err# TODO + catch err # TODO err isa Resolve.ResolverError || rethrow() allow_reresolve || rethrow() @debug err @@ -2336,8 +2420,8 @@ function sandbox(fn::Function, ctx::Context, target::PackageSpec, "then you probably want to pass the `allow_reresolve = false` kwarg ", "when calling the `Pkg.test` function.", ) - printpkgstyle(ctx.io, :Test, msg, color=Base.warn_color()) - Pkg.update(temp_ctx; skip_writing_project=true, update_registry=false, io=ctx.io) + printpkgstyle(ctx.io, :Test, msg, color = Base.warn_color()) + Pkg.update(temp_ctx; skip_writing_project = true, update_registry = false, io = ctx.io) printpkgstyle(ctx.io, :Test, "Successfully re-resolved") @debug "Using _clean_ dep graph" end @@ -2376,7 +2460,7 @@ function gen_target_project(ctx::Context, pkg::PackageSpec, source_path::String, env = ctx.env registries = ctx.registries test_project = Types.Project() - if projectfile_path(source_path; strict=true) === nothing + if projectfile_path(source_path; strict = true) === nothing # no project file, assuming this is an old REQUIRE package test_project.deps = copy(env.manifest[pkg.uuid].deps) if target == "test" @@ -2384,10 +2468,10 @@ function gen_target_project(ctx::Context, pkg::PackageSpec, source_path::String, if isfile(test_REQUIRE_path) @warn "using test/REQUIRE files is deprecated and current support is lacking in some areas" test_pkgs = parse_REQUIRE(test_REQUIRE_path) - package_specs = [PackageSpec(name=pkg) for pkg in test_pkgs] + package_specs = [PackageSpec(name = pkg) for pkg in test_pkgs] registry_resolve!(registries, package_specs) stdlib_resolve!(package_specs) - ensure_resolved(ctx, env.manifest, package_specs, registry=true) + ensure_resolved(ctx, env.manifest, package_specs, registry = true) for spec in package_specs test_project.deps[spec.name] = spec.uuid end @@ -2423,12 +2507,14 @@ end testdir(source_path::String) = joinpath(source_path, "test") testfile(source_path::String) = joinpath(testdir(source_path), "runtests.jl") -function test(ctx::Context, pkgs::Vector{PackageSpec}; - coverage=false, julia_args::Cmd=``, test_args::Cmd=``, - test_fn=nothing, - force_latest_compatible_version::Bool=false, - allow_earlier_backwards_compatible_versions::Bool=true, - allow_reresolve::Bool=true) +function test( + ctx::Context, pkgs::Vector{PackageSpec}; + coverage = false, julia_args::Cmd = ``, test_args::Cmd = ``, + test_fn = nothing, + force_latest_compatible_version::Bool = false, + allow_earlier_backwards_compatible_versions::Bool = true, + allow_reresolve::Bool = true + ) Pkg.instantiate(ctx; allow_autoprecomp = false) # do precomp later within sandbox # load manifest data @@ -2449,16 +2535,18 @@ function test(ctx::Context, pkgs::Vector{PackageSpec}; # See if we can find the test files for all packages missing_runtests = String[] - source_paths = String[] # source_path is the package root (not /src) + source_paths = String[] # source_path is the package root (not /src) for pkg in pkgs sourcepath = project_rel_path(ctx.env, source_path(ctx.env.manifest_file, pkg, ctx.julia_version)) # TODO !isfile(testfile(sourcepath)) && push!(missing_runtests, pkg.name) push!(source_paths, sourcepath) end if !isempty(missing_runtests) - pkgerror(length(missing_runtests) == 1 ? "Package " : "Packages ", - join(missing_runtests, ", "), - " did not provide a `test/runtests.jl` file") + pkgerror( + length(missing_runtests) == 1 ? "Package " : "Packages ", + join(missing_runtests, ", "), + " did not provide a `test/runtests.jl` file" + ) end # sandbox @@ -2470,15 +2558,15 @@ function test(ctx::Context, pkgs::Vector{PackageSpec}; proj = Base.locate_project_file(abspath(testdir(source_path))) env = EnvCache(proj) # Instantiate test env - Pkg.instantiate(Context(env=env); allow_autoprecomp = false) - status(env, ctx.registries; mode=PKGMODE_COMBINED, io=ctx.io, ignore_indent = false, show_usagetips = false) + Pkg.instantiate(Context(env = env); allow_autoprecomp = false) + status(env, ctx.registries; mode = PKGMODE_COMBINED, io = ctx.io, ignore_indent = false, show_usagetips = false) flags = gen_subprocess_flags(source_path; coverage, julia_args) if should_autoprecompile() cacheflags = Base.CacheFlags(parse(UInt8, read(`$(Base.julia_cmd()) $(flags) --eval 'show(ccall(:jl_cache_flags, UInt8, ()))'`, String))) # Don't warn about already loaded packages, since we are going to run tests in a new # subprocess anyway. - Pkg.precompile(; io=ctx.io, warn_loaded = false, configs = flags => cacheflags) + Pkg.precompile(; io = ctx.io, warn_loaded = false, configs = flags => cacheflags) end printpkgstyle(ctx.io, :Testing, "Running tests...") @@ -2513,15 +2601,15 @@ function test(ctx::Context, pkgs::Vector{PackageSpec}; end # now we sandbox printpkgstyle(ctx.io, :Testing, pkg.name) - sandbox(ctx, pkg, testdir(source_path), test_project_override; preferences=test_project_preferences, force_latest_compatible_version, allow_earlier_backwards_compatible_versions, allow_reresolve) do + sandbox(ctx, pkg, testdir(source_path), test_project_override; preferences = test_project_preferences, force_latest_compatible_version, allow_earlier_backwards_compatible_versions, allow_reresolve) do test_fn !== nothing && test_fn() - sandbox_ctx = Context(;io=ctx.io) - status(sandbox_ctx.env, sandbox_ctx.registries; mode=PKGMODE_COMBINED, io=sandbox_ctx.io, ignore_indent = false, show_usagetips = false) + sandbox_ctx = Context(; io = ctx.io) + status(sandbox_ctx.env, sandbox_ctx.registries; mode = PKGMODE_COMBINED, io = sandbox_ctx.io, ignore_indent = false, show_usagetips = false) flags = gen_subprocess_flags(source_path; coverage, julia_args) if should_autoprecompile() cacheflags = Base.CacheFlags(parse(UInt8, read(`$(Base.julia_cmd()) $(flags) --eval 'show(ccall(:jl_cache_flags, UInt8, ()))'`, String))) - Pkg.precompile(sandbox_ctx; io=sandbox_ctx.io, configs = flags => cacheflags) + Pkg.precompile(sandbox_ctx; io = sandbox_ctx.io, configs = flags => cacheflags) end printpkgstyle(ctx.io, :Testing, "Running tests...") @@ -2539,7 +2627,7 @@ function test(ctx::Context, pkgs::Vector{PackageSpec}; # TODO: Should be included in Base function signal_name(signal::Integer) - if signal == Base.SIGHUP + return if signal == Base.SIGHUP "HUP" elseif signal == Base.SIGINT "INT" @@ -2557,9 +2645,9 @@ function test(ctx::Context, pkgs::Vector{PackageSpec}; end # report errors - if !isempty(pkgs_errored) + return if !isempty(pkgs_errored) function reason(p) - if Base.process_signaled(p) + return if Base.process_signaled(p) " (received signal: " * signal_name(p.termsignal) * ")" elseif Base.process_exited(p) && p.exitcode != 1 " (exit code: " * string(p.exitcode) * ")" @@ -2608,7 +2696,7 @@ end # Display -function stat_rep(x::PackageSpec; name=true) +function stat_rep(x::PackageSpec; name = true) name = name ? "$(x.name)" : "" version = x.version == VersionSpec() ? "" : "v$(x.version)" rev = "" @@ -2619,7 +2707,7 @@ function stat_rep(x::PackageSpec; name=true) repo = Operations.is_tracking_repo(x) ? "`$(x.repo.source)$(subdir_str)#$(rev)`" : "" path = Operations.is_tracking_path(x) ? "$(pathrepr(x.path))" : "" pinned = x.pinned ? "⚲" : "" - return join(filter(!isempty, [name,version,repo,path,pinned]), " ") + return join(filter(!isempty, [name, version, repo, path, pinned]), " ") end print_single(io::IO, pkg::PackageSpec) = print(io, stat_rep(pkg)) @@ -2627,20 +2715,20 @@ print_single(io::IO, pkg::PackageSpec) = print(io, stat_rep(pkg)) is_instantiated(::Nothing) = false is_instantiated(x::PackageSpec) = x.version != VersionSpec() || is_stdlib(x.uuid) # Compare an old and new node of the dependency graph and print a single line to summarize the change -function print_diff(io::IO, old::Union{Nothing,PackageSpec}, new::Union{Nothing,PackageSpec}) - if !is_instantiated(old) && is_instantiated(new) - printstyled(io, "+ $(stat_rep(new))"; color=:light_green) +function print_diff(io::IO, old::Union{Nothing, PackageSpec}, new::Union{Nothing, PackageSpec}) + return if !is_instantiated(old) && is_instantiated(new) + printstyled(io, "+ $(stat_rep(new))"; color = :light_green) elseif !is_instantiated(new) - printstyled(io, "- $(stat_rep(old))"; color=:light_red) + printstyled(io, "- $(stat_rep(old))"; color = :light_red) elseif is_tracking_registry(old) && is_tracking_registry(new) && - new.version isa VersionNumber && old.version isa VersionNumber && new.version != old.version + new.version isa VersionNumber && old.version isa VersionNumber && new.version != old.version if new.version > old.version - printstyled(io, "↑ $(stat_rep(old)) ⇒ $(stat_rep(new; name=false))"; color=:light_yellow) + printstyled(io, "↑ $(stat_rep(old)) ⇒ $(stat_rep(new; name = false))"; color = :light_yellow) else - printstyled(io, "↓ $(stat_rep(old)) ⇒ $(stat_rep(new; name=false))"; color=:light_magenta) + printstyled(io, "↓ $(stat_rep(old)) ⇒ $(stat_rep(new; name = false))"; color = :light_magenta) end else - printstyled(io, "~ $(stat_rep(old)) ⇒ $(stat_rep(new; name=false))"; color=:light_yellow) + printstyled(io, "~ $(stat_rep(old)) ⇒ $(stat_rep(new; name = false))"; color = :light_yellow) end end @@ -2656,11 +2744,11 @@ function status_compat_info(pkg::PackageSpec, env::EnvCache, regs::Vector{Regist reg_compat_info = Registry.compat_info(info) versions = keys(reg_compat_info) versions = filter(v -> !Registry.isyanked(info, v), versions) - max_version_reg = maximum(versions; init=v"0") + max_version_reg = maximum(versions; init = v"0") max_version = max(max_version, max_version_reg) compat_spec = get_compat_workspace(env, pkg.name) versions_in_compat = filter(in(compat_spec), keys(reg_compat_info)) - max_version_in_compat = max(max_version_in_compat, maximum(versions_in_compat; init=v"0")) + max_version_in_compat = max(max_version_in_compat, maximum(versions_in_compat; init = v"0")) end max_version == v"0" && return nothing pkg.version >= max_version && return nothing @@ -2729,7 +2817,7 @@ function status_compat_info(pkg::PackageSpec, env::EnvCache, regs::Vector{Regist return sort!(unique!(packages_holding_back)), max_version, max_version_in_compat end -function diff_array(old_env::Union{EnvCache,Nothing}, new_env::EnvCache; manifest=true, workspace=false) +function diff_array(old_env::Union{EnvCache, Nothing}, new_env::EnvCache; manifest = true, workspace = false) function index_pkgs(pkgs, uuid) idx = findfirst(pkg -> pkg.uuid == uuid, pkgs) return idx === nothing ? nothing : pkgs[idx] @@ -2741,9 +2829,9 @@ function diff_array(old_env::Union{EnvCache,Nothing}, new_env::EnvCache; manifes new = manifest ? load_all_deps_loadable(new_env) : load_project_deps(new_env.project, new_env.project_file, new_env.manifest, new_env.manifest_file) end - T, S = Union{UUID,Nothing}, Union{PackageSpec,Nothing} + T, S = Union{UUID, Nothing}, Union{PackageSpec, Nothing} if old_env === nothing - return Tuple{T,S,S}[(pkg.uuid, nothing, pkg)::Tuple{T,S,S} for pkg in new] + return Tuple{T, S, S}[(pkg.uuid, nothing, pkg)::Tuple{T, S, S} for pkg in new] end if workspace old = manifest ? load_all_deps(old_env) : load_direct_deps(old_env) @@ -2752,10 +2840,10 @@ function diff_array(old_env::Union{EnvCache,Nothing}, new_env::EnvCache; manifes end # merge old and new into single array all_uuids = union(T[pkg.uuid for pkg in old], T[pkg.uuid for pkg in new]) - return Tuple{T,S,S}[(uuid, index_pkgs(old, uuid), index_pkgs(new, uuid))::Tuple{T,S,S} for uuid in all_uuids] + return Tuple{T, S, S}[(uuid, index_pkgs(old, uuid), index_pkgs(new, uuid))::Tuple{T, S, S} for uuid in all_uuids] end -function is_package_downloaded(manifest_file::String, pkg::PackageSpec; platform=HostPlatform()) +function is_package_downloaded(manifest_file::String, pkg::PackageSpec; platform = HostPlatform()) sourcepath = source_path(manifest_file, pkg) sourcepath === nothing && return false isdir(sourcepath) || return false @@ -2777,11 +2865,13 @@ function status_ext_info(pkg::PackageSpec, env::EnvCache) # Note: `get_extension` returns nothing for stdlibs that are loaded via `require_stdlib` ext_loaded = (Base.get_extension(Base.PkgId(pkg.uuid, pkg.name), Symbol(ext)) !== nothing) # Check if deps are loaded - extdeps_info= Tuple{String, Bool}[] + extdeps_info = Tuple{String, Bool}[] for extdep in extdeps if !(haskey(weakdepses, extdep) || haskey(depses, extdep)) - pkgerror(isnothing(pkg.name) ? "M" : "$(pkg.name) has a malformed Project.toml, ", - "the extension package $extdep is not listed in [weakdeps] or [deps]") + pkgerror( + isnothing(pkg.name) ? "M" : "$(pkg.name) has a malformed Project.toml, ", + "the extension package $extdep is not listed in [weakdeps] or [deps]" + ) end uuid = get(weakdepses, extdep, nothing) if uuid === nothing @@ -2813,29 +2903,35 @@ struct PackageStatusData extinfo::Union{Nothing, Vector{ExtInfo}} end -function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registries::Vector{Registry.RegistryInstance}, header::Symbol, - uuids::Vector, names::Vector; manifest=true, diff=false, ignore_indent::Bool, workspace::Bool, outdated::Bool, extensions::Bool, io::IO, - mode::PackageMode, hidden_upgrades_info::Bool, show_usagetips::Bool=true) - not_installed_indicator = sprint((io, args) -> printstyled(io, args...; color=Base.error_color()), "→", context=io) - upgradable_indicator = sprint((io, args) -> printstyled(io, args...; color=:green), "⌃", context=io) - heldback_indicator = sprint((io, args) -> printstyled(io, args...; color=Base.warn_color()), "⌅", context=io) +function print_status( + env::EnvCache, old_env::Union{Nothing, EnvCache}, registries::Vector{Registry.RegistryInstance}, header::Symbol, + uuids::Vector, names::Vector; manifest = true, diff = false, ignore_indent::Bool, workspace::Bool, outdated::Bool, extensions::Bool, io::IO, + mode::PackageMode, hidden_upgrades_info::Bool, show_usagetips::Bool = true + ) + not_installed_indicator = sprint((io, args) -> printstyled(io, args...; color = Base.error_color()), "→", context = io) + upgradable_indicator = sprint((io, args) -> printstyled(io, args...; color = :green), "⌃", context = io) + heldback_indicator = sprint((io, args) -> printstyled(io, args...; color = Base.warn_color()), "⌅", context = io) filter = !isempty(uuids) || !isempty(names) # setup xs = diff_array(old_env, env; manifest, workspace) # filter and return early if possible if isempty(xs) && !diff - printpkgstyle(io, header, "$(pathrepr(manifest ? env.manifest_file : env.project_file)) (empty " * - (manifest ? "manifest" : "project") * ")", ignore_indent) + printpkgstyle( + io, header, "$(pathrepr(manifest ? env.manifest_file : env.project_file)) (empty " * + (manifest ? "manifest" : "project") * ")", ignore_indent + ) return nothing end - no_changes = all(p-> p[2] == p[3], xs) + no_changes = all(p -> p[2] == p[3], xs) if no_changes printpkgstyle(io, Symbol("No packages added to or removed from"), "$(pathrepr(manifest ? env.manifest_file : env.project_file))", ignore_indent) else xs = !filter ? xs : eltype(xs)[(id, old, new) for (id, old, new) in xs if (id in uuids || something(new, old).name in names)] if isempty(xs) - printpkgstyle(io, Symbol("No Matches"), - "in $(diff ? "diff for " : "")$(pathrepr(manifest ? env.manifest_file : env.project_file))", ignore_indent) + printpkgstyle( + io, Symbol("No Matches"), + "in $(diff ? "diff for " : "")$(pathrepr(manifest ? env.manifest_file : env.project_file))", ignore_indent + ) return nothing end # main print @@ -2936,23 +3032,23 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie # show if package is yanked pkg_spec = something(pkg.new, pkg.old) if is_pkgversion_yanked(pkg_spec, registries) - printstyled(io, " [yanked]"; color=:yellow) + printstyled(io, " [yanked]"; color = :yellow) end if outdated && !diff && pkg.compat_data !== nothing packages_holding_back, max_version, max_version_compat = pkg.compat_data if pkg.new.version !== max_version_compat && max_version_compat != max_version - printstyled(io, " [